diff --git a/cli/app/deploy.go b/cli/app/deploy.go index f4bfb390..259fba73 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -166,7 +166,7 @@ recipes. app.Env[k] = v } - composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env) + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/new.go b/cli/app/new.go index cba41dec..34ee69ea 100644 --- a/cli/app/new.go +++ b/cli/app/new.go @@ -2,10 +2,8 @@ package app import ( "fmt" - "path" "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" @@ -53,6 +51,7 @@ var appNewCommand = cli.Command{ internal.PassFlag, internal.SecretsFlag, internal.OfflineFlag, + internal.ChaosFlag, }, Before: internal.SubCommandBefore, ArgsUsage: "[]", @@ -60,14 +59,18 @@ var appNewCommand = cli.Command{ Action: func(c *cli.Context) error { recipe := internal.ValidateRecipe(c) - if !internal.Offline { - if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil { + if !internal.Chaos { + if err := recipePkg.EnsureIsClean(recipe.Name); err != nil { + logrus.Fatal(err) + } + if !internal.Offline { + if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil { + logrus.Fatal(err) + } + } + if err := recipePkg.EnsureLatest(recipe.Name); err != nil { logrus.Fatal(err) } - } - - if err := recipePkg.EnsureLatest(recipe.Name); err != nil { - logrus.Fatal(err) } if err := ensureServerFlag(); err != nil { @@ -90,29 +93,43 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } - if err := promptForSecrets(internal.Domain); err != nil { - logrus.Fatal(err) - } - var secrets AppSecrets var secretTable *jsontable.JSONTable if internal.Secrets { + sampleEnv, err := recipe.SampleEnv() + if err != nil { + logrus.Fatal(err) + } + + composeFiles, err := config.GetComposeFiles(recipe.Name, sampleEnv) + if err != nil { + logrus.Fatal(err) + } + + secretsConfig, err := secret.ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name) + if err != nil { + return err + } + + if err := promptForSecrets(recipe.Name, secretsConfig); err != nil { + logrus.Fatal(err) + } + cl, err := client.New(internal.NewAppServer) if err != nil { logrus.Fatal(err) } - secrets, err := createSecrets(cl, sanitisedAppName) + secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName) if err != nil { logrus.Fatal(err) } secretCols := []string{"Name", "Value"} secretTable = formatter.CreateTable(secretCols) - for secret := range secrets { - secretTable.Append([]string{secret, secrets[secret]}) + for name, val := range secrets { + secretTable.Append([]string{name, val}) } - } if internal.NewAppServer == "default" { @@ -123,7 +140,6 @@ var appNewCommand = cli.Command{ table := formatter.CreateTable(tableCol) table.Append([]string{internal.NewAppServer, recipe.Name, internal.Domain}) - fmt.Println("") fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name)) fmt.Println("") table.Render() @@ -133,14 +149,13 @@ var appNewCommand = cli.Command{ fmt.Println("") fmt.Println("You can deploy this app by running the following:") fmt.Println(fmt.Sprintf("\n abra app deploy %s", internal.Domain)) - fmt.Println("") if len(secrets) > 0 { + fmt.Println("") fmt.Println("Here are your generated secrets:") fmt.Println("") secretTable.Render() - fmt.Println("") - logrus.Warn("generated secrets are not shown again, please take note of them *now*") + logrus.Warn("generated secrets are not shown again, please take note of them NOW") } return nil @@ -151,21 +166,14 @@ var appNewCommand = cli.Command{ type AppSecrets map[string]string // createSecrets creates all secrets for a new app. -func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets, error) { - appEnvPath := path.Join( - config.ABRA_DIR, - "servers", - internal.NewAppServer, - fmt.Sprintf("%s.env", internal.Domain), - ) - - appEnv, err := config.ReadEnv(appEnvPath) - if err != nil { - return nil, err +func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) { + // NOTE(d1): trim to match app.StackName() implementation + if len(sanitisedAppName) > 45 { + logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45]) + sanitisedAppName = sanitisedAppName[:45] } - secretEnvVars := secret.ReadSecretEnvVars(appEnv) - secrets, err := secret.GenerateSecrets(cl, secretEnvVars, sanitisedAppName, internal.NewAppServer) + secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer) if err != nil { return nil, err } @@ -183,6 +191,7 @@ func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets } } } + return secrets, nil } @@ -206,15 +215,9 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error { } // promptForSecrets asks if we should generate secrets for a new app. -func promptForSecrets(appName string) error { - app, err := app.Get(appName) - if err != nil { - return err - } - - secretEnvVars := secret.ReadSecretEnvVars(app.Env) - if len(secretEnvVars) == 0 { - logrus.Debugf("%s has no secrets to generate, skipping...", app.Recipe) +func promptForSecrets(recipeName string, secretsConfig map[string]string) error { + if len(secretsConfig) == 0 { + logrus.Debugf("%s has no secrets to generate, skipping...", recipeName) return nil } diff --git a/cli/app/rollback.go b/cli/app/rollback.go index feec6b78..c8104877 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -42,7 +42,7 @@ useful if the container runtime has gotten into a weird state. This action could be destructive, please ensure you have a copy of your app data beforehand. -Chas mode ("--chaos") will deploy your local checkout of a recipe as-is, +Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is, including unstaged changes and can be useful for live hacking and testing new recipes. `, @@ -202,7 +202,7 @@ recipes. app.Env[k] = v } - composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env) + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/secret.go b/cli/app/secret.go index a638cccf..31b86c23 100644 --- a/cli/app/secret.go +++ b/cli/app/secret.go @@ -12,6 +12,7 @@ import ( "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/secret" "github.com/docker/docker/api/types" dockerClient "github.com/docker/docker/client" @@ -42,12 +43,35 @@ var appSecretGenerateCommand = cli.Command{ internal.DebugFlag, allSecretsFlag, internal.PassFlag, + internal.MachineReadableFlag, + internal.OfflineFlag, + internal.ChaosFlag, }, Before: internal.SubCommandBefore, BashComplete: autocomplete.AppNameComplete, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Chaos { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Offline { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + if err := recipe.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + if len(c.Args()) == 1 && !allSecrets { err := errors.New("missing arguments / or '--all'") internal.ShowSubcommandHelpAndError(c, err) @@ -58,18 +82,26 @@ var appSecretGenerateCommand = cli.Command{ internal.ShowSubcommandHelpAndError(c, err) } + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + if err != nil { + logrus.Fatal(err) + } + + secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe) + if err != nil { + logrus.Fatal(err) + } + secretsToCreate := make(map[string]string) - secretEnvVars := secret.ReadSecretEnvVars(app.Env) if allSecrets { - secretsToCreate = secretEnvVars + secretsToCreate = secretsConfig } else { secretName := c.Args().Get(1) secretVersion := c.Args().Get(2) matches := false - for sec := range secretEnvVars { - parsed := secret.ParseSecretEnvVarName(sec) - if secretName == parsed { - secretsToCreate[sec] = secretVersion + for name := range secretsConfig { + if secretName == name { + secretsToCreate[name] = secretVersion matches = true } } @@ -107,8 +139,13 @@ var appSecretGenerateCommand = cli.Command{ for name, val := range secretVals { table.Append([]string{name, val}) } - table.Render() - logrus.Warn("generated secrets are not shown again, please take note of them *now*") + + if internal.MachineReadable { + table.JSONRender() + } else { + table.Render() + } + logrus.Warn("generated secrets are not shown again, please take note of them NOW") return nil }, @@ -198,6 +235,8 @@ var appSecretRmCommand = cli.Command{ internal.NoInputFlag, rmAllSecretsFlag, internal.PassRemoveFlag, + internal.OfflineFlag, + internal.ChaosFlag, }, Before: internal.SubCommandBefore, ArgsUsage: " []", @@ -211,7 +250,36 @@ Example: `, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) - secrets := secret.ReadSecretEnvVars(app.Env) + + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Chaos { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Offline { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + if err := recipe.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + if err != nil { + logrus.Fatal(err) + } + + secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe) + if err != nil { + logrus.Fatal(err) + } if c.Args().Get(1) != "" && rmAllSecrets { internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '' and '--all' together")) @@ -243,15 +311,13 @@ Example: match := false secretToRm := c.Args().Get(1) - for sec := range secrets { - secretName := secret.ParseSecretEnvVarName(sec) - - secVal, err := secret.ParseSecretEnvVarValue(secrets[sec]) + for secretName, secretValue := range secretsConfig { + val, err := secret.ParseSecretValue(secretValue) if err != nil { logrus.Fatal(err) } - secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version) + secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) if _, ok := remoteSecretNames[secretRemoteName]; ok { if secretToRm != "" { if secretName == secretToRm { @@ -288,13 +354,44 @@ var appSecretLsCommand = cli.Command{ Aliases: []string{"ls"}, Flags: []cli.Flag{ internal.DebugFlag, + internal.OfflineFlag, + internal.ChaosFlag, }, Before: internal.SubCommandBefore, Usage: "List all secrets", BashComplete: autocomplete.AppNameComplete, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) - secrets := secret.ReadSecretEnvVars(app.Env) + + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Chaos { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Offline { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + if err := recipe.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + if err != nil { + logrus.Fatal(err) + } + + secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe) + if err != nil { + logrus.Fatal(err) + } tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"} table := formatter.CreateTable(tableCol) @@ -319,18 +416,17 @@ var appSecretLsCommand = cli.Command{ remoteSecretNames[cont.Spec.Annotations.Name] = true } - for sec := range secrets { + for secretName, secretValue := range secretsConfig { createdRemote := false - secretName := secret.ParseSecretEnvVarName(sec) - secVal, err := secret.ParseSecretEnvVarValue(secrets[sec]) + val, err := secret.ParseSecretValue(secretValue) if err != nil { logrus.Fatal(err) } - secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version) + secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) if _, ok := remoteSecretNames[secretRemoteName]; ok { createdRemote = true } - tableRow := []string{secretName, secVal.Version, secretRemoteName, strconv.FormatBool(createdRemote)} + tableRow := []string{secretName, val.Version, secretRemoteName, strconv.FormatBool(createdRemote)} table.Append(tableRow) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index bc31f484..845c15fd 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -47,7 +47,7 @@ useful if the container runtime has gotten into a weird state. This action could be destructive, please ensure you have a copy of your app data beforehand. -Chas mode ("--chaos") will deploy your local checkout of a recipe as-is, +Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is, including unstaged changes and can be useful for live hacking and testing new recipes. `, @@ -234,7 +234,7 @@ recipes. app.Env[k] = v } - composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env) + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/pkg/config/app.go b/pkg/config/app.go index 0ad683d2..95dc7784 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -76,7 +76,7 @@ func (a App) StackName() string { func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) { filters := filters.NewArgs() - composeFiles, err := GetAppComposeFiles(a.Recipe, a.Env) + composeFiles, err := GetComposeFiles(a.Recipe, a.Env) if err != nil { return filters, err } @@ -277,7 +277,7 @@ func GetAppServiceNames(appName string) ([]string, error) { return serviceNames, err } - composeFiles, err := GetAppComposeFiles(app.Recipe, app.Env) + composeFiles, err := GetComposeFiles(app.Recipe, app.Env) if err != nil { return serviceNames, err } @@ -437,9 +437,10 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str return statuses, nil } -// GetAppComposeFiles gets the list of compose files for an app which should be -// merged into a composetypes.Config while respecting the COMPOSE_FILE env var. -func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) { +// GetComposeFiles gets the list of compose files for an app (or recipe if you +// don't already have an app) which should be merged into a composetypes.Config +// while respecting the COMPOSE_FILE env var. +func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) { var composeFiles []string if _, ok := appEnv["COMPOSE_FILE"]; !ok { diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index aed3da4e..622d24c7 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -249,6 +249,15 @@ func Get(recipeName string, offline bool) (Recipe, error) { }, nil } +func (r Recipe) SampleEnv() (map[string]string, error) { + envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") + sampleEnv, err := config.ReadEnv(envSamplePath) + if err != nil { + return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name) + } + return sampleEnv, nil +} + // EnsureExists ensures that a recipe is locally cloned func EnsureExists(recipeName string) error { recipeDir := path.Join(config.RECIPES_DIR, recipeName) diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index 892b9249..e528dcb3 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -5,13 +5,14 @@ package secret import ( "fmt" - "regexp" + "slices" "strconv" "strings" "sync" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/upstream/stack" + loader "coopcloud.tech/abra/pkg/upstream/stack" "github.com/decentral1se/passgen" dockerClient "github.com/docker/docker/client" "github.com/sirupsen/logrus" @@ -60,40 +61,54 @@ func GeneratePassphrases(count uint) ([]string, error) { return passphrases, nil } -// ReadSecretEnvVars reads secret env vars from an app env var config. -func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string { - secretEnvVars := make(map[string]string) +// ReadSecretsConfig reads secret names/versions from the recipe config. The +// function generalises appEnv/composeFiles because some times you have an app +// and some times you don't (as the caller). We need to be able to handle the +// "app new" case where we pass in the .env.sample and the "secret generate" +// case where the app is created. +func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeName string) (map[string]string, error) { + secretConfigs := make(map[string]string) - for envVar := range appEnv { - regex := regexp.MustCompile(`^SECRET.*VERSION.*`) - if string(regex.Find([]byte(envVar))) != "" { - secretEnvVars[envVar] = appEnv[envVar] + opts := stack.Deploy{Composefiles: composeFiles} + config, err := loader.LoadComposefile(opts, appEnv) + if err != nil { + return secretConfigs, err + } + + var enabledSecrets []string + for _, service := range config.Services { + for _, secret := range service.Secrets { + enabledSecrets = append(enabledSecrets, secret.Source) } } - logrus.Debugf("read %s as secrets from %s", secretEnvVars, appEnv) + if len(enabledSecrets) == 0 { + logrus.Debugf("not generating app secrets, none enabled in recipe config") + return secretConfigs, nil + } - return secretEnvVars + for _, secret := range config.Secrets { + firstIdx := strings.Index(secret.Name, "_") + lastIdx := strings.LastIndex(secret.Name, "_") + secretName := secret.Name[firstIdx+1 : lastIdx] + + if secret.Name != "" && string(secret.Name[len(secret.Name)-1]) == "_" { + return secretConfigs, fmt.Errorf("missing version for secret? (%s)", secretName) + } + + if !(slices.Contains(enabledSecrets, secretName)) { + logrus.Debugf("%s not enabled in recipe config, not generating", secretName) + continue + } + + secretVersion := secret.Name[lastIdx+1:] + secretConfigs[secretName] = secretVersion + } + + return secretConfigs, nil } -func ParseSecretEnvVarName(secretEnvVar string) string { - withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_") - withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION") - name := strings.ToLower(withoutSuffix) - logrus.Debugf("parsed %s as name from %s", name, secretEnvVar) - return name -} - -func ParseGeneratedSecretName(secret string, appEnv config.App) string { - name := fmt.Sprintf("%s_", appEnv.StackName()) - withoutAppName := strings.TrimPrefix(secret, name) - idx := strings.LastIndex(withoutAppName, "_") - parsed := withoutAppName[:idx] - logrus.Debugf("parsed %s as name from %s", parsed, secret) - return parsed -} - -func ParseSecretEnvVarValue(secret string) (secretValue, error) { +func ParseSecretValue(secret string) (secretValue, error) { values := strings.Split(secret, "#") if len(values) == 0 { return secretValue{}, fmt.Errorf("unable to parse %s", secret) @@ -118,30 +133,29 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) { } // GenerateSecrets generates secrets locally and sends them to a remote server for storage. -func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, appName, server string) (map[string]string, error) { +func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, appName, server string) (map[string]string, error) { secrets := make(map[string]string) var mutex sync.Mutex var wg sync.WaitGroup - ch := make(chan error, len(secretEnvVars)) - for secretEnvVar := range secretEnvVars { + ch := make(chan error, len(secretsFromConfig)) + for n, v := range secretsFromConfig { wg.Add(1) - go func(s string) { + go func(secretName, secretValue string) { defer wg.Done() - secretName := ParseSecretEnvVarName(s) - secretValue, err := ParseSecretEnvVarValue(secretEnvVars[s]) + parsedSecretValue, err := ParseSecretValue(secretValue) if err != nil { ch <- err return } - secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version) + secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version) logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server) - if secretValue.Length > 0 { - passwords, err := GeneratePasswords(1, uint(secretValue.Length)) + if parsedSecretValue.Length > 0 { + passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length)) if err != nil { ch <- err return @@ -182,12 +196,12 @@ func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, a secrets[secretName] = passphrases[0] } ch <- nil - }(secretEnvVar) + }(n, v) } wg.Wait() - for range secretEnvVars { + for range secretsFromConfig { err := <-ch if err != nil { return nil, err