From d09aedb5fea8d77de9c061a302a3c7b559998201 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Wed, 29 Nov 2023 21:34:36 +0100 Subject: [PATCH] a little less hacky --- cli/app/new.go | 10 +-- cli/app/secret.go | 33 ++++----- pkg/secret/secret.go | 127 +++++++++++------------------------ pkg/upstream/stack/loader.go | 13 +++- 4 files changed, 69 insertions(+), 114 deletions(-) diff --git a/cli/app/new.go b/cli/app/new.go index 27364d31..023fac5a 100644 --- a/cli/app/new.go +++ b/cli/app/new.go @@ -108,7 +108,7 @@ var appNewCommand = cli.Command{ } envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample") - secretsConfig, modifiers, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name) + secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name) if err != nil { return err } @@ -122,7 +122,7 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } - secrets, err = createSecrets(cl, secretsConfig, modifiers, sanitisedAppName) + secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName) if err != nil { logrus.Fatal(err) } @@ -168,14 +168,14 @@ var appNewCommand = cli.Command{ type AppSecrets map[string]string // createSecrets creates all secrets for a new app. -func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, modifiers map[string]map[string]string, sanitisedAppName string) (AppSecrets, error) { +func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) { // NOTE(d1): trim to match app.StackName() implementation if len(sanitisedAppName) > 45 { logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45]) sanitisedAppName = sanitisedAppName[:45] } - secrets, err := secret.GenerateSecrets(cl, secretsConfig, modifiers, sanitisedAppName, internal.NewAppServer) + secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer) if err != nil { return nil, err } @@ -217,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error { } // promptForSecrets asks if we should generate secrets for a new app. -func promptForSecrets(recipeName string, secretsConfig map[string]string) error { +func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error { if len(secretsConfig) == 0 { logrus.Debugf("%s has no secrets to generate, skipping...", recipeName) return nil diff --git a/cli/app/secret.go b/cli/app/secret.go index 191625ac..d01a77b4 100644 --- a/cli/app/secret.go +++ b/cli/app/secret.go @@ -87,28 +87,24 @@ var appSecretGenerateCommand = cli.Command{ logrus.Fatal(err) } - secretsConfig, secretModifiers, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe) + secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe) if err != nil { logrus.Fatal(err) } - secretsToCreate := make(map[string]string) - if allSecrets { - secretsToCreate = secretsConfig - } else { + if !allSecrets { secretName := c.Args().Get(1) secretVersion := c.Args().Get(2) - matches := false - for name := range secretsConfig { - if secretName == name { - secretsToCreate[name] = secretVersion - matches = true + s, ok := secrets[secretName] + if !ok { + logrus.Fatalf("%s doesn't exist in the env config?", secretName) + } else { + s.Version = secretVersion + secrets = map[string]secret.SecretValue{ + secretName: s, } } - if !matches { - logrus.Fatalf("%s doesn't exist in the env config?", secretName) - } } cl, err := client.New(app.Server) @@ -116,7 +112,7 @@ var appSecretGenerateCommand = cli.Command{ logrus.Fatal(err) } - secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, secretModifiers, app.StackName(), app.Server) + secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server) if err != nil { logrus.Fatal(err) } @@ -276,7 +272,7 @@ Example: logrus.Fatal(err) } - secretsConfig, _, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe) + secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe) if err != nil { logrus.Fatal(err) } @@ -311,12 +307,7 @@ Example: match := false secretToRm := c.Args().Get(1) - for secretName, secretValue := range secretsConfig { - val, err := secret.ParseSecretValue(secretValue) - if err != nil { - logrus.Fatal(err) - } - + for secretName, val := range secrets { secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) if _, ok := remoteSecretNames[secretRemoteName]; ok { if secretToRm != "" { diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index 0eb64913..7fb96460 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -22,9 +22,9 @@ import ( "github.com/sirupsen/logrus" ) -// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config +// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config // secret definition. -type secretValue struct { +type SecretValue struct { Version string Length int } @@ -68,19 +68,20 @@ func GeneratePassphrases(count uint) ([]string, error) { // and some times you don't (as the caller). We need to be able to handle the // "app new" case where we pass in the .env.sample and the "secret generate" // case where the app is created. -func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, map[string]map[string]string, error) { - secretConfigs := make(map[string]string) - secretModifiers := make(map[string]map[string]string) - +func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) { appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true}) if err != nil { - return secretConfigs, nil, err + return nil, err } opts := stack.Deploy{Composefiles: composeFiles} config, err := loader.LoadComposefile(opts, appEnv) if err != nil { - return secretConfigs, appModifiers, err + return nil, err + } + configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation) + if err != nil { + return nil, err } var enabledSecrets []string @@ -92,12 +93,13 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri if len(enabledSecrets) == 0 { logrus.Debugf("not generating app secrets, none enabled in recipe config") - return secretConfigs, appModifiers, nil + return nil, nil } + secretValues := map[string]SecretValue{} for secretId, secretConfig := range config.Secrets { if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" { - return secretConfigs, appModifiers, fmt.Errorf("missing version for secret? (%s)", secretId) + return nil, fmt.Errorf("missing version for secret? (%s)", secretId) } if !(slices.Contains(enabledSecrets, secretId)) { @@ -107,87 +109,45 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri lastIdx := strings.LastIndex(secretConfig.Name, "_") secretVersion := secretConfig.Name[lastIdx+1:] - secretConfigs[secretId] = secretVersion + value := SecretValue{Version: secretVersion} - // THIS IS SOOOO HACKY for k, v := range appModifiers { - if strings.Contains(k, strings.ToUpper(secretId)) { - secretModifiers[secretId] = v + log.Println(configWithoutEnv.Secrets[secretId].Name, k) + if strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) { + lengthRaw, ok := v["length"] + if ok { + length, err := strconv.Atoi(lengthRaw) + if err != nil { + return nil, err + } + value.Length = length + } + break } } + secretValues[secretId] = value } - return secretConfigs, secretModifiers, nil -} - -func ParseSecretValueUsingModifiers(secret string, modifiers map[string]string) (secretValue, error) { - length := 0 - if modifiers != nil { - lengthRaw, ok := modifiers["length"] - if ok { - var err error - length, err = strconv.Atoi(lengthRaw) - if err != nil { - return secretValue{}, err - } - } - } - return secretValue{Version: secret, Length: length}, nil -} - -func ParseSecretValue(secret string) (secretValue, error) { - values := strings.Split(secret, "#") - if len(values) == 0 { - return secretValue{}, fmt.Errorf("unable to parse %s", secret) - } - - if len(values) == 1 { - return secretValue{Version: values[0], Length: 0}, nil - } - - split := strings.Split(values[1], "=") - parsed := split[len(split)-1] - stripped := strings.ReplaceAll(parsed, " ", "") - length, err := strconv.Atoi(stripped) - if err != nil { - return secretValue{}, err - } - version := strings.ReplaceAll(values[0], " ", "") - - logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret) - - return secretValue{Version: version, Length: length}, nil + return secretValues, nil } // GenerateSecrets generates secrets locally and sends them to a remote server for storage. -func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, modifiersFromConfig map[string]map[string]string, appName, server string) (map[string]string, error) { - secrets := make(map[string]string) - +func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) { + secretsGenerated := map[string]string{} var mutex sync.Mutex var wg sync.WaitGroup - ch := make(chan error, len(secretsFromConfig)) - for n, v := range secretsFromConfig { + ch := make(chan error, len(secrets)) + for n, v := range secrets { wg.Add(1) - mods := make(map[string]string) - if modifiersFromConfig != nil { - mods = modifiersFromConfig[n] - } - log.Println(n) - go func(secretName, secretValue string, modifiers map[string]string) { + go func(secretName string, secret SecretValue) { defer wg.Done() - parsedSecretValue, err := ParseSecretValueUsingModifiers(secretValue, modifiers) - if err != nil { - ch <- err - return - } - - secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version) + secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version) logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server) - if parsedSecretValue.Length > 0 { - passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length)) + if secret.Length > 0 { + passwords, err := GeneratePasswords(1, uint(secret.Length)) if err != nil { ch <- err return @@ -205,7 +165,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin mutex.Lock() defer mutex.Unlock() - secrets[secretName] = passwords[0] + secretsGenerated[secretName] = passwords[0] } else { passphrases, err := GeneratePassphrases(1) if err != nil { @@ -225,15 +185,15 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin mutex.Lock() defer mutex.Unlock() - secrets[secretName] = passphrases[0] + secretsGenerated[secretName] = passphrases[0] } ch <- nil - }(n, v, mods) + }(n, v) } wg.Wait() - for range secretsFromConfig { + for range secrets { err := <-ch if err != nil { return nil, err @@ -242,7 +202,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin logrus.Debugf("generated and stored %s on %s", secrets, server) - return secrets, nil + return secretsGenerated, nil } type secretStatus struct { @@ -264,7 +224,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, return secStats, err } - secretsConfig, _, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe) + secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe) if err != nil { return secStats, err } @@ -284,14 +244,9 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, remoteSecretNames[cont.Spec.Annotations.Name] = true } - for secretName, secretValue := range secretsConfig { + for secretName, val := range secretsConfig { createdRemote := false - val, err := ParseSecretValue(secretValue) - if err != nil { - return secStats, err - } - secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) if _, ok := remoteSecretNames[secretRemoteName]; ok { createdRemote = true diff --git a/pkg/upstream/stack/loader.go b/pkg/upstream/stack/loader.go index 51b678b0..9790903f 100644 --- a/pkg/upstream/stack/loader.go +++ b/pkg/upstream/stack/loader.go @@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) { opts.SkipValidation = false } +// SkipInterpolation skip interpolating environment variables. +func SkipInterpolation(opts *loader.Options) { + opts.SkipInterpolation = true +} + // LoadComposefile parse the composefile specified in the cli and returns its Config and version. -func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) { +func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) { configDetails, err := getConfigDetails(opts.Composefiles, appEnv) if err != nil { return nil, err } + if options == nil { + options = []func(*loader.Options){DontSkipValidation} + } + dicts := getDictsFrom(configDetails.ConfigFiles) - config, err := loader.Load(configDetails, DontSkipValidation) + config, err := loader.Load(configDetails, options...) if err != nil { if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok { return nil, fmt.Errorf("compose file contains unsupported options: %s",