a little less hacky
continuous-integration/drone/pr Build is failing Details

This commit is contained in:
p4u1 2023-11-29 21:34:36 +01:00
parent be89cdb39c
commit 899909b2ed
4 changed files with 69 additions and 114 deletions

View File

@ -108,7 +108,7 @@ var appNewCommand = cli.Command{
} }
envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample") envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
secretsConfig, modifiers, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name) secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
if err != nil { if err != nil {
return err return err
} }
@ -122,7 +122,7 @@ var appNewCommand = cli.Command{
logrus.Fatal(err) logrus.Fatal(err)
} }
secrets, err = createSecrets(cl, secretsConfig, modifiers, sanitisedAppName) secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
if err != nil { if err != nil {
logrus.Fatal(err) logrus.Fatal(err)
} }
@ -168,14 +168,14 @@ var appNewCommand = cli.Command{
type AppSecrets map[string]string type AppSecrets map[string]string
// createSecrets creates all secrets for a new app. // createSecrets creates all secrets for a new app.
func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, modifiers map[string]map[string]string, sanitisedAppName string) (AppSecrets, error) { func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
// NOTE(d1): trim to match app.StackName() implementation // NOTE(d1): trim to match app.StackName() implementation
if len(sanitisedAppName) > 45 { if len(sanitisedAppName) > 45 {
logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45]) logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
sanitisedAppName = sanitisedAppName[:45] sanitisedAppName = sanitisedAppName[:45]
} }
secrets, err := secret.GenerateSecrets(cl, secretsConfig, modifiers, sanitisedAppName, internal.NewAppServer) secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -217,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
} }
// promptForSecrets asks if we should generate secrets for a new app. // promptForSecrets asks if we should generate secrets for a new app.
func promptForSecrets(recipeName string, secretsConfig map[string]string) error { func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
if len(secretsConfig) == 0 { if len(secretsConfig) == 0 {
logrus.Debugf("%s has no secrets to generate, skipping...", recipeName) logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
return nil return nil

View File

@ -87,28 +87,24 @@ var appSecretGenerateCommand = cli.Command{
logrus.Fatal(err) logrus.Fatal(err)
} }
secretsConfig, secretModifiers, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe) secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
if err != nil { if err != nil {
logrus.Fatal(err) logrus.Fatal(err)
} }
secretsToCreate := make(map[string]string) if !allSecrets {
if allSecrets {
secretsToCreate = secretsConfig
} else {
secretName := c.Args().Get(1) secretName := c.Args().Get(1)
secretVersion := c.Args().Get(2) secretVersion := c.Args().Get(2)
matches := false s, ok := secrets[secretName]
for name := range secretsConfig { if !ok {
if secretName == name { logrus.Fatalf("%s doesn't exist in the env config?", secretName)
secretsToCreate[name] = secretVersion } else {
matches = true s.Version = secretVersion
secrets = map[string]secret.SecretValue{
secretName: s,
} }
} }
if !matches {
logrus.Fatalf("%s doesn't exist in the env config?", secretName)
}
} }
cl, err := client.New(app.Server) cl, err := client.New(app.Server)
@ -116,7 +112,7 @@ var appSecretGenerateCommand = cli.Command{
logrus.Fatal(err) logrus.Fatal(err)
} }
secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, secretModifiers, app.StackName(), app.Server) secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
if err != nil { if err != nil {
logrus.Fatal(err) logrus.Fatal(err)
} }
@ -276,7 +272,7 @@ Example:
logrus.Fatal(err) logrus.Fatal(err)
} }
secretsConfig, _, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe) secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
if err != nil { if err != nil {
logrus.Fatal(err) logrus.Fatal(err)
} }
@ -311,12 +307,7 @@ Example:
match := false match := false
secretToRm := c.Args().Get(1) secretToRm := c.Args().Get(1)
for secretName, secretValue := range secretsConfig { for secretName, val := range secrets {
val, err := secret.ParseSecretValue(secretValue)
if err != nil {
logrus.Fatal(err)
}
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
if _, ok := remoteSecretNames[secretRemoteName]; ok { if _, ok := remoteSecretNames[secretRemoteName]; ok {
if secretToRm != "" { if secretToRm != "" {

View File

@ -22,9 +22,9 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config // SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
// secret definition. // secret definition.
type secretValue struct { type SecretValue struct {
Version string Version string
Length int Length int
} }
@ -68,19 +68,20 @@ func GeneratePassphrases(count uint) ([]string, error) {
// and some times you don't (as the caller). We need to be able to handle the // and some times you don't (as the caller). We need to be able to handle the
// "app new" case where we pass in the .env.sample and the "secret generate" // "app new" case where we pass in the .env.sample and the "secret generate"
// case where the app is created. // case where the app is created.
func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, map[string]map[string]string, error) { func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
secretConfigs := make(map[string]string)
secretModifiers := make(map[string]map[string]string)
appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true}) appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
if err != nil { if err != nil {
return secretConfigs, nil, err return nil, err
} }
opts := stack.Deploy{Composefiles: composeFiles} opts := stack.Deploy{Composefiles: composeFiles}
config, err := loader.LoadComposefile(opts, appEnv) config, err := loader.LoadComposefile(opts, appEnv)
if err != nil { if err != nil {
return secretConfigs, appModifiers, err return nil, err
}
configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
if err != nil {
return nil, err
} }
var enabledSecrets []string var enabledSecrets []string
@ -92,12 +93,13 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
if len(enabledSecrets) == 0 { if len(enabledSecrets) == 0 {
logrus.Debugf("not generating app secrets, none enabled in recipe config") logrus.Debugf("not generating app secrets, none enabled in recipe config")
return secretConfigs, appModifiers, nil return nil, nil
} }
secretValues := map[string]SecretValue{}
for secretId, secretConfig := range config.Secrets { for secretId, secretConfig := range config.Secrets {
if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" { if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
return secretConfigs, appModifiers, fmt.Errorf("missing version for secret? (%s)", secretId) return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
} }
if !(slices.Contains(enabledSecrets, secretId)) { if !(slices.Contains(enabledSecrets, secretId)) {
@ -107,87 +109,45 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
lastIdx := strings.LastIndex(secretConfig.Name, "_") lastIdx := strings.LastIndex(secretConfig.Name, "_")
secretVersion := secretConfig.Name[lastIdx+1:] secretVersion := secretConfig.Name[lastIdx+1:]
secretConfigs[secretId] = secretVersion value := SecretValue{Version: secretVersion}
// THIS IS SOOOO HACKY
for k, v := range appModifiers { for k, v := range appModifiers {
if strings.Contains(k, strings.ToUpper(secretId)) { log.Println(configWithoutEnv.Secrets[secretId].Name, k)
secretModifiers[secretId] = v if strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
lengthRaw, ok := v["length"]
if ok {
length, err := strconv.Atoi(lengthRaw)
if err != nil {
return nil, err
}
value.Length = length
}
break
} }
} }
secretValues[secretId] = value
} }
return secretConfigs, secretModifiers, nil return secretValues, nil
}
func ParseSecretValueUsingModifiers(secret string, modifiers map[string]string) (secretValue, error) {
length := 0
if modifiers != nil {
lengthRaw, ok := modifiers["length"]
if ok {
var err error
length, err = strconv.Atoi(lengthRaw)
if err != nil {
return secretValue{}, err
}
}
}
return secretValue{Version: secret, Length: length}, nil
}
func ParseSecretValue(secret string) (secretValue, error) {
values := strings.Split(secret, "#")
if len(values) == 0 {
return secretValue{}, fmt.Errorf("unable to parse %s", secret)
}
if len(values) == 1 {
return secretValue{Version: values[0], Length: 0}, nil
}
split := strings.Split(values[1], "=")
parsed := split[len(split)-1]
stripped := strings.ReplaceAll(parsed, " ", "")
length, err := strconv.Atoi(stripped)
if err != nil {
return secretValue{}, err
}
version := strings.ReplaceAll(values[0], " ", "")
logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
return secretValue{Version: version, Length: length}, nil
} }
// GenerateSecrets generates secrets locally and sends them to a remote server for storage. // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, modifiersFromConfig map[string]map[string]string, appName, server string) (map[string]string, error) { func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
secrets := make(map[string]string) secretsGenerated := map[string]string{}
var mutex sync.Mutex var mutex sync.Mutex
var wg sync.WaitGroup var wg sync.WaitGroup
ch := make(chan error, len(secretsFromConfig)) ch := make(chan error, len(secrets))
for n, v := range secretsFromConfig { for n, v := range secrets {
wg.Add(1) wg.Add(1)
mods := make(map[string]string)
if modifiersFromConfig != nil {
mods = modifiersFromConfig[n]
}
log.Println(n)
go func(secretName, secretValue string, modifiers map[string]string) { go func(secretName string, secret SecretValue) {
defer wg.Done() defer wg.Done()
parsedSecretValue, err := ParseSecretValueUsingModifiers(secretValue, modifiers) secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
if err != nil {
ch <- err
return
}
secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version)
logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server) logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
if parsedSecretValue.Length > 0 { if secret.Length > 0 {
passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length)) passwords, err := GeneratePasswords(1, uint(secret.Length))
if err != nil { if err != nil {
ch <- err ch <- err
return return
@ -205,7 +165,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
mutex.Lock() mutex.Lock()
defer mutex.Unlock() defer mutex.Unlock()
secrets[secretName] = passwords[0] secretsGenerated[secretName] = passwords[0]
} else { } else {
passphrases, err := GeneratePassphrases(1) passphrases, err := GeneratePassphrases(1)
if err != nil { if err != nil {
@ -225,15 +185,15 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
mutex.Lock() mutex.Lock()
defer mutex.Unlock() defer mutex.Unlock()
secrets[secretName] = passphrases[0] secretsGenerated[secretName] = passphrases[0]
} }
ch <- nil ch <- nil
}(n, v, mods) }(n, v)
} }
wg.Wait() wg.Wait()
for range secretsFromConfig { for range secrets {
err := <-ch err := <-ch
if err != nil { if err != nil {
return nil, err return nil, err
@ -242,7 +202,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
logrus.Debugf("generated and stored %s on %s", secrets, server) logrus.Debugf("generated and stored %s on %s", secrets, server)
return secrets, nil return secretsGenerated, nil
} }
type secretStatus struct { type secretStatus struct {
@ -264,7 +224,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
return secStats, err return secStats, err
} }
secretsConfig, _, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe) secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
if err != nil { if err != nil {
return secStats, err return secStats, err
} }
@ -284,14 +244,9 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
remoteSecretNames[cont.Spec.Annotations.Name] = true remoteSecretNames[cont.Spec.Annotations.Name] = true
} }
for secretName, secretValue := range secretsConfig { for secretName, val := range secretsConfig {
createdRemote := false createdRemote := false
val, err := ParseSecretValue(secretValue)
if err != nil {
return secStats, err
}
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
if _, ok := remoteSecretNames[secretRemoteName]; ok { if _, ok := remoteSecretNames[secretRemoteName]; ok {
createdRemote = true createdRemote = true

View File

@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) {
opts.SkipValidation = false opts.SkipValidation = false
} }
// SkipInterpolation skip interpolating environment variables.
func SkipInterpolation(opts *loader.Options) {
opts.SkipInterpolation = true
}
// LoadComposefile parse the composefile specified in the cli and returns its Config and version. // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) { func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
configDetails, err := getConfigDetails(opts.Composefiles, appEnv) configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if options == nil {
options = []func(*loader.Options){DontSkipValidation}
}
dicts := getDictsFrom(configDetails.ConfigFiles) dicts := getDictsFrom(configDetails.ConfigFiles)
config, err := loader.Load(configDetails, DontSkipValidation) config, err := loader.Load(configDetails, options...)
if err != nil { if err != nil {
if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok { if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
return nil, fmt.Errorf("compose file contains unsupported options: %s", return nil, fmt.Errorf("compose file contains unsupported options: %s",