a little less hacky
continuous-integration/drone/pr Build is failing
Details
continuous-integration/drone/pr Build is failing
Details
This commit is contained in:
parent
be89cdb39c
commit
899909b2ed
|
@ -108,7 +108,7 @@ var appNewCommand = cli.Command{
|
||||||
}
|
}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
|
||||||
secretsConfig, modifiers, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
|
secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -122,7 +122,7 @@ var appNewCommand = cli.Command{
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secrets, err = createSecrets(cl, secretsConfig, modifiers, sanitisedAppName)
|
secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -168,14 +168,14 @@ var appNewCommand = cli.Command{
|
||||||
type AppSecrets map[string]string
|
type AppSecrets map[string]string
|
||||||
|
|
||||||
// createSecrets creates all secrets for a new app.
|
// createSecrets creates all secrets for a new app.
|
||||||
func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, modifiers map[string]map[string]string, sanitisedAppName string) (AppSecrets, error) {
|
func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
|
||||||
// NOTE(d1): trim to match app.StackName() implementation
|
// NOTE(d1): trim to match app.StackName() implementation
|
||||||
if len(sanitisedAppName) > 45 {
|
if len(sanitisedAppName) > 45 {
|
||||||
logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
|
logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
|
||||||
sanitisedAppName = sanitisedAppName[:45]
|
sanitisedAppName = sanitisedAppName[:45]
|
||||||
}
|
}
|
||||||
|
|
||||||
secrets, err := secret.GenerateSecrets(cl, secretsConfig, modifiers, sanitisedAppName, internal.NewAppServer)
|
secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -217,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// promptForSecrets asks if we should generate secrets for a new app.
|
// promptForSecrets asks if we should generate secrets for a new app.
|
||||||
func promptForSecrets(recipeName string, secretsConfig map[string]string) error {
|
func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
|
||||||
if len(secretsConfig) == 0 {
|
if len(secretsConfig) == 0 {
|
||||||
logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
|
logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -87,28 +87,24 @@ var appSecretGenerateCommand = cli.Command{
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsConfig, secretModifiers, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsToCreate := make(map[string]string)
|
if !allSecrets {
|
||||||
if allSecrets {
|
|
||||||
secretsToCreate = secretsConfig
|
|
||||||
} else {
|
|
||||||
secretName := c.Args().Get(1)
|
secretName := c.Args().Get(1)
|
||||||
secretVersion := c.Args().Get(2)
|
secretVersion := c.Args().Get(2)
|
||||||
matches := false
|
s, ok := secrets[secretName]
|
||||||
for name := range secretsConfig {
|
if !ok {
|
||||||
if secretName == name {
|
logrus.Fatalf("%s doesn't exist in the env config?", secretName)
|
||||||
secretsToCreate[name] = secretVersion
|
} else {
|
||||||
matches = true
|
s.Version = secretVersion
|
||||||
|
secrets = map[string]secret.SecretValue{
|
||||||
|
secretName: s,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !matches {
|
|
||||||
logrus.Fatalf("%s doesn't exist in the env config?", secretName)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cl, err := client.New(app.Server)
|
cl, err := client.New(app.Server)
|
||||||
|
@ -116,7 +112,7 @@ var appSecretGenerateCommand = cli.Command{
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, secretModifiers, app.StackName(), app.Server)
|
secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -276,7 +272,7 @@ Example:
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsConfig, _, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -311,12 +307,7 @@ Example:
|
||||||
|
|
||||||
match := false
|
match := false
|
||||||
secretToRm := c.Args().Get(1)
|
secretToRm := c.Args().Get(1)
|
||||||
for secretName, secretValue := range secretsConfig {
|
for secretName, val := range secrets {
|
||||||
val, err := secret.ParseSecretValue(secretValue)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
||||||
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
||||||
if secretToRm != "" {
|
if secretToRm != "" {
|
||||||
|
|
|
@ -22,9 +22,9 @@ import (
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
|
// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
|
||||||
// secret definition.
|
// secret definition.
|
||||||
type secretValue struct {
|
type SecretValue struct {
|
||||||
Version string
|
Version string
|
||||||
Length int
|
Length int
|
||||||
}
|
}
|
||||||
|
@ -68,19 +68,20 @@ func GeneratePassphrases(count uint) ([]string, error) {
|
||||||
// and some times you don't (as the caller). We need to be able to handle the
|
// and some times you don't (as the caller). We need to be able to handle the
|
||||||
// "app new" case where we pass in the .env.sample and the "secret generate"
|
// "app new" case where we pass in the .env.sample and the "secret generate"
|
||||||
// case where the app is created.
|
// case where the app is created.
|
||||||
func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, map[string]map[string]string, error) {
|
func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
|
||||||
secretConfigs := make(map[string]string)
|
|
||||||
secretModifiers := make(map[string]map[string]string)
|
|
||||||
|
|
||||||
appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
|
appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return secretConfigs, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
opts := stack.Deploy{Composefiles: composeFiles}
|
opts := stack.Deploy{Composefiles: composeFiles}
|
||||||
config, err := loader.LoadComposefile(opts, appEnv)
|
config, err := loader.LoadComposefile(opts, appEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return secretConfigs, appModifiers, err
|
return nil, err
|
||||||
|
}
|
||||||
|
configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var enabledSecrets []string
|
var enabledSecrets []string
|
||||||
|
@ -92,12 +93,13 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
|
||||||
|
|
||||||
if len(enabledSecrets) == 0 {
|
if len(enabledSecrets) == 0 {
|
||||||
logrus.Debugf("not generating app secrets, none enabled in recipe config")
|
logrus.Debugf("not generating app secrets, none enabled in recipe config")
|
||||||
return secretConfigs, appModifiers, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
secretValues := map[string]SecretValue{}
|
||||||
for secretId, secretConfig := range config.Secrets {
|
for secretId, secretConfig := range config.Secrets {
|
||||||
if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
|
if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
|
||||||
return secretConfigs, appModifiers, fmt.Errorf("missing version for secret? (%s)", secretId)
|
return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !(slices.Contains(enabledSecrets, secretId)) {
|
if !(slices.Contains(enabledSecrets, secretId)) {
|
||||||
|
@ -107,87 +109,45 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
|
||||||
|
|
||||||
lastIdx := strings.LastIndex(secretConfig.Name, "_")
|
lastIdx := strings.LastIndex(secretConfig.Name, "_")
|
||||||
secretVersion := secretConfig.Name[lastIdx+1:]
|
secretVersion := secretConfig.Name[lastIdx+1:]
|
||||||
secretConfigs[secretId] = secretVersion
|
value := SecretValue{Version: secretVersion}
|
||||||
|
|
||||||
// THIS IS SOOOO HACKY
|
|
||||||
for k, v := range appModifiers {
|
for k, v := range appModifiers {
|
||||||
if strings.Contains(k, strings.ToUpper(secretId)) {
|
log.Println(configWithoutEnv.Secrets[secretId].Name, k)
|
||||||
secretModifiers[secretId] = v
|
if strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
|
||||||
|
lengthRaw, ok := v["length"]
|
||||||
|
if ok {
|
||||||
|
length, err := strconv.Atoi(lengthRaw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
value.Length = length
|
||||||
|
}
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
secretValues[secretId] = value
|
||||||
}
|
}
|
||||||
|
|
||||||
return secretConfigs, secretModifiers, nil
|
return secretValues, nil
|
||||||
}
|
|
||||||
|
|
||||||
func ParseSecretValueUsingModifiers(secret string, modifiers map[string]string) (secretValue, error) {
|
|
||||||
length := 0
|
|
||||||
if modifiers != nil {
|
|
||||||
lengthRaw, ok := modifiers["length"]
|
|
||||||
if ok {
|
|
||||||
var err error
|
|
||||||
length, err = strconv.Atoi(lengthRaw)
|
|
||||||
if err != nil {
|
|
||||||
return secretValue{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return secretValue{Version: secret, Length: length}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseSecretValue(secret string) (secretValue, error) {
|
|
||||||
values := strings.Split(secret, "#")
|
|
||||||
if len(values) == 0 {
|
|
||||||
return secretValue{}, fmt.Errorf("unable to parse %s", secret)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(values) == 1 {
|
|
||||||
return secretValue{Version: values[0], Length: 0}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
split := strings.Split(values[1], "=")
|
|
||||||
parsed := split[len(split)-1]
|
|
||||||
stripped := strings.ReplaceAll(parsed, " ", "")
|
|
||||||
length, err := strconv.Atoi(stripped)
|
|
||||||
if err != nil {
|
|
||||||
return secretValue{}, err
|
|
||||||
}
|
|
||||||
version := strings.ReplaceAll(values[0], " ", "")
|
|
||||||
|
|
||||||
logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
|
|
||||||
|
|
||||||
return secretValue{Version: version, Length: length}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
|
// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
|
||||||
func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, modifiersFromConfig map[string]map[string]string, appName, server string) (map[string]string, error) {
|
func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
|
||||||
secrets := make(map[string]string)
|
secretsGenerated := map[string]string{}
|
||||||
|
|
||||||
var mutex sync.Mutex
|
var mutex sync.Mutex
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
ch := make(chan error, len(secretsFromConfig))
|
ch := make(chan error, len(secrets))
|
||||||
for n, v := range secretsFromConfig {
|
for n, v := range secrets {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
mods := make(map[string]string)
|
|
||||||
if modifiersFromConfig != nil {
|
|
||||||
mods = modifiersFromConfig[n]
|
|
||||||
}
|
|
||||||
log.Println(n)
|
|
||||||
|
|
||||||
go func(secretName, secretValue string, modifiers map[string]string) {
|
go func(secretName string, secret SecretValue) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
parsedSecretValue, err := ParseSecretValueUsingModifiers(secretValue, modifiers)
|
secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
|
||||||
if err != nil {
|
|
||||||
ch <- err
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version)
|
|
||||||
logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
|
logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
|
||||||
|
|
||||||
if parsedSecretValue.Length > 0 {
|
if secret.Length > 0 {
|
||||||
passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length))
|
passwords, err := GeneratePasswords(1, uint(secret.Length))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch <- err
|
ch <- err
|
||||||
return
|
return
|
||||||
|
@ -205,7 +165,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
|
||||||
|
|
||||||
mutex.Lock()
|
mutex.Lock()
|
||||||
defer mutex.Unlock()
|
defer mutex.Unlock()
|
||||||
secrets[secretName] = passwords[0]
|
secretsGenerated[secretName] = passwords[0]
|
||||||
} else {
|
} else {
|
||||||
passphrases, err := GeneratePassphrases(1)
|
passphrases, err := GeneratePassphrases(1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -225,15 +185,15 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
|
||||||
|
|
||||||
mutex.Lock()
|
mutex.Lock()
|
||||||
defer mutex.Unlock()
|
defer mutex.Unlock()
|
||||||
secrets[secretName] = passphrases[0]
|
secretsGenerated[secretName] = passphrases[0]
|
||||||
}
|
}
|
||||||
ch <- nil
|
ch <- nil
|
||||||
}(n, v, mods)
|
}(n, v)
|
||||||
}
|
}
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
for range secretsFromConfig {
|
for range secrets {
|
||||||
err := <-ch
|
err := <-ch
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -242,7 +202,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
|
||||||
|
|
||||||
logrus.Debugf("generated and stored %s on %s", secrets, server)
|
logrus.Debugf("generated and stored %s on %s", secrets, server)
|
||||||
|
|
||||||
return secrets, nil
|
return secretsGenerated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type secretStatus struct {
|
type secretStatus struct {
|
||||||
|
@ -264,7 +224,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
|
||||||
return secStats, err
|
return secStats, err
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsConfig, _, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return secStats, err
|
return secStats, err
|
||||||
}
|
}
|
||||||
|
@ -284,14 +244,9 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
|
||||||
remoteSecretNames[cont.Spec.Annotations.Name] = true
|
remoteSecretNames[cont.Spec.Annotations.Name] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
for secretName, secretValue := range secretsConfig {
|
for secretName, val := range secretsConfig {
|
||||||
createdRemote := false
|
createdRemote := false
|
||||||
|
|
||||||
val, err := ParseSecretValue(secretValue)
|
|
||||||
if err != nil {
|
|
||||||
return secStats, err
|
|
||||||
}
|
|
||||||
|
|
||||||
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
||||||
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
||||||
createdRemote = true
|
createdRemote = true
|
||||||
|
|
|
@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) {
|
||||||
opts.SkipValidation = false
|
opts.SkipValidation = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SkipInterpolation skip interpolating environment variables.
|
||||||
|
func SkipInterpolation(opts *loader.Options) {
|
||||||
|
opts.SkipInterpolation = true
|
||||||
|
}
|
||||||
|
|
||||||
// LoadComposefile parse the composefile specified in the cli and returns its Config and version.
|
// LoadComposefile parse the composefile specified in the cli and returns its Config and version.
|
||||||
func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
|
func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
|
||||||
configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
|
configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if options == nil {
|
||||||
|
options = []func(*loader.Options){DontSkipValidation}
|
||||||
|
}
|
||||||
|
|
||||||
dicts := getDictsFrom(configDetails.ConfigFiles)
|
dicts := getDictsFrom(configDetails.ConfigFiles)
|
||||||
config, err := loader.Load(configDetails, DontSkipValidation)
|
config, err := loader.Load(configDetails, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
|
if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
|
||||||
return nil, fmt.Errorf("compose file contains unsupported options: %s",
|
return nil, fmt.Errorf("compose file contains unsupported options: %s",
|
||||||
|
|
Loading…
Reference in New Issue