From 0076b312534eb67e9659586c065a9494fbb750c1 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Sat, 6 Jul 2024 16:13:17 +0200 Subject: [PATCH] new package envfile and move GetComposeFiles to recipe package --- cli/app/deploy.go | 5 +- cli/app/new.go | 2 +- cli/app/ps.go | 2 +- cli/app/rollback.go | 5 +- cli/app/secret.go | 4 +- cli/app/upgrade.go | 5 +- cli/updater/updater.go | 11 ++- pkg/app/app.go | 95 ++++++++++++++++--- pkg/app/compose.go | 3 +- pkg/compose/compose.go | 6 +- pkg/{app/env.go => envfile/envfile.go} | 86 +++-------------- .../env_test.go => envfile/envfile_test.go} | 2 +- pkg/lint/recipe.go | 4 +- pkg/recipe/recipe.go | 6 +- pkg/secret/secret.go | 6 +- 15 files changed, 128 insertions(+), 114 deletions(-) rename pkg/{app/env.go => envfile/envfile.go} (53%) rename pkg/{app/env_test.go => envfile/envfile_test.go} (99%) diff --git a/cli/app/deploy.go b/cli/app/deploy.go index bd948803..06925199 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -6,6 +6,7 @@ import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/secret" appPkg "coopcloud.tech/abra/pkg/app" @@ -179,7 +180,7 @@ recipes. } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath) if err != nil { logrus.Fatal(err) } @@ -187,7 +188,7 @@ recipes. app.Env[k] = v } - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/new.go b/cli/app/new.go index acd37205..395ec2d0 100644 --- a/cli/app/new.go +++ b/cli/app/new.go @@ -136,7 +136,7 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } - composeFiles, err := appPkg.GetComposeFiles(recipe.Name, sampleEnv) + composeFiles, err := recipePkg.GetComposeFiles(recipe.Name, sampleEnv) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/ps.go b/cli/app/ps.go index b69c6f70..e5917407 100644 --- a/cli/app/ps.go +++ b/cli/app/ps.go @@ -79,7 +79,7 @@ var appPsCommand = cli.Command{ // showPSOutput renders ps output. func showPSOutput(c *cli.Context, app appPkg.App, cl *dockerClient.Client) { - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) return diff --git a/cli/app/rollback.go b/cli/app/rollback.go index 0e4e7440..4be20cf4 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -7,6 +7,7 @@ import ( appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/lint" "coopcloud.tech/abra/pkg/recipe" stack "coopcloud.tech/abra/pkg/upstream/stack" @@ -199,7 +200,7 @@ recipes. } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath) if err != nil { logrus.Fatal(err) } @@ -207,7 +208,7 @@ recipes. app.Env[k] = v } - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/secret.go b/cli/app/secret.go index 4e02f928..6f4b4a2e 100644 --- a/cli/app/secret.go +++ b/cli/app/secret.go @@ -87,7 +87,7 @@ var appSecretGenerateCommand = cli.Command{ internal.ShowSubcommandHelpAndError(c, err) } - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } @@ -284,7 +284,7 @@ Example: } } - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index 1af5a106..6079c5b1 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -9,6 +9,7 @@ import ( "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/lint" "coopcloud.tech/abra/pkg/recipe" recipePkg "coopcloud.tech/abra/pkg/recipe" @@ -233,7 +234,7 @@ recipes. } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath) if err != nil { logrus.Fatal(err) } @@ -241,7 +242,7 @@ recipes. app.Env[k] = v } - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipePkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/updater/updater.go b/cli/updater/updater.go index a716def6..0c39510e 100644 --- a/cli/updater/updater.go +++ b/cli/updater/updater.go @@ -11,6 +11,7 @@ import ( appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/lint" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/convert" @@ -193,7 +194,7 @@ func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool } // getEnv reads env variables from docker services. -func getEnv(cl *dockerclient.Client, stackName string) (appPkg.AppEnv, error) { +func getEnv(cl *dockerclient.Client, stackName string) (envfile.AppEnv, error) { envMap := make(map[string]string) filter := filters.NewArgs() filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName)) @@ -340,9 +341,9 @@ func processRecipeRepoVersion(recipeName, version string) error { } // mergeAbraShEnv merges abra.sh env vars into the app env vars. -func mergeAbraShEnv(recipeName string, env appPkg.AppEnv) error { +func mergeAbraShEnv(recipeName string, env envfile.AppEnv) error { abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh") - abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath) if err != nil { return err } @@ -356,7 +357,7 @@ func mergeAbraShEnv(recipeName string, env appPkg.AppEnv) error { } // createDeployConfig merges and enriches the compose config for the deployment. -func createDeployConfig(recipeName string, stackName string, env appPkg.AppEnv) (*composetypes.Config, stack.Deploy, error) { +func createDeployConfig(recipeName string, stackName string, env envfile.AppEnv) (*composetypes.Config, stack.Deploy, error) { env["STACK_NAME"] = stackName deployOpts := stack.Deploy{ @@ -366,7 +367,7 @@ func createDeployConfig(recipeName string, stackName string, env appPkg.AppEnv) Detach: false, } - composeFiles, err := appPkg.GetComposeFiles(recipeName, env) + composeFiles, err := recipe.GetComposeFiles(recipeName, env) if err != nil { return nil, deployOpts, err } diff --git a/pkg/app/app.go b/pkg/app/app.go index 2a2289c5..6cb657c9 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -1,14 +1,19 @@ package app import ( + "bufio" "fmt" "os" "path" + "regexp" + "sort" "strings" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/formatter" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/convert" "coopcloud.tech/abra/pkg/upstream/stack" @@ -81,19 +86,13 @@ type App struct { Name AppName Recipe string Domain string - Env AppEnv + Env envfile.AppEnv Server string Path string } // Type aliases to make code hints easier to understand -// AppEnv is a map of the values in an apps env config -type AppEnv = map[string]string - -// AppModifiers is a map of modifiers in an apps env config -type AppModifiers = map[string]map[string]string - // AppName is AppName type AppName = string @@ -162,7 +161,7 @@ func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (f return filters, nil } - composeFiles, err := GetComposeFiles(a.Recipe, a.Env) + composeFiles, err := recipe.GetComposeFiles(a.Recipe, a.Env) if err != nil { return filters, err } @@ -231,7 +230,7 @@ func (a ByName) Less(i, j int) bool { } func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) { - env, err := ReadEnv(appFile.Path) + env, err := envfile.ReadEnv(appFile.Path) if err != nil { return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error()) } @@ -247,7 +246,7 @@ func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) { } // NewApp creates new App object -func NewApp(env AppEnv, name string, appFile AppFile) (App, error) { +func NewApp(env envfile.AppEnv, name string, appFile AppFile) (App, error) { domain := env["DOMAIN"] recipe, exists := env["RECIPE"] @@ -318,7 +317,7 @@ func GetAppServiceNames(appName string) ([]string, error) { return serviceNames, err } - composeFiles, err := GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { return serviceNames, err } @@ -481,7 +480,7 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str // GetAppComposeConfig retrieves a compose specification for a recipe. This // specification is the result of a merge of all the compose.**.yml files in // the recipe repository. -func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) { +func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv envfile.AppEnv) (*composetypes.Config, error) { compose, err := loader.LoadComposefile(opts, appEnv) if err != nil { return &composetypes.Config{}, err @@ -493,7 +492,7 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp } // ExposeAllEnv exposes all env variables to the app container -func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) { +func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv envfile.AppEnv) { for _, service := range compose.Services { if service.Name == "app" { logrus.Debugf("Add the following environment to the app service config of %s:", stackName) @@ -508,3 +507,73 @@ func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) } } } + +func CheckEnv(app App) ([]envfile.EnvVar, error) { + var envVars []envfile.EnvVar + + envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample") + if _, err := os.Stat(envSamplePath); err != nil { + if os.IsNotExist(err) { + return envVars, fmt.Errorf("%s does not exist?", envSamplePath) + } + return envVars, err + } + + envSample, err := envfile.ReadEnv(envSamplePath) + if err != nil { + return envVars, err + } + + var keys []string + for key := range envSample { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + if _, ok := app.Env[key]; ok { + envVars = append(envVars, envfile.EnvVar{Name: key, Present: true}) + } else { + envVars = append(envVars, envfile.EnvVar{Name: key, Present: false}) + } + } + + return envVars, nil +} + +// ReadAbraShCmdNames reads the names of commands. +func ReadAbraShCmdNames(abraSh string) ([]string, error) { + var cmdNames []string + + file, err := os.Open(abraSh) + if err != nil { + if os.IsNotExist(err) { + return cmdNames, nil + } + return cmdNames, err + } + defer file.Close() + + cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`) + if err != nil { + return cmdNames, err + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + matches := cmdNameRegex.FindStringSubmatch(line) + if len(matches) > 0 { + cmdNames = append(cmdNames, matches[1]) + } + } + + if len(cmdNames) > 0 { + logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh) + } else { + logrus.Debugf("read 0 command names from %s", abraSh) + } + + return cmdNames, nil +} diff --git a/pkg/app/compose.go b/pkg/app/compose.go index 771f56b6..f3109fdd 100644 --- a/pkg/app/compose.go +++ b/pkg/app/compose.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" + "coopcloud.tech/abra/pkg/envfile" composetypes "github.com/docker/cli/cli/compose/types" "github.com/sirupsen/logrus" ) @@ -46,7 +47,7 @@ func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosV // SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the // auto update process for this app. The default if this variable is not set is to disable // the auto update process. -func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) { +func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv envfile.AppEnv) { for _, service := range compose.Services { if service.Name == "app" { enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"] diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go index d6753327..6d116a97 100644 --- a/pkg/compose/compose.go +++ b/pkg/compose/compose.go @@ -7,8 +7,8 @@ import ( "path/filepath" "strings" - "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/upstream/stack" loader "coopcloud.tech/abra/pkg/upstream/stack" @@ -30,7 +30,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) { opts := stack.Deploy{Composefiles: []string{composeFile}} envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") - sampleEnv, err := app.ReadEnv(envSamplePath) + sampleEnv, err := envfile.ReadEnv(envSamplePath) if err != nil { return false, err } @@ -98,7 +98,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error { opts := stack.Deploy{Composefiles: []string{composeFile}} envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") - sampleEnv, err := app.ReadEnv(envSamplePath) + sampleEnv, err := envfile.ReadEnv(envSamplePath) if err != nil { return err } diff --git a/pkg/app/env.go b/pkg/envfile/envfile.go similarity index 53% rename from pkg/app/env.go rename to pkg/envfile/envfile.go index 5daa526a..2692b2f2 100644 --- a/pkg/app/env.go +++ b/pkg/envfile/envfile.go @@ -1,19 +1,27 @@ -package app +package envfile import ( "bufio" "fmt" "os" - "path" "regexp" - "sort" "strings" - "coopcloud.tech/abra/pkg/config" "git.coopcloud.tech/coop-cloud/godotenv" "github.com/sirupsen/logrus" ) +// envVarModifiers is a list of env var modifier strings. These are added to +// env vars as comments and modify their processing by Abra, e.g. determining +// how long secrets should be. +var envVarModifiers = []string{"length"} + +// AppEnv is a map of the values in an apps env config +type AppEnv = map[string]string + +// AppModifiers is a map of modifiers in an apps env config +type AppModifiers = map[string]map[string]string + // ReadEnv loads an app envivornment into a map. func ReadEnv(filePath string) (AppEnv, error) { var envVars AppEnv @@ -87,73 +95,3 @@ type EnvVar struct { Name string Present bool } - -func CheckEnv(app App) ([]EnvVar, error) { - var envVars []EnvVar - - envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample") - if _, err := os.Stat(envSamplePath); err != nil { - if os.IsNotExist(err) { - return envVars, fmt.Errorf("%s does not exist?", envSamplePath) - } - return envVars, err - } - - envSample, err := ReadEnv(envSamplePath) - if err != nil { - return envVars, err - } - - var keys []string - for key := range envSample { - keys = append(keys, key) - } - - sort.Strings(keys) - - for _, key := range keys { - if _, ok := app.Env[key]; ok { - envVars = append(envVars, EnvVar{Name: key, Present: true}) - } else { - envVars = append(envVars, EnvVar{Name: key, Present: false}) - } - } - - return envVars, nil -} - -// ReadAbraShCmdNames reads the names of commands. -func ReadAbraShCmdNames(abraSh string) ([]string, error) { - var cmdNames []string - - file, err := os.Open(abraSh) - if err != nil { - if os.IsNotExist(err) { - return cmdNames, nil - } - return cmdNames, err - } - defer file.Close() - - cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`) - if err != nil { - return cmdNames, err - } - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - matches := cmdNameRegex.FindStringSubmatch(line) - if len(matches) > 0 { - cmdNames = append(cmdNames, matches[1]) - } - } - - if len(cmdNames) > 0 { - logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh) - } else { - logrus.Debugf("read 0 command names from %s", abraSh) - } - - return cmdNames, nil -} diff --git a/pkg/app/env_test.go b/pkg/envfile/envfile_test.go similarity index 99% rename from pkg/app/env_test.go rename to pkg/envfile/envfile_test.go index 33f894c1..9baea2e8 100644 --- a/pkg/app/env_test.go +++ b/pkg/envfile/envfile_test.go @@ -1,4 +1,4 @@ -package app_test +package envfile_test import ( "fmt" diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go index b861161b..175e98af 100644 --- a/pkg/lint/recipe.go +++ b/pkg/lint/recipe.go @@ -6,8 +6,8 @@ import ( "os" "path" - "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/recipe" recipePkg "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" @@ -235,7 +235,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) { // therefore no matching traefik deploy label will be present. func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) { envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample") - sampleEnv, err := app.ReadEnv(envSamplePath) + sampleEnv, err := envfile.ReadEnv(envSamplePath) if err != nil { return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name) } diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index 465a8529..8638a96a 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -12,10 +12,10 @@ import ( "strconv" "strings" - "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/compose" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" "coopcloud.tech/abra/pkg/formatter" gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/abra/pkg/limit" @@ -228,7 +228,7 @@ func Get(recipeName string, offline bool) (Recipe, error) { } envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") - sampleEnv, err := app.ReadEnv(envSamplePath) + sampleEnv, err := envfile.ReadEnv(envSamplePath) if err != nil { return Recipe{}, err } @@ -258,7 +258,7 @@ func Get(recipeName string, offline bool) (Recipe, error) { func (r Recipe) SampleEnv() (map[string]string, error) { envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") - sampleEnv, err := app.ReadEnv(envSamplePath) + sampleEnv, err := envfile.ReadEnv(envSamplePath) if err != nil { return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name) } diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index ca6da99b..9e7ff09d 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -14,6 +14,8 @@ import ( appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/envfile" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/stack" loader "coopcloud.tech/abra/pkg/upstream/stack" "github.com/decentral1se/passgen" @@ -82,7 +84,7 @@ func GeneratePassphrases(count uint) ([]string, error) { // "app new" case where we pass in the .env.sample and the "secret generate" // case where the app is created. func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) { - appEnv, appModifiers, err := appPkg.ReadEnvWithModifiers(appEnvPath) + appEnv, appModifiers, err := envfile.ReadEnvWithModifiers(appEnvPath) if err != nil { return nil, err } @@ -244,7 +246,7 @@ type secretStatuses []secretStatus func PollSecretsStatus(cl *dockerClient.Client, app appPkg.App) (secretStatuses, error) { var secStats secretStatuses - composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env) if err != nil { return secStats, err }