diff --git a/cli/app/check.go b/cli/app/check.go index 6b0e8455..ec95f8a4 100644 --- a/cli/app/check.go +++ b/cli/app/check.go @@ -2,8 +2,8 @@ package app import ( "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/recipe" recipePkg "coopcloud.tech/abra/pkg/recipe" @@ -61,7 +61,7 @@ ${FOO:} syntax). "check" does not confirm or deny this for you.`, tableCol := []string{"recipe env sample", "app env"} table := formatter.CreateTable(tableCol) - envVars, err := config.CheckEnv(app) + envVars, err := appPkg.CheckEnv(app) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/cmd.go b/cli/app/cmd.go index 002eecfa..416812cb 100644 --- a/cli/app/cmd.go +++ b/cli/app/cmd.go @@ -11,6 +11,7 @@ import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/app" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" @@ -142,7 +143,7 @@ Example: logrus.Fatal(err) } - serviceNames, err := config.GetAppServiceNames(app.Name) + serviceNames, err := appPkg.GetAppServiceNames(app.Name) if err != nil { logrus.Fatal(err) } @@ -261,9 +262,9 @@ var appCmdListCommand = cli.Command{ }, } -func getShCmdNames(app config.App) ([]string, error) { +func getShCmdNames(app appPkg.App) ([]string, error) { abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - cmdNames, err := config.ReadAbraShCmdNames(abraShPath) + cmdNames, err := appPkg.ReadAbraShCmdNames(abraShPath) if err != nil { return nil, err } diff --git a/cli/app/config.go b/cli/app/config.go index 9404fae4..7dd15eb2 100644 --- a/cli/app/config.go +++ b/cli/app/config.go @@ -6,8 +6,8 @@ import ( "os/exec" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/config" "github.com/AlecAivazis/survey/v2" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -30,7 +30,7 @@ var appConfigCommand = cli.Command{ internal.ShowSubcommandHelpAndError(c, errors.New("no app provided")) } - files, err := config.LoadAppFiles("") + files, err := appPkg.LoadAppFiles("") if err != nil { logrus.Fatal(err) } diff --git a/cli/app/deploy.go b/cli/app/deploy.go index 87feae8f..bd948803 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -8,6 +8,7 @@ import ( "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/secret" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/dns" @@ -178,7 +179,7 @@ recipes. } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - abraShEnv, err := config.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) if err != nil { logrus.Fatal(err) } @@ -186,7 +187,7 @@ recipes. app.Env[k] = v } - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } @@ -198,18 +199,18 @@ recipes. ResolveImage: stack.ResolveImageAlways, Detach: false, } - compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) + compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { logrus.Fatal(err) } - config.ExposeAllEnv(stackName, compose, app.Env) - config.SetRecipeLabel(compose, stackName, app.Recipe) - config.SetChaosLabel(compose, stackName, internal.Chaos) - config.SetChaosVersionLabel(compose, stackName, version) - config.SetUpdateLabel(compose, stackName, app.Env) + appPkg.ExposeAllEnv(stackName, compose, app.Env) + appPkg.SetRecipeLabel(compose, stackName, app.Recipe) + appPkg.SetChaosLabel(compose, stackName, internal.Chaos) + appPkg.SetChaosVersionLabel(compose, stackName, version) + appPkg.SetUpdateLabel(compose, stackName, app.Env) - envVars, err := config.CheckEnv(app) + envVars, err := appPkg.CheckEnv(app) if err != nil { logrus.Fatal(err) } @@ -237,7 +238,7 @@ recipes. logrus.Warn("skipping domain checks as requested") } - stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName) + stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/errors.go b/cli/app/errors.go index 04e7e68d..a3fb974d 100644 --- a/cli/app/errors.go +++ b/cli/app/errors.go @@ -8,9 +8,9 @@ import ( "time" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" stack "coopcloud.tech/abra/pkg/upstream/stack" containerTypes "github.com/docker/docker/api/types/container" @@ -87,7 +87,7 @@ the logs. }, } -func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error { +func checkErrors(c *cli.Context, cl *dockerClient.Client, app appPkg.App) error { recipe, err := recipe.Get(app.Recipe, internal.Offline) if err != nil { return err diff --git a/cli/app/list.go b/cli/app/list.go index 08dff6f2..63fab5bc 100644 --- a/cli/app/list.go +++ b/cli/app/list.go @@ -8,7 +8,7 @@ import ( "strings" "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/config" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" @@ -83,17 +83,17 @@ can take some time. }, Before: internal.SubCommandBefore, Action: func(c *cli.Context) error { - appFiles, err := config.LoadAppFiles(listAppServer) + appFiles, err := appPkg.LoadAppFiles(listAppServer) if err != nil { logrus.Fatal(err) } - apps, err := config.GetApps(appFiles, recipeFilter) + apps, err := appPkg.GetApps(appFiles, recipeFilter) if err != nil { logrus.Fatal(err) } - sort.Sort(config.ByServerAndRecipe(apps)) + sort.Sort(appPkg.ByServerAndRecipe(apps)) statuses := make(map[string]map[string]string) var catl recipe.RecipeCatalogue @@ -105,7 +105,7 @@ can take some time. } } - statuses, err = config.GetAppStatuses(apps, internal.MachineReadable) + statuses, err = appPkg.GetAppStatuses(apps, internal.MachineReadable) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/logs.go b/cli/app/logs.go index d54dae1e..e03ba07b 100644 --- a/cli/app/logs.go +++ b/cli/app/logs.go @@ -9,9 +9,9 @@ import ( "time" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/stack" "github.com/docker/docker/api/types" @@ -74,7 +74,7 @@ var appLogsCommand = cli.Command{ // tailLogs prints logs for the given app with optional service names to be // filtered on. It also checks if the latest task is not runnning and then // prints the past tasks. -func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) error { +func tailLogs(cl *dockerClient.Client, app appPkg.App, serviceNames []string) error { f, err := app.Filters(true, false, serviceNames...) if err != nil { return err diff --git a/cli/app/new.go b/cli/app/new.go index e31e970d..acd37205 100644 --- a/cli/app/new.go +++ b/cli/app/new.go @@ -5,6 +5,7 @@ import ( "path" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" @@ -115,10 +116,10 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } - sanitisedAppName := config.SanitiseAppName(internal.Domain) + sanitisedAppName := appPkg.SanitiseAppName(internal.Domain) logrus.Debugf("%s sanitised as %s for new app", internal.Domain, sanitisedAppName) - if err := config.TemplateAppEnvSample( + if err := appPkg.TemplateAppEnvSample( recipe.Name, internal.Domain, internal.NewAppServer, @@ -135,13 +136,13 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } - composeFiles, err := config.GetComposeFiles(recipe.Name, sampleEnv) + composeFiles, err := appPkg.GetComposeFiles(recipe.Name, sampleEnv) if err != nil { logrus.Fatal(err) } envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample") - secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, config.StackName(internal.Domain)) + secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, appPkg.StackName(internal.Domain)) if err != nil { return err } diff --git a/cli/app/ps.go b/cli/app/ps.go index 31ab7e61..b69c6f70 100644 --- a/cli/app/ps.go +++ b/cli/app/ps.go @@ -7,9 +7,9 @@ import ( "time" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/recipe" abraService "coopcloud.tech/abra/pkg/service" @@ -53,7 +53,7 @@ var appPsCommand = cli.Command{ logrus.Fatalf("%s is not deployed?", app.Name) } - statuses, err := config.GetAppStatuses([]config.App{app}, true) + statuses, err := appPkg.GetAppStatuses([]appPkg.App{app}, true) if statusMeta, ok := statuses[app.StackName()]; ok { if _, exists := statusMeta["chaos"]; !exists { if err := recipe.EnsureVersion(app.Recipe, deployedVersion); err != nil { @@ -78,8 +78,8 @@ var appPsCommand = cli.Command{ } // showPSOutput renders ps output. -func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) { - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) +func showPSOutput(c *cli.Context, app appPkg.App, cl *dockerClient.Client) { + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) return @@ -91,7 +91,7 @@ func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) { Prune: false, ResolveImage: stack.ResolveImageAlways, } - compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) + compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { logrus.Fatal(err) return diff --git a/cli/app/rollback.go b/cli/app/rollback.go index 43b1fbba..0e4e7440 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/lint" @@ -198,7 +199,7 @@ recipes. } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - abraShEnv, err := config.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) if err != nil { logrus.Fatal(err) } @@ -206,7 +207,7 @@ recipes. app.Env[k] = v } - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } @@ -217,15 +218,15 @@ recipes. ResolveImage: stack.ResolveImageAlways, Detach: false, } - compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) + compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { logrus.Fatal(err) } - config.ExposeAllEnv(stackName, compose, app.Env) - config.SetRecipeLabel(compose, stackName, app.Recipe) - config.SetChaosLabel(compose, stackName, internal.Chaos) - config.SetChaosVersionLabel(compose, stackName, chosenDowngrade) - config.SetUpdateLabel(compose, stackName, app.Env) + appPkg.ExposeAllEnv(stackName, compose, app.Env) + appPkg.SetRecipeLabel(compose, stackName, app.Recipe) + appPkg.SetChaosLabel(compose, stackName, internal.Chaos) + appPkg.SetChaosVersionLabel(compose, stackName, chosenDowngrade) + appPkg.SetUpdateLabel(compose, stackName, app.Env) // NOTE(d1): no release notes implemeneted for rolling back if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil { diff --git a/cli/app/secret.go b/cli/app/secret.go index c621d51d..4e02f928 100644 --- a/cli/app/secret.go +++ b/cli/app/secret.go @@ -9,9 +9,9 @@ import ( "strings" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/secret" @@ -87,7 +87,7 @@ var appSecretGenerateCommand = cli.Command{ internal.ShowSubcommandHelpAndError(c, err) } - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } @@ -221,7 +221,7 @@ Example: } // secretRm removes a secret. -func secretRm(cl *dockerClient.Client, app config.App, secretName, parsed string) error { +func secretRm(cl *dockerClient.Client, app appPkg.App, secretName, parsed string) error { if err := cl.SecretRemove(context.Background(), secretName); err != nil { return err } @@ -284,7 +284,7 @@ Example: } } - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/undeploy.go b/cli/app/undeploy.go index 7b50908d..7dce914f 100644 --- a/cli/app/undeploy.go +++ b/cli/app/undeploy.go @@ -6,9 +6,9 @@ import ( "time" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" stack "coopcloud.tech/abra/pkg/upstream/stack" "github.com/docker/docker/api/types/filters" @@ -28,7 +28,7 @@ var pruneFlag = &cli.BoolFlag{ // pruneApp runs the equivalent of a "docker system prune" but only filtering // against resources connected with the app deployment. It is not a system wide // prune. Volumes are not pruned to avoid unwated data loss. -func pruneApp(c *cli.Context, cl *dockerClient.Client, app config.App) error { +func pruneApp(c *cli.Context, cl *dockerClient.Client, app appPkg.App) error { stackName := app.StackName() ctx := context.Background() diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index e2f6df6c..1af5a106 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -5,6 +5,7 @@ import ( "fmt" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" @@ -232,7 +233,7 @@ recipes. } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") - abraShEnv, err := config.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) if err != nil { logrus.Fatal(err) } @@ -240,7 +241,7 @@ recipes. app.Env[k] = v } - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { logrus.Fatal(err) } @@ -251,17 +252,17 @@ recipes. ResolveImage: stack.ResolveImageAlways, Detach: false, } - compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) + compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { logrus.Fatal(err) } - config.ExposeAllEnv(stackName, compose, app.Env) - config.SetRecipeLabel(compose, stackName, app.Recipe) - config.SetChaosLabel(compose, stackName, internal.Chaos) - config.SetChaosVersionLabel(compose, stackName, chosenUpgrade) - config.SetUpdateLabel(compose, stackName, app.Env) + appPkg.ExposeAllEnv(stackName, compose, app.Env) + appPkg.SetRecipeLabel(compose, stackName, app.Recipe) + appPkg.SetChaosLabel(compose, stackName, internal.Chaos) + appPkg.SetChaosVersionLabel(compose, stackName, chosenUpgrade) + appPkg.SetUpdateLabel(compose, stackName, app.Env) - envVars, err := config.CheckEnv(app) + envVars, err := appPkg.CheckEnv(app) if err != nil { logrus.Fatal(err) } @@ -282,7 +283,7 @@ recipes. logrus.Fatal(err) } - stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName) + stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName) if err != nil { logrus.Fatal(err) } diff --git a/cli/internal/command.go b/cli/internal/command.go index 13c007be..10f38b7b 100644 --- a/cli/internal/command.go +++ b/cli/internal/command.go @@ -8,7 +8,7 @@ import ( "os/exec" "strings" - "coopcloud.tech/abra/pkg/config" + appPkg "coopcloud.tech/abra/pkg/app" containerPkg "coopcloud.tech/abra/pkg/container" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/upstream/container" @@ -21,7 +21,7 @@ import ( ) // RunCmdRemote executes an abra.sh command in the target service -func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error { +func RunCmdRemote(cl *dockerClient.Client, app appPkg.App, abraSh, serviceName, cmdName, cmdArgs string) error { filters := filters.NewArgs() filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) diff --git a/cli/internal/deploy.go b/cli/internal/deploy.go index ca30aa57..28eaacaa 100644 --- a/cli/internal/deploy.go +++ b/cli/internal/deploy.go @@ -7,6 +7,7 @@ import ( "path" "strings" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" "github.com/AlecAivazis/survey/v2" @@ -15,7 +16,7 @@ import ( ) // NewVersionOverview shows an upgrade or downgrade overview -func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error { +func NewVersionOverview(app appPkg.App, currentVersion, newVersion, releaseNotes string) error { tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"} table := formatter.CreateTable(tableCol) @@ -82,7 +83,7 @@ func GetReleaseNotes(recipeName, version string) (string, error) { // PostCmds parses a string of commands and executes them inside of the respective services // the commands string must have the following format: // " | |... " -func PostCmds(cl *dockerClient.Client, app config.App, commands string) error { +func PostCmds(cl *dockerClient.Client, app appPkg.App, commands string) error { abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh") if _, err := os.Stat(abraSh); err != nil { if os.IsNotExist(err) { @@ -108,7 +109,7 @@ func PostCmds(cl *dockerClient.Client, app config.App, commands string) error { return err } - serviceNames, err := config.GetAppServiceNames(app.Name) + serviceNames, err := appPkg.GetAppServiceNames(app.Name) if err != nil { return err } @@ -135,7 +136,7 @@ func PostCmds(cl *dockerClient.Client, app config.App, commands string) error { } // DeployOverview shows a deployment overview -func DeployOverview(app config.App, version, message string) error { +func DeployOverview(app appPkg.App, version, message string) error { tableCol := []string{"server", "recipe", "config", "domain", "version"} table := formatter.CreateTable(tableCol) diff --git a/cli/internal/validate.go b/cli/internal/validate.go index 46a442d6..fa03d698 100644 --- a/cli/internal/validate.go +++ b/cli/internal/validate.go @@ -78,7 +78,7 @@ func ValidateRecipe(c *cli.Context) recipe.Recipe { } // ValidateApp ensures the app name arg is valid. -func ValidateApp(c *cli.Context) config.App { +func ValidateApp(c *cli.Context) app.App { appName := c.Args().First() if appName == "" { diff --git a/cli/updater/updater.go b/cli/updater/updater.go index 74266d8b..a716def6 100644 --- a/cli/updater/updater.go +++ b/cli/updater/updater.go @@ -8,6 +8,7 @@ import ( "strings" "coopcloud.tech/abra/cli/internal" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/lint" @@ -192,7 +193,7 @@ func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool } // getEnv reads env variables from docker services. -func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) { +func getEnv(cl *dockerclient.Client, stackName string) (appPkg.AppEnv, error) { envMap := make(map[string]string) filter := filters.NewArgs() filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName)) @@ -339,9 +340,9 @@ func processRecipeRepoVersion(recipeName, version string) error { } // mergeAbraShEnv merges abra.sh env vars into the app env vars. -func mergeAbraShEnv(recipeName string, env config.AppEnv) error { +func mergeAbraShEnv(recipeName string, env appPkg.AppEnv) error { abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh") - abraShEnv, err := config.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath) if err != nil { return err } @@ -355,7 +356,7 @@ func mergeAbraShEnv(recipeName string, env config.AppEnv) error { } // createDeployConfig merges and enriches the compose config for the deployment. -func createDeployConfig(recipeName string, stackName string, env config.AppEnv) (*composetypes.Config, stack.Deploy, error) { +func createDeployConfig(recipeName string, stackName string, env appPkg.AppEnv) (*composetypes.Config, stack.Deploy, error) { env["STACK_NAME"] = stackName deployOpts := stack.Deploy{ @@ -365,23 +366,23 @@ func createDeployConfig(recipeName string, stackName string, env config.AppEnv) Detach: false, } - composeFiles, err := config.GetComposeFiles(recipeName, env) + composeFiles, err := appPkg.GetComposeFiles(recipeName, env) if err != nil { return nil, deployOpts, err } deployOpts.Composefiles = composeFiles - compose, err := config.GetAppComposeConfig(stackName, deployOpts, env) + compose, err := appPkg.GetAppComposeConfig(stackName, deployOpts, env) if err != nil { return nil, deployOpts, err } - config.ExposeAllEnv(stackName, compose, env) + appPkg.ExposeAllEnv(stackName, compose, env) // after the upgrade the deployment won't be in chaos state anymore - config.SetChaosLabel(compose, stackName, false) - config.SetRecipeLabel(compose, stackName, recipeName) - config.SetUpdateLabel(compose, stackName, env) + appPkg.SetChaosLabel(compose, stackName, false) + appPkg.SetRecipeLabel(compose, stackName, recipeName) + appPkg.SetUpdateLabel(compose, stackName, env) return compose, deployOpts, nil } @@ -436,7 +437,7 @@ func upgrade(cl *dockerclient.Client, stackName, recipeName, return err } - app := config.App{ + app := appPkg.App{ Name: stackName, Recipe: recipeName, Server: SERVER, diff --git a/pkg/app/app.go b/pkg/app/app.go index 26a84e28..2e250916 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -1,22 +1,34 @@ package app import ( + "fmt" + "os" + "path" "strings" + "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/formatter" + "coopcloud.tech/abra/pkg/upstream/convert" + "coopcloud.tech/abra/pkg/upstream/stack" + + loader "coopcloud.tech/abra/pkg/upstream/stack" + composetypes "github.com/docker/cli/cli/compose/types" + "github.com/docker/docker/api/types/filters" + "github.com/schollz/progressbar/v3" "github.com/sirupsen/logrus" ) // Get retrieves an app -func Get(appName string) (config.App, error) { - files, err := config.LoadAppFiles("") +func Get(appName string) (App, error) { + files, err := LoadAppFiles("") if err != nil { - return config.App{}, err + return App{}, err } - app, err := config.GetApp(files, appName) + app, err := GetApp(files, appName) if err != nil { - return config.App{}, err + return App{}, err } logrus.Debugf("retrieved %s for %s", app, appName) @@ -24,19 +36,530 @@ func Get(appName string) (config.App, error) { return app, nil } -// deployedServiceSpec represents a deployed service of an app. -type deployedServiceSpec struct { - Name string - Version string +// GetApp loads an apps settings, reading it from file, in preparation to use +// it. It should only be used when ready to use the env file to keep IO +// operations down. +func GetApp(apps AppFiles, name AppName) (App, error) { + appFile, exists := apps[name] + if !exists { + return App{}, fmt.Errorf("cannot find app with name %s", name) + } + + app, err := ReadAppEnvFile(appFile, name) + if err != nil { + return App{}, err + } + + return app, nil } -// VersionSpec represents a deployed app and associated metadata. -type VersionSpec map[string]deployedServiceSpec +// GetApps returns a slice of Apps with their env files read from a given +// slice of AppFiles. +func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) { + var apps []App -// ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label. -func ParseServiceName(label string) string { - idx := strings.LastIndex(label, "_") - serviceName := label[idx+1:] - logrus.Debugf("parsed %s as service name from %s", serviceName, label) - return serviceName + for name := range appFiles { + app, err := GetApp(appFiles, name) + if err != nil { + return nil, err + } + + if recipeFilter != "" { + if app.Recipe == recipeFilter { + apps = append(apps, app) + } + } else { + apps = append(apps, app) + } + } + + return apps, nil +} + +// App reprents an app with its env file read into memory +type App struct { + Name AppName + Recipe string + Domain string + Env AppEnv + Server string + Path string +} + +// Type aliases to make code hints easier to understand + +// AppEnv is a map of the values in an apps env config +type AppEnv = map[string]string + +// AppModifiers is a map of modifiers in an apps env config +type AppModifiers = map[string]map[string]string + +// AppName is AppName +type AppName = string + +// AppFile represents app env files on disk without reading the contents +type AppFile struct { + Path string + Server string +} + +// AppFiles is a slice of appfiles +type AppFiles map[AppName]AppFile + +// See documentation of config.StackName +func (a App) StackName() string { + if _, exists := a.Env["STACK_NAME"]; exists { + return a.Env["STACK_NAME"] + } + + stackName := StackName(a.Name) + + a.Env["STACK_NAME"] = stackName + + return stackName +} + +// StackName gets whatever the docker safe (uses the right delimiting +// character, e.g. "_") stack name is for the app. In general, you don't want +// to use this to show anything to end-users, you want use a.Name instead. +func StackName(appName string) string { + stackName := SanitiseAppName(appName) + + if len(stackName) > config.MAX_SANITISED_APP_NAME_LENGTH { + logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:config.MAX_SANITISED_APP_NAME_LENGTH]) + stackName = stackName[:config.MAX_SANITISED_APP_NAME_LENGTH] + } + + return stackName +} + +// Filters retrieves app filters for querying the container runtime. By default +// it filters on all services in the app. It is also possible to pass an +// otional list of service names, which get filtered instead. +// +// Due to upstream issues, filtering works different depending on what you're +// querying. So, for example, secrets don't work with regex! The caller needs +// to implement their own validation that the right secrets are matched. In +// order to handle these cases, we provide the `appendServiceNames` / +// `exactMatch` modifiers. +func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) { + filters := filters.NewArgs() + if len(services) > 0 { + for _, serviceName := range services { + filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch)) + } + return filters, nil + } + + // When not appending the service name, just add one filter for the whole + // stack. + if !appendServiceNames { + f := fmt.Sprintf("%s", a.StackName()) + if exactMatch { + f = fmt.Sprintf("^%s", f) + } + filters.Add("name", f) + return filters, nil + } + + composeFiles, err := GetComposeFiles(a.Recipe, a.Env) + if err != nil { + return filters, err + } + + opts := stack.Deploy{Composefiles: composeFiles} + compose, err := GetAppComposeConfig(a.Recipe, opts, a.Env) + if err != nil { + return filters, err + } + + for _, service := range compose.Services { + f := ServiceFilter(a.StackName(), service.Name, exactMatch) + filters.Add("name", f) + } + + return filters, nil +} + +// ServiceFilter creates a filter string for filtering a service in the docker +// container runtime. When exact match is true, it uses regex to match the +// string exactly. +func ServiceFilter(stack, service string, exact bool) string { + if exact { + return fmt.Sprintf("^%s_%s", stack, service) + } + return fmt.Sprintf("%s_%s", stack, service) +} + +// ByServer sort a slice of Apps +type ByServer []App + +func (a ByServer) Len() int { return len(a) } +func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByServer) Less(i, j int) bool { + return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server) +} + +// ByServerAndRecipe sort a slice of Apps +type ByServerAndRecipe []App + +func (a ByServerAndRecipe) Len() int { return len(a) } +func (a ByServerAndRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByServerAndRecipe) Less(i, j int) bool { + if a[i].Server == a[j].Server { + return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe) + } + return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server) +} + +// ByRecipe sort a slice of Apps +type ByRecipe []App + +func (a ByRecipe) Len() int { return len(a) } +func (a ByRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByRecipe) Less(i, j int) bool { + return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe) +} + +// ByName sort a slice of Apps +type ByName []App + +func (a ByName) Len() int { return len(a) } +func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByName) Less(i, j int) bool { + return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name) +} + +func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) { + env, err := ReadEnv(appFile.Path) + if err != nil { + return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error()) + } + + logrus.Debugf("read env %s from %s", env, appFile.Path) + + app, err := NewApp(env, name, appFile) + if err != nil { + return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error()) + } + + return app, nil +} + +// NewApp creates new App object +func NewApp(env AppEnv, name string, appFile AppFile) (App, error) { + domain := env["DOMAIN"] + + recipe, exists := env["RECIPE"] + if !exists { + recipe, exists = env["TYPE"] + if !exists { + return App{}, fmt.Errorf("%s is missing the TYPE env var?", name) + } + } + + return App{ + Name: name, + Domain: domain, + Recipe: recipe, + Env: env, + Server: appFile.Server, + Path: appFile.Path, + }, nil +} + +// LoadAppFiles gets all app files for a given set of servers or all servers. +func LoadAppFiles(servers ...string) (AppFiles, error) { + appFiles := make(AppFiles) + if len(servers) == 1 { + if servers[0] == "" { + // Empty servers flag, one string will always be passed + var err error + servers, err = config.GetAllFoldersInDirectory(config.SERVERS_DIR) + if err != nil { + return appFiles, err + } + } + } + + logrus.Debugf("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", ")) + + for _, server := range servers { + serverDir := path.Join(config.SERVERS_DIR, server) + files, err := config.GetAllFilesInDirectory(serverDir) + if err != nil { + return appFiles, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server) + } + + for _, file := range files { + appName := strings.TrimSuffix(file.Name(), ".env") + appFilePath := path.Join(config.SERVERS_DIR, server, file.Name()) + appFiles[appName] = AppFile{ + Path: appFilePath, + Server: server, + } + } + } + + return appFiles, nil +} + +// GetAppServiceNames retrieves a list of app service names. +func GetAppServiceNames(appName string) ([]string, error) { + var serviceNames []string + + appFiles, err := LoadAppFiles("") + if err != nil { + return serviceNames, err + } + + app, err := GetApp(appFiles, appName) + if err != nil { + return serviceNames, err + } + + composeFiles, err := GetComposeFiles(app.Recipe, app.Env) + if err != nil { + return serviceNames, err + } + + opts := stack.Deploy{Composefiles: composeFiles} + compose, err := GetAppComposeConfig(app.Recipe, opts, app.Env) + if err != nil { + return serviceNames, err + } + + for _, service := range compose.Services { + serviceNames = append(serviceNames, service.Name) + } + + return serviceNames, nil +} + +// GetAppNames retrieves a list of app names. +func GetAppNames() ([]string, error) { + var appNames []string + + appFiles, err := LoadAppFiles("") + if err != nil { + return appNames, err + } + + apps, err := GetApps(appFiles, "") + if err != nil { + return appNames, err + } + + for _, app := range apps { + appNames = append(appNames, app.Name) + } + + return appNames, nil +} + +// TemplateAppEnvSample copies the example env file for the app into the users +// env files. +func TemplateAppEnvSample(recipeName, appName, server, domain string) error { + envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") + envSample, err := os.ReadFile(envSamplePath) + if err != nil { + return err + } + + appEnvPath := path.Join(config.ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName)) + if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) { + return fmt.Errorf("%s already exists?", appEnvPath) + } + + err = os.WriteFile(appEnvPath, envSample, 0o664) + if err != nil { + return err + } + + read, err := os.ReadFile(appEnvPath) + if err != nil { + return err + } + + newContents := strings.Replace(string(read), recipeName+".example.com", domain, -1) + + err = os.WriteFile(appEnvPath, []byte(newContents), 0) + if err != nil { + return err + } + + logrus.Debugf("copied & templated %s to %s", envSamplePath, appEnvPath) + + return nil +} + +// SanitiseAppName makes a app name usable with Docker by replacing illegal +// characters. +func SanitiseAppName(name string) string { + return strings.ReplaceAll(name, ".", "_") +} + +// GetAppStatuses queries servers to check the deployment status of given apps. +func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) { + statuses := make(map[string]map[string]string) + + servers := make(map[string]struct{}) + for _, app := range apps { + if _, ok := servers[app.Server]; !ok { + servers[app.Server] = struct{}{} + } + } + + var bar *progressbar.ProgressBar + if !MachineReadable { + bar = formatter.CreateProgressbar(len(servers), "querying remote servers...") + } + + ch := make(chan stack.StackStatus, len(servers)) + for server := range servers { + cl, err := client.New(server) + if err != nil { + return statuses, err + } + + go func(s string) { + ch <- stack.GetAllDeployedServices(cl, s) + if !MachineReadable { + bar.Add(1) + } + }(server) + } + + for range servers { + status := <-ch + if status.Err != nil { + return statuses, status.Err + } + + for _, service := range status.Services { + result := make(map[string]string) + name := service.Spec.Labels[convert.LabelNamespace] + + if _, ok := statuses[name]; !ok { + result["status"] = "deployed" + } + + labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name) + chaos, ok := service.Spec.Labels[labelKey] + if ok { + result["chaos"] = chaos + } + + labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name) + if chaosVersion, ok := service.Spec.Labels[labelKey]; ok { + result["chaosVersion"] = chaosVersion + } + + labelKey = fmt.Sprintf("coop-cloud.%s.autoupdate", name) + if autoUpdate, ok := service.Spec.Labels[labelKey]; ok { + result["autoUpdate"] = autoUpdate + } else { + result["autoUpdate"] = "false" + } + + labelKey = fmt.Sprintf("coop-cloud.%s.version", name) + if version, ok := service.Spec.Labels[labelKey]; ok { + result["version"] = version + } else { + continue + } + + statuses[name] = result + } + } + + logrus.Debugf("retrieved app statuses: %s", statuses) + + return statuses, nil +} + +// ensurePathExists ensures that a path exists. +func ensurePathExists(path string) error { + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { + return err + } + return nil +} + +// GetComposeFiles gets the list of compose files for an app (or recipe if you +// don't already have an app) which should be merged into a composetypes.Config +// while respecting the COMPOSE_FILE env var. +func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) { + var composeFiles []string + + composeFileEnvVar, ok := appEnv["COMPOSE_FILE"] + if !ok { + path := fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, recipe) + if err := ensurePathExists(path); err != nil { + return composeFiles, err + } + logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path) + composeFiles = append(composeFiles, path) + return composeFiles, nil + } + + if !strings.Contains(composeFileEnvVar, ":") { + path := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipe, composeFileEnvVar) + if err := ensurePathExists(path); err != nil { + return composeFiles, err + } + logrus.Debugf("COMPOSE_FILE detected, loading %s", path) + composeFiles = append(composeFiles, path) + return composeFiles, nil + } + + numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1 + envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles) + if len(envVars) != numComposeFiles { + return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar) + } + + for _, file := range envVars { + path := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipe, file) + if err := ensurePathExists(path); err != nil { + return composeFiles, err + } + composeFiles = append(composeFiles, path) + } + + logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", ")) + logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe) + + return composeFiles, nil +} + +// GetAppComposeConfig retrieves a compose specification for a recipe. This +// specification is the result of a merge of all the compose.**.yml files in +// the recipe repository. +func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) { + compose, err := loader.LoadComposefile(opts, appEnv) + if err != nil { + return &composetypes.Config{}, err + } + + logrus.Debugf("retrieved %s for %s", compose.Filename, recipe) + + return compose, nil +} + +// ExposeAllEnv exposes all env variables to the app container +func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) { + for _, service := range compose.Services { + if service.Name == "app" { + logrus.Debugf("Add the following environment to the app service config of %s:", stackName) + for k, v := range appEnv { + _, exists := service.Environment[k] + if !exists { + value := v + service.Environment[k] = &value + logrus.Debugf("Add Key: %s Value: %s to %s", k, value, stackName) + } + } + } + } } diff --git a/pkg/config/app_test.go b/pkg/app/app_test.go similarity index 88% rename from pkg/config/app_test.go rename to pkg/app/app_test.go index 0823f4e2..a568c1bf 100644 --- a/pkg/config/app_test.go +++ b/pkg/app/app_test.go @@ -1,4 +1,4 @@ -package config_test +package app_test import ( "encoding/json" @@ -6,6 +6,7 @@ import ( "reflect" "testing" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" "github.com/docker/docker/api/types/filters" @@ -14,7 +15,7 @@ import ( ) func TestNewApp(t *testing.T) { - app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile) + app, err := appPkg.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile) if err != nil { t.Fatal(err) } @@ -24,7 +25,7 @@ func TestNewApp(t *testing.T) { } func TestReadAppEnvFile(t *testing.T) { - app, err := config.ReadAppEnvFile(ExpectedAppFile, AppName) + app, err := appPkg.ReadAppEnvFile(ExpectedAppFile, AppName) if err != nil { t.Fatal(err) } @@ -34,7 +35,7 @@ func TestReadAppEnvFile(t *testing.T) { } func TestGetApp(t *testing.T) { - app, err := config.GetApp(ExpectedAppFiles, AppName) + app, err := appPkg.GetApp(ExpectedAppFiles, AppName) if err != nil { t.Fatal(err) } @@ -82,7 +83,7 @@ func TestGetComposeFiles(t *testing.T) { } for _, test := range tests { - composeFiles, err := config.GetComposeFiles(r.Name, test.appEnv) + composeFiles, err := appPkg.GetComposeFiles(r.Name, test.appEnv) if err != nil { t.Fatal(err) } @@ -103,7 +104,7 @@ func TestGetComposeFilesError(t *testing.T) { } for _, test := range tests { - _, err := config.GetComposeFiles(r.Name, test.appEnv) + _, err := appPkg.GetComposeFiles(r.Name, test.appEnv) if err == nil { t.Fatalf("should have failed: %v", test.appEnv) } @@ -112,16 +113,16 @@ func TestGetComposeFilesError(t *testing.T) { func TestFilters(t *testing.T) { oldDir := config.RECIPES_DIR - config.RECIPES_DIR = "./testdir" + config.RECIPES_DIR = "./testdata" defer func() { config.RECIPES_DIR = oldDir }() - app, err := config.NewApp(config.AppEnv{ + app, err := appPkg.NewApp(appPkg.AppEnv{ "DOMAIN": "test.example.com", "RECIPE": "test-recipe", - }, "test_example_com", config.AppFile{ - Path: "./testdir/filtertest.end", + }, "test_example_com", appPkg.AppFile{ + Path: "./testdata/filtertest.end", Server: "local", }) if err != nil { diff --git a/pkg/app/compose.go b/pkg/app/compose.go new file mode 100644 index 00000000..771f56b6 --- /dev/null +++ b/pkg/app/compose.go @@ -0,0 +1,87 @@ +package app + +import ( + "fmt" + "strconv" + + composetypes "github.com/docker/cli/cli/compose/types" + "github.com/sirupsen/logrus" +) + +// SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container +// to signal which recipe is connected to the deployed app +func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) { + for _, service := range compose.Services { + if service.Name == "app" { + logrus.Debugf("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName) + labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName) + service.Deploy.Labels[labelKey] = recipe + } + } +} + +// SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container +// to signal if the app is deployed in chaos mode +func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) { + for _, service := range compose.Services { + if service.Name == "app" { + logrus.Debugf("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName) + labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName) + service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos) + } + } +} + +// SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container +func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) { + for _, service := range compose.Services { + if service.Name == "app" { + logrus.Debugf("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName) + labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName) + service.Deploy.Labels[labelKey] = chaosVersion + } + } +} + +// SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the +// auto update process for this app. The default if this variable is not set is to disable +// the auto update process. +func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) { + for _, service := range compose.Services { + if service.Name == "app" { + enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"] + if !exists { + enable_auto_update = "false" + } + logrus.Debugf("set label 'coop-cloud.%s.autoupdate' to %s for %s", stackName, enable_auto_update, stackName) + labelKey := fmt.Sprintf("coop-cloud.%s.autoupdate", stackName) + service.Deploy.Labels[labelKey] = enable_auto_update + } + } +} + +// GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files +func GetLabel(compose *composetypes.Config, stackName string, label string) string { + for _, service := range compose.Services { + if service.Name == "app" { + labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label) + logrus.Debugf("get label '%s'", labelKey) + if labelValue, ok := service.Deploy.Labels[labelKey]; ok { + return labelValue + } + } + } + logrus.Debugf("no %s label found for %s", label, stackName) + return "" +} + +// GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value +func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) { + timeout := 50 // Default Timeout + var err error = nil + if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" { + logrus.Debugf("timeout label: %s", timeoutLabel) + timeout, err = strconv.Atoi(timeoutLabel) + } + return timeout, err +} diff --git a/pkg/app/env.go b/pkg/app/env.go new file mode 100644 index 00000000..5daa526a --- /dev/null +++ b/pkg/app/env.go @@ -0,0 +1,159 @@ +package app + +import ( + "bufio" + "fmt" + "os" + "path" + "regexp" + "sort" + "strings" + + "coopcloud.tech/abra/pkg/config" + "git.coopcloud.tech/coop-cloud/godotenv" + "github.com/sirupsen/logrus" +) + +// ReadEnv loads an app envivornment into a map. +func ReadEnv(filePath string) (AppEnv, error) { + var envVars AppEnv + + envVars, _, err := godotenv.Read(filePath) + if err != nil { + return nil, err + } + + logrus.Debugf("read %s from %s", envVars, filePath) + + return envVars, nil +} + +// ReadEnv loads an app envivornment and their modifiers in two different maps. +func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) { + var envVars AppEnv + + envVars, mods, err := godotenv.Read(filePath) + if err != nil { + return nil, mods, err + } + + logrus.Debugf("read %s from %s", envVars, filePath) + + return envVars, mods, nil +} + +// ReadAbraShEnvVars reads env vars from an abra.sh recipe file. +func ReadAbraShEnvVars(abraSh string) (map[string]string, error) { + envVars := make(map[string]string) + + file, err := os.Open(abraSh) + if err != nil { + if os.IsNotExist(err) { + return envVars, nil + } + return envVars, err + } + defer file.Close() + + exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`) + if err != nil { + return envVars, err + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + txt := scanner.Text() + if exportRegex.MatchString(txt) { + splitVals := strings.Split(txt, "export ") + envVarDef := splitVals[len(splitVals)-1] + keyVal := strings.Split(envVarDef, "=") + if len(keyVal) != 2 { + return envVars, fmt.Errorf("couldn't parse %s", txt) + } + envVars[keyVal[0]] = keyVal[1] + } + } + + if len(envVars) > 0 { + logrus.Debugf("read %s from %s", envVars, abraSh) + } else { + logrus.Debugf("read 0 env var exports from %s", abraSh) + } + + return envVars, nil +} + +type EnvVar struct { + Name string + Present bool +} + +func CheckEnv(app App) ([]EnvVar, error) { + var envVars []EnvVar + + envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample") + if _, err := os.Stat(envSamplePath); err != nil { + if os.IsNotExist(err) { + return envVars, fmt.Errorf("%s does not exist?", envSamplePath) + } + return envVars, err + } + + envSample, err := ReadEnv(envSamplePath) + if err != nil { + return envVars, err + } + + var keys []string + for key := range envSample { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + if _, ok := app.Env[key]; ok { + envVars = append(envVars, EnvVar{Name: key, Present: true}) + } else { + envVars = append(envVars, EnvVar{Name: key, Present: false}) + } + } + + return envVars, nil +} + +// ReadAbraShCmdNames reads the names of commands. +func ReadAbraShCmdNames(abraSh string) ([]string, error) { + var cmdNames []string + + file, err := os.Open(abraSh) + if err != nil { + if os.IsNotExist(err) { + return cmdNames, nil + } + return cmdNames, err + } + defer file.Close() + + cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`) + if err != nil { + return cmdNames, err + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + matches := cmdNameRegex.FindStringSubmatch(line) + if len(matches) > 0 { + cmdNames = append(cmdNames, matches[1]) + } + } + + if len(cmdNames) > 0 { + logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh) + } else { + logrus.Debugf("read 0 command names from %s", abraSh) + } + + return cmdNames, nil +} diff --git a/pkg/config/env_test.go b/pkg/app/env_test.go similarity index 89% rename from pkg/config/env_test.go rename to pkg/app/env_test.go index f0626e6b..33f894c1 100644 --- a/pkg/config/env_test.go +++ b/pkg/app/env_test.go @@ -1,4 +1,4 @@ -package config_test +package app_test import ( "fmt" @@ -9,6 +9,8 @@ import ( "strings" "testing" + "coopcloud.tech/abra/pkg/app" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" ) @@ -29,12 +31,12 @@ var ( ServerName = "evil.corp" ) -var ExpectedAppEnv = config.AppEnv{ +var ExpectedAppEnv = app.AppEnv{ "DOMAIN": "ecloud.evil.corp", "RECIPE": "ecloud", } -var ExpectedApp = config.App{ +var ExpectedApp = app.App{ Name: AppName, Recipe: ExpectedAppEnv["RECIPE"], Domain: ExpectedAppEnv["DOMAIN"], @@ -43,12 +45,12 @@ var ExpectedApp = config.App{ Server: ExpectedAppFile.Server, } -var ExpectedAppFile = config.AppFile{ +var ExpectedAppFile = app.AppFile{ Path: path.Join(ValidAbraConf, "servers", ServerName, AppName+".env"), Server: ServerName, } -var ExpectedAppFiles = map[string]config.AppFile{ +var ExpectedAppFiles = map[string]app.AppFile{ AppName: ExpectedAppFile, } @@ -77,7 +79,7 @@ func TestGetAllFilesInDirectory(t *testing.T) { } func TestReadEnv(t *testing.T) { - env, err := config.ReadEnv(ExpectedAppFile.Path) + env, err := app.ReadEnv(ExpectedAppFile.Path) if err != nil { t.Fatal(err) } @@ -100,7 +102,7 @@ func TestReadAbraShEnvVars(t *testing.T) { } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh") - abraShEnv, err := config.ReadAbraShEnvVars(abraShPath) + abraShEnv, err := app.ReadAbraShEnvVars(abraShPath) if err != nil { t.Fatal(err) } @@ -130,7 +132,7 @@ func TestReadAbraShCmdNames(t *testing.T) { } abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh") - cmdNames, err := config.ReadAbraShCmdNames(abraShPath) + cmdNames, err := app.ReadAbraShCmdNames(abraShPath) if err != nil { t.Fatal(err) } @@ -155,12 +157,12 @@ func TestCheckEnv(t *testing.T) { } envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") - envSample, err := config.ReadEnv(envSamplePath) + envSample, err := app.ReadEnv(envSamplePath) if err != nil { t.Fatal(err) } - app := config.App{ + app := app.App{ Name: "test-app", Recipe: r.Name, Domain: "example.com", @@ -169,7 +171,7 @@ func TestCheckEnv(t *testing.T) { Server: "example.com", } - envVars, err := config.CheckEnv(app) + envVars, err := appPkg.CheckEnv(app) if err != nil { t.Fatal(err) } @@ -189,14 +191,14 @@ func TestCheckEnvError(t *testing.T) { } envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") - envSample, err := config.ReadEnv(envSamplePath) + envSample, err := app.ReadEnv(envSamplePath) if err != nil { t.Fatal(err) } delete(envSample, "DOMAIN") - app := config.App{ + app := app.App{ Name: "test-app", Recipe: r.Name, Domain: "example.com", @@ -205,7 +207,7 @@ func TestCheckEnvError(t *testing.T) { Server: "example.com", } - envVars, err := config.CheckEnv(app) + envVars, err := appPkg.CheckEnv(app) if err != nil { t.Fatal(err) } @@ -225,7 +227,7 @@ func TestEnvVarCommentsRemoved(t *testing.T) { } envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") - envSample, err := config.ReadEnv(envSamplePath) + envSample, err := app.ReadEnv(envSamplePath) if err != nil { t.Fatal(err) } @@ -257,7 +259,7 @@ func TestEnvVarModifiersIncluded(t *testing.T) { } envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") - envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath) + envSample, modifiers, err := app.ReadEnvWithModifiers(envSamplePath) if err != nil { t.Fatal(err) } diff --git a/pkg/config/testdir/filtertest.env b/pkg/app/testdata/filtertest.env similarity index 100% rename from pkg/config/testdir/filtertest.env rename to pkg/app/testdata/filtertest.env diff --git a/pkg/config/testdir/test-recipe/compose.yml b/pkg/app/testdata/test-recipe/compose.yml similarity index 100% rename from pkg/config/testdir/test-recipe/compose.yml rename to pkg/app/testdata/test-recipe/compose.yml diff --git a/pkg/autocomplete/autocomplete.go b/pkg/autocomplete/autocomplete.go index 7b28f134..5a499e2b 100644 --- a/pkg/autocomplete/autocomplete.go +++ b/pkg/autocomplete/autocomplete.go @@ -3,7 +3,7 @@ package autocomplete import ( "fmt" - "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -11,7 +11,7 @@ import ( // AppNameComplete copletes app names. func AppNameComplete(c *cli.Context) { - appNames, err := config.GetAppNames() + appNames, err := app.GetAppNames() if err != nil { logrus.Warn(err) } @@ -26,7 +26,7 @@ func AppNameComplete(c *cli.Context) { } func ServiceNameComplete(appName string) { - serviceNames, err := config.GetAppServiceNames(appName) + serviceNames, err := app.GetAppServiceNames(appName) if err != nil { return } @@ -67,7 +67,7 @@ func RecipeVersionComplete(recipeName string) { // ServerNameComplete completes server names. func ServerNameComplete(c *cli.Context) { - files, err := config.LoadAppFiles("") + files, err := app.LoadAppFiles("") if err != nil { logrus.Fatal(err) } diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go index 8a641a6f..d6753327 100644 --- a/pkg/compose/compose.go +++ b/pkg/compose/compose.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/upstream/stack" @@ -29,7 +30,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) { opts := stack.Deploy{Composefiles: []string{composeFile}} envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") - sampleEnv, err := config.ReadEnv(envSamplePath) + sampleEnv, err := app.ReadEnv(envSamplePath) if err != nil { return false, err } @@ -97,7 +98,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error { opts := stack.Deploy{Composefiles: []string{composeFile}} envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") - sampleEnv, err := config.ReadEnv(envSamplePath) + sampleEnv, err := app.ReadEnv(envSamplePath) if err != nil { return err } diff --git a/pkg/config/app.go b/pkg/config/app.go deleted file mode 100644 index 601d0636..00000000 --- a/pkg/config/app.go +++ /dev/null @@ -1,627 +0,0 @@ -package config - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strconv" - "strings" - - "github.com/schollz/progressbar/v3" - - "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/formatter" - "coopcloud.tech/abra/pkg/upstream/convert" - loader "coopcloud.tech/abra/pkg/upstream/stack" - stack "coopcloud.tech/abra/pkg/upstream/stack" - composetypes "github.com/docker/cli/cli/compose/types" - "github.com/docker/docker/api/types/filters" - "github.com/sirupsen/logrus" -) - -// Type aliases to make code hints easier to understand - -// AppEnv is a map of the values in an apps env config -type AppEnv = map[string]string - -// AppModifiers is a map of modifiers in an apps env config -type AppModifiers = map[string]map[string]string - -// AppName is AppName -type AppName = string - -// AppFile represents app env files on disk without reading the contents -type AppFile struct { - Path string - Server string -} - -// AppFiles is a slice of appfiles -type AppFiles map[AppName]AppFile - -// App reprents an app with its env file read into memory -type App struct { - Name AppName - Recipe string - Domain string - Env AppEnv - Server string - Path string -} - -// See documentation of config.StackName -func (a App) StackName() string { - if _, exists := a.Env["STACK_NAME"]; exists { - return a.Env["STACK_NAME"] - } - - stackName := StackName(a.Name) - - a.Env["STACK_NAME"] = stackName - - return stackName -} - -// StackName gets whatever the docker safe (uses the right delimiting -// character, e.g. "_") stack name is for the app. In general, you don't want -// to use this to show anything to end-users, you want use a.Name instead. -func StackName(appName string) string { - stackName := SanitiseAppName(appName) - - if len(stackName) > MAX_SANITISED_APP_NAME_LENGTH { - logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:MAX_SANITISED_APP_NAME_LENGTH]) - stackName = stackName[:MAX_SANITISED_APP_NAME_LENGTH] - } - - return stackName -} - -// Filters retrieves app filters for querying the container runtime. By default -// it filters on all services in the app. It is also possible to pass an -// otional list of service names, which get filtered instead. -// -// Due to upstream issues, filtering works different depending on what you're -// querying. So, for example, secrets don't work with regex! The caller needs -// to implement their own validation that the right secrets are matched. In -// order to handle these cases, we provide the `appendServiceNames` / -// `exactMatch` modifiers. -func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) { - filters := filters.NewArgs() - if len(services) > 0 { - for _, serviceName := range services { - filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch)) - } - return filters, nil - } - - // When not appending the service name, just add one filter for the whole - // stack. - if !appendServiceNames { - f := fmt.Sprintf("%s", a.StackName()) - if exactMatch { - f = fmt.Sprintf("^%s", f) - } - filters.Add("name", f) - return filters, nil - } - - composeFiles, err := GetComposeFiles(a.Recipe, a.Env) - if err != nil { - return filters, err - } - - opts := stack.Deploy{Composefiles: composeFiles} - compose, err := GetAppComposeConfig(a.Recipe, opts, a.Env) - if err != nil { - return filters, err - } - - for _, service := range compose.Services { - f := ServiceFilter(a.StackName(), service.Name, exactMatch) - filters.Add("name", f) - } - - return filters, nil -} - -// ServiceFilter creates a filter string for filtering a service in the docker -// container runtime. When exact match is true, it uses regex to match the -// string exactly. -func ServiceFilter(stack, service string, exact bool) string { - if exact { - return fmt.Sprintf("^%s_%s", stack, service) - } - return fmt.Sprintf("%s_%s", stack, service) -} - -// ByServer sort a slice of Apps -type ByServer []App - -func (a ByServer) Len() int { return len(a) } -func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByServer) Less(i, j int) bool { - return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server) -} - -// ByServerAndRecipe sort a slice of Apps -type ByServerAndRecipe []App - -func (a ByServerAndRecipe) Len() int { return len(a) } -func (a ByServerAndRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByServerAndRecipe) Less(i, j int) bool { - if a[i].Server == a[j].Server { - return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe) - } - return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server) -} - -// ByRecipe sort a slice of Apps -type ByRecipe []App - -func (a ByRecipe) Len() int { return len(a) } -func (a ByRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByRecipe) Less(i, j int) bool { - return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe) -} - -// ByName sort a slice of Apps -type ByName []App - -func (a ByName) Len() int { return len(a) } -func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByName) Less(i, j int) bool { - return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name) -} - -func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) { - env, err := ReadEnv(appFile.Path) - if err != nil { - return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error()) - } - - logrus.Debugf("read env %s from %s", env, appFile.Path) - - app, err := NewApp(env, name, appFile) - if err != nil { - return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error()) - } - - return app, nil -} - -// NewApp creates new App object -func NewApp(env AppEnv, name string, appFile AppFile) (App, error) { - domain := env["DOMAIN"] - - recipe, exists := env["RECIPE"] - if !exists { - recipe, exists = env["TYPE"] - if !exists { - return App{}, fmt.Errorf("%s is missing the TYPE env var?", name) - } - } - - return App{ - Name: name, - Domain: domain, - Recipe: recipe, - Env: env, - Server: appFile.Server, - Path: appFile.Path, - }, nil -} - -// LoadAppFiles gets all app files for a given set of servers or all servers. -func LoadAppFiles(servers ...string) (AppFiles, error) { - appFiles := make(AppFiles) - if len(servers) == 1 { - if servers[0] == "" { - // Empty servers flag, one string will always be passed - var err error - servers, err = GetAllFoldersInDirectory(SERVERS_DIR) - if err != nil { - return appFiles, err - } - } - } - - logrus.Debugf("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", ")) - - for _, server := range servers { - serverDir := path.Join(SERVERS_DIR, server) - files, err := GetAllFilesInDirectory(serverDir) - if err != nil { - return appFiles, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server) - } - - for _, file := range files { - appName := strings.TrimSuffix(file.Name(), ".env") - appFilePath := path.Join(SERVERS_DIR, server, file.Name()) - appFiles[appName] = AppFile{ - Path: appFilePath, - Server: server, - } - } - } - - return appFiles, nil -} - -// GetApp loads an apps settings, reading it from file, in preparation to use -// it. It should only be used when ready to use the env file to keep IO -// operations down. -func GetApp(apps AppFiles, name AppName) (App, error) { - appFile, exists := apps[name] - if !exists { - return App{}, fmt.Errorf("cannot find app with name %s", name) - } - - app, err := ReadAppEnvFile(appFile, name) - if err != nil { - return App{}, err - } - - return app, nil -} - -// GetApps returns a slice of Apps with their env files read from a given -// slice of AppFiles. -func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) { - var apps []App - - for name := range appFiles { - app, err := GetApp(appFiles, name) - if err != nil { - return nil, err - } - - if recipeFilter != "" { - if app.Recipe == recipeFilter { - apps = append(apps, app) - } - } else { - apps = append(apps, app) - } - } - - return apps, nil -} - -// GetAppServiceNames retrieves a list of app service names. -func GetAppServiceNames(appName string) ([]string, error) { - var serviceNames []string - - appFiles, err := LoadAppFiles("") - if err != nil { - return serviceNames, err - } - - app, err := GetApp(appFiles, appName) - if err != nil { - return serviceNames, err - } - - composeFiles, err := GetComposeFiles(app.Recipe, app.Env) - if err != nil { - return serviceNames, err - } - - opts := stack.Deploy{Composefiles: composeFiles} - compose, err := GetAppComposeConfig(app.Recipe, opts, app.Env) - if err != nil { - return serviceNames, err - } - - for _, service := range compose.Services { - serviceNames = append(serviceNames, service.Name) - } - - return serviceNames, nil -} - -// GetAppNames retrieves a list of app names. -func GetAppNames() ([]string, error) { - var appNames []string - - appFiles, err := LoadAppFiles("") - if err != nil { - return appNames, err - } - - apps, err := GetApps(appFiles, "") - if err != nil { - return appNames, err - } - - for _, app := range apps { - appNames = append(appNames, app.Name) - } - - return appNames, nil -} - -// TemplateAppEnvSample copies the example env file for the app into the users -// env files. -func TemplateAppEnvSample(recipeName, appName, server, domain string) error { - envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample") - envSample, err := ioutil.ReadFile(envSamplePath) - if err != nil { - return err - } - - appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName)) - if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) { - return fmt.Errorf("%s already exists?", appEnvPath) - } - - err = ioutil.WriteFile(appEnvPath, envSample, 0o664) - if err != nil { - return err - } - - read, err := ioutil.ReadFile(appEnvPath) - if err != nil { - return err - } - - newContents := strings.Replace(string(read), recipeName+".example.com", domain, -1) - - err = ioutil.WriteFile(appEnvPath, []byte(newContents), 0) - if err != nil { - return err - } - - logrus.Debugf("copied & templated %s to %s", envSamplePath, appEnvPath) - - return nil -} - -// SanitiseAppName makes a app name usable with Docker by replacing illegal -// characters. -func SanitiseAppName(name string) string { - return strings.ReplaceAll(name, ".", "_") -} - -// GetAppStatuses queries servers to check the deployment status of given apps. -func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) { - statuses := make(map[string]map[string]string) - - servers := make(map[string]struct{}) - for _, app := range apps { - if _, ok := servers[app.Server]; !ok { - servers[app.Server] = struct{}{} - } - } - - var bar *progressbar.ProgressBar - if !MachineReadable { - bar = formatter.CreateProgressbar(len(servers), "querying remote servers...") - } - - ch := make(chan stack.StackStatus, len(servers)) - for server := range servers { - cl, err := client.New(server) - if err != nil { - return statuses, err - } - - go func(s string) { - ch <- stack.GetAllDeployedServices(cl, s) - if !MachineReadable { - bar.Add(1) - } - }(server) - } - - for range servers { - status := <-ch - if status.Err != nil { - return statuses, status.Err - } - - for _, service := range status.Services { - result := make(map[string]string) - name := service.Spec.Labels[convert.LabelNamespace] - - if _, ok := statuses[name]; !ok { - result["status"] = "deployed" - } - - labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name) - chaos, ok := service.Spec.Labels[labelKey] - if ok { - result["chaos"] = chaos - } - - labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name) - if chaosVersion, ok := service.Spec.Labels[labelKey]; ok { - result["chaosVersion"] = chaosVersion - } - - labelKey = fmt.Sprintf("coop-cloud.%s.autoupdate", name) - if autoUpdate, ok := service.Spec.Labels[labelKey]; ok { - result["autoUpdate"] = autoUpdate - } else { - result["autoUpdate"] = "false" - } - - labelKey = fmt.Sprintf("coop-cloud.%s.version", name) - if version, ok := service.Spec.Labels[labelKey]; ok { - result["version"] = version - } else { - continue - } - - statuses[name] = result - } - } - - logrus.Debugf("retrieved app statuses: %s", statuses) - - return statuses, nil -} - -// ensurePathExists ensures that a path exists. -func ensurePathExists(path string) error { - if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { - return err - } - return nil -} - -// GetComposeFiles gets the list of compose files for an app (or recipe if you -// don't already have an app) which should be merged into a composetypes.Config -// while respecting the COMPOSE_FILE env var. -func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) { - var composeFiles []string - - composeFileEnvVar, ok := appEnv["COMPOSE_FILE"] - if !ok { - path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe) - if err := ensurePathExists(path); err != nil { - return composeFiles, err - } - logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path) - composeFiles = append(composeFiles, path) - return composeFiles, nil - } - - if !strings.Contains(composeFileEnvVar, ":") { - path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar) - if err := ensurePathExists(path); err != nil { - return composeFiles, err - } - logrus.Debugf("COMPOSE_FILE detected, loading %s", path) - composeFiles = append(composeFiles, path) - return composeFiles, nil - } - - numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1 - envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles) - if len(envVars) != numComposeFiles { - return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar) - } - - for _, file := range envVars { - path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file) - if err := ensurePathExists(path); err != nil { - return composeFiles, err - } - composeFiles = append(composeFiles, path) - } - - logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", ")) - logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe) - - return composeFiles, nil -} - -// GetAppComposeConfig retrieves a compose specification for a recipe. This -// specification is the result of a merge of all the compose.**.yml files in -// the recipe repository. -func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) { - compose, err := loader.LoadComposefile(opts, appEnv) - if err != nil { - return &composetypes.Config{}, err - } - - logrus.Debugf("retrieved %s for %s", compose.Filename, recipe) - - return compose, nil -} - -// ExposeAllEnv exposes all env variables to the app container -func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) { - for _, service := range compose.Services { - if service.Name == "app" { - logrus.Debugf("Add the following environment to the app service config of %s:", stackName) - for k, v := range appEnv { - _, exists := service.Environment[k] - if !exists { - value := v - service.Environment[k] = &value - logrus.Debugf("Add Key: %s Value: %s to %s", k, value, stackName) - } - } - } - } -} - -// SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container -// to signal which recipe is connected to the deployed app -func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) { - for _, service := range compose.Services { - if service.Name == "app" { - logrus.Debugf("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName) - labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName) - service.Deploy.Labels[labelKey] = recipe - } - } -} - -// SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container -// to signal if the app is deployed in chaos mode -func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) { - for _, service := range compose.Services { - if service.Name == "app" { - logrus.Debugf("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName) - labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName) - service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos) - } - } -} - -// SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container -func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) { - for _, service := range compose.Services { - if service.Name == "app" { - logrus.Debugf("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName) - labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName) - service.Deploy.Labels[labelKey] = chaosVersion - } - } -} - -// SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the -// auto update process for this app. The default if this variable is not set is to disable -// the auto update process. -func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) { - for _, service := range compose.Services { - if service.Name == "app" { - enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"] - if !exists { - enable_auto_update = "false" - } - logrus.Debugf("set label 'coop-cloud.%s.autoupdate' to %s for %s", stackName, enable_auto_update, stackName) - labelKey := fmt.Sprintf("coop-cloud.%s.autoupdate", stackName) - service.Deploy.Labels[labelKey] = enable_auto_update - } - } -} - -// GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files -func GetLabel(compose *composetypes.Config, stackName string, label string) string { - for _, service := range compose.Services { - if service.Name == "app" { - labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label) - logrus.Debugf("get label '%s'", labelKey) - if labelValue, ok := service.Deploy.Labels[labelKey]; ok { - return labelValue - } - } - } - logrus.Debugf("no %s label found for %s", label, stackName) - return "" -} - -// GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value -func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) { - timeout := 50 // Default Timeout - var err error = nil - if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" { - logrus.Debugf("timeout label: %s", timeoutLabel) - timeout, err = strconv.Atoi(timeoutLabel) - } - return timeout, err -} diff --git a/pkg/config/env.go b/pkg/config/env.go index d2ba1598..e9c080b8 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -1,18 +1,14 @@ package config import ( - "bufio" "fmt" "io/fs" "io/ioutil" "os" "path" "path/filepath" - "regexp" - "sort" "strings" - "git.coopcloud.tech/coop-cloud/godotenv" "github.com/sirupsen/logrus" ) @@ -21,11 +17,6 @@ const MAX_DOCKER_SECRET_LENGTH = 64 var BackupbotLabel = "coop-cloud.backupbot.enabled" -// envVarModifiers is a list of env var modifier strings. These are added to -// env vars as comments and modify their processing by Abra, e.g. determining -// how long secrets should be. -var envVarModifiers = []string{"length"} - // GetServers retrieves all servers. func GetServers() ([]string, error) { var servers []string @@ -40,34 +31,6 @@ func GetServers() ([]string, error) { return servers, nil } -// ReadEnv loads an app envivornment into a map. -func ReadEnv(filePath string) (AppEnv, error) { - var envVars AppEnv - - envVars, _, err := godotenv.Read(filePath) - if err != nil { - return nil, err - } - - logrus.Debugf("read %s from %s", envVars, filePath) - - return envVars, nil -} - -// ReadEnv loads an app envivornment and their modifiers in two different maps. -func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) { - var envVars AppEnv - - envVars, mods, err := godotenv.Read(filePath) - if err != nil { - return nil, mods, err - } - - logrus.Debugf("read %s from %s", envVars, filePath) - - return envVars, mods, nil -} - // ReadServerNames retrieves all server names. func ReadServerNames() ([]string, error) { serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR) @@ -143,119 +106,3 @@ func GetAllFoldersInDirectory(directory string) ([]string, error) { return folders, nil } - -// ReadAbraShEnvVars reads env vars from an abra.sh recipe file. -func ReadAbraShEnvVars(abraSh string) (map[string]string, error) { - envVars := make(map[string]string) - - file, err := os.Open(abraSh) - if err != nil { - if os.IsNotExist(err) { - return envVars, nil - } - return envVars, err - } - defer file.Close() - - exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`) - if err != nil { - return envVars, err - } - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - txt := scanner.Text() - if exportRegex.MatchString(txt) { - splitVals := strings.Split(txt, "export ") - envVarDef := splitVals[len(splitVals)-1] - keyVal := strings.Split(envVarDef, "=") - if len(keyVal) != 2 { - return envVars, fmt.Errorf("couldn't parse %s", txt) - } - envVars[keyVal[0]] = keyVal[1] - } - } - - if len(envVars) > 0 { - logrus.Debugf("read %s from %s", envVars, abraSh) - } else { - logrus.Debugf("read 0 env var exports from %s", abraSh) - } - - return envVars, nil -} - -type EnvVar struct { - Name string - Present bool -} - -func CheckEnv(app App) ([]EnvVar, error) { - var envVars []EnvVar - - envSamplePath := path.Join(RECIPES_DIR, app.Recipe, ".env.sample") - if _, err := os.Stat(envSamplePath); err != nil { - if os.IsNotExist(err) { - return envVars, fmt.Errorf("%s does not exist?", envSamplePath) - } - return envVars, err - } - - envSample, err := ReadEnv(envSamplePath) - if err != nil { - return envVars, err - } - - var keys []string - for key := range envSample { - keys = append(keys, key) - } - - sort.Strings(keys) - - for _, key := range keys { - if _, ok := app.Env[key]; ok { - envVars = append(envVars, EnvVar{Name: key, Present: true}) - } else { - envVars = append(envVars, EnvVar{Name: key, Present: false}) - } - } - - return envVars, nil -} - -// ReadAbraShCmdNames reads the names of commands. -func ReadAbraShCmdNames(abraSh string) ([]string, error) { - var cmdNames []string - - file, err := os.Open(abraSh) - if err != nil { - if os.IsNotExist(err) { - return cmdNames, nil - } - return cmdNames, err - } - defer file.Close() - - cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`) - if err != nil { - return cmdNames, err - } - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - matches := cmdNameRegex.FindStringSubmatch(line) - if len(matches) > 0 { - cmdNames = append(cmdNames, matches[1]) - } - } - - if len(cmdNames) > 0 { - logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh) - } else { - logrus.Debugf("read 0 command names from %s", abraSh) - } - - return cmdNames, nil -} diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go index 1fc96f82..b861161b 100644 --- a/pkg/lint/recipe.go +++ b/pkg/lint/recipe.go @@ -6,6 +6,7 @@ import ( "os" "path" + "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" recipePkg "coopcloud.tech/abra/pkg/recipe" @@ -234,7 +235,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) { // therefore no matching traefik deploy label will be present. func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) { envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample") - sampleEnv, err := config.ReadEnv(envSamplePath) + sampleEnv, err := app.ReadEnv(envSamplePath) if err != nil { return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name) } diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index c9df9feb..3f4111ad 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/compose" "coopcloud.tech/abra/pkg/config" @@ -227,7 +228,7 @@ func Get(recipeName string, offline bool) (Recipe, error) { } envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample") - sampleEnv, err := config.ReadEnv(envSamplePath) + sampleEnv, err := app.ReadEnv(envSamplePath) if err != nil { return Recipe{}, err } @@ -257,7 +258,7 @@ func Get(recipeName string, offline bool) (Recipe, error) { func (r Recipe) SampleEnv() (map[string]string, error) { envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") - sampleEnv, err := config.ReadEnv(envSamplePath) + sampleEnv, err := app.ReadEnv(envSamplePath) if err != nil { return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name) } diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index f574a033..ca6da99b 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -11,6 +11,7 @@ import ( "strings" "sync" + appPkg "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/upstream/stack" @@ -81,7 +82,7 @@ func GeneratePassphrases(count uint) ([]string, error) { // "app new" case where we pass in the .env.sample and the "secret generate" // case where the app is created. func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) { - appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath) + appEnv, appModifiers, err := appPkg.ReadEnvWithModifiers(appEnvPath) if err != nil { return nil, err } @@ -240,10 +241,10 @@ type secretStatuses []secretStatus // PollSecretsStatus checks status of secrets by comparing the local recipe // config and deploymend server state. -func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, error) { +func PollSecretsStatus(cl *dockerClient.Client, app appPkg.App) (secretStatuses, error) { var secStats secretStatuses - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env) if err != nil { return secStats, err }