new package envfile and move GetComposeFiles to recipe package

cli/app/deploy.go

@@ -6,6 +6,7 @@ import (
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/secret"
 
 	appPkg "coopcloud.tech/abra/pkg/app"
@@ -179,7 +180,7 @@ recipes.
 		}
 
 		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-		abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath)
+		abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -187,7 +188,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/new.go

@@ -136,7 +136,7 @@ var appNewCommand = cli.Command{
 				logrus.Fatal(err)
 			}
 
-			composeFiles, err := appPkg.GetComposeFiles(recipe.Name, sampleEnv)
+			composeFiles, err := recipePkg.GetComposeFiles(recipe.Name, sampleEnv)
 			if err != nil {
 				logrus.Fatal(err)
 			}

cli/app/ps.go

@@ -79,7 +79,7 @@ var appPsCommand = cli.Command{
 
 // showPSOutput renders ps output.
 func showPSOutput(c *cli.Context, app appPkg.App, cl *dockerClient.Client) {
-	composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+	composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 	if err != nil {
 		logrus.Fatal(err)
 		return

cli/app/rollback.go

@@ -7,6 +7,7 @@ import (
 	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
@@ -199,7 +200,7 @@ recipes.
 		}
 
 		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-		abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath)
+		abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -207,7 +208,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/secret.go

@@ -87,7 +87,7 @@ var appSecretGenerateCommand = cli.Command{
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
-		composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -284,7 +284,7 @@ Example:
 			}
 		}
 
-		composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/upgrade.go

@@ -9,6 +9,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
@@ -233,7 +234,7 @@ recipes.
 		}
 
 		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-		abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath)
+		abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -241,7 +242,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := recipePkg.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/updater/updater.go

@@ -11,6 +11,7 @@ import (
 	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/convert"
@@ -193,7 +194,7 @@ func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool
 }
 
 // getEnv reads env variables from docker services.
-func getEnv(cl *dockerclient.Client, stackName string) (appPkg.AppEnv, error) {
+func getEnv(cl *dockerclient.Client, stackName string) (envfile.AppEnv, error) {
 	envMap := make(map[string]string)
 	filter := filters.NewArgs()
 	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
@@ -340,9 +341,9 @@ func processRecipeRepoVersion(recipeName, version string) error {
 }
 
 // mergeAbraShEnv merges abra.sh env vars into the app env vars.
-func mergeAbraShEnv(recipeName string, env appPkg.AppEnv) error {
+func mergeAbraShEnv(recipeName string, env envfile.AppEnv) error {
 	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
-	abraShEnv, err := appPkg.ReadAbraShEnvVars(abraShPath)
+	abraShEnv, err := envfile.ReadAbraShEnvVars(abraShPath)
 	if err != nil {
 		return err
 	}
@@ -356,7 +357,7 @@ func mergeAbraShEnv(recipeName string, env appPkg.AppEnv) error {
 }
 
 // createDeployConfig merges and enriches the compose config for the deployment.
-func createDeployConfig(recipeName string, stackName string, env appPkg.AppEnv) (*composetypes.Config, stack.Deploy, error) {
+func createDeployConfig(recipeName string, stackName string, env envfile.AppEnv) (*composetypes.Config, stack.Deploy, error) {
 	env["STACK_NAME"] = stackName
 
 	deployOpts := stack.Deploy{
@@ -366,7 +367,7 @@ func createDeployConfig(recipeName string, stackName string, env appPkg.AppEnv)
 		Detach:       false,
 	}
 
-	composeFiles, err := appPkg.GetComposeFiles(recipeName, env)
+	composeFiles, err := recipe.GetComposeFiles(recipeName, env)
 	if err != nil {
 		return nil, deployOpts, err
 	}

pkg/app/app.go

@@ -1,14 +1,19 @@
 package app
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"path"
+	"regexp"
+	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 
@@ -81,19 +86,13 @@ type App struct {
 	Name   AppName
 	Recipe string
 	Domain string
-	Env    AppEnv
+	Env    envfile.AppEnv
 	Server string
 	Path   string
 }
 
 // Type aliases to make code hints easier to understand
 
-// AppEnv is a map of the values in an apps env config
-type AppEnv = map[string]string
-
-// AppModifiers is a map of modifiers in an apps env config
-type AppModifiers = map[string]map[string]string
-
 // AppName is AppName
 type AppName = string
 
@@ -162,7 +161,7 @@ func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (f
 		return filters, nil
 	}
 
-	composeFiles, err := GetComposeFiles(a.Recipe, a.Env)
+	composeFiles, err := recipe.GetComposeFiles(a.Recipe, a.Env)
 	if err != nil {
 		return filters, err
 	}
@@ -231,7 +230,7 @@ func (a ByName) Less(i, j int) bool {
 }
 
 func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path)
+	env, err := envfile.ReadEnv(appFile.Path)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
@@ -247,7 +246,7 @@ func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
 }
 
 // NewApp creates new App object
-func NewApp(env AppEnv, name string, appFile AppFile) (App, error) {
+func NewApp(env envfile.AppEnv, name string, appFile AppFile) (App, error) {
 	domain := env["DOMAIN"]
 
 	recipe, exists := env["RECIPE"]
@@ -318,7 +317,7 @@ func GetAppServiceNames(appName string) ([]string, error) {
 		return serviceNames, err
 	}
 
-	composeFiles, err := GetComposeFiles(app.Recipe, app.Env)
+	composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 	if err != nil {
 		return serviceNames, err
 	}
@@ -481,7 +480,7 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
 // GetAppComposeConfig retrieves a compose specification for a recipe. This
 // specification is the result of a merge of all the compose.**.yml files in
 // the recipe repository.
-func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) {
+func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv envfile.AppEnv) (*composetypes.Config, error) {
 	compose, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
 		return &composetypes.Config{}, err
@@ -493,7 +492,7 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp
 }
 
 // ExposeAllEnv exposes all env variables to the app container
-func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) {
+func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv envfile.AppEnv) {
 	for _, service := range compose.Services {
 		if service.Name == "app" {
 			logrus.Debugf("Add the following environment to the app service config of %s:", stackName)
@@ -508,3 +507,73 @@ func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv)
 		}
 	}
 }
+
+func CheckEnv(app App) ([]envfile.EnvVar, error) {
+	var envVars []envfile.EnvVar
+
+	envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
+	if _, err := os.Stat(envSamplePath); err != nil {
+		if os.IsNotExist(err) {
+			return envVars, fmt.Errorf("%s does not exist?", envSamplePath)
+		}
+		return envVars, err
+	}
+
+	envSample, err := envfile.ReadEnv(envSamplePath)
+	if err != nil {
+		return envVars, err
+	}
+
+	var keys []string
+	for key := range envSample {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		if _, ok := app.Env[key]; ok {
+			envVars = append(envVars, envfile.EnvVar{Name: key, Present: true})
+		} else {
+			envVars = append(envVars, envfile.EnvVar{Name: key, Present: false})
+		}
+	}
+
+	return envVars, nil
+}
+
+// ReadAbraShCmdNames reads the names of commands.
+func ReadAbraShCmdNames(abraSh string) ([]string, error) {
+	var cmdNames []string
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cmdNames, nil
+		}
+		return cmdNames, err
+	}
+	defer file.Close()
+
+	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
+	if err != nil {
+		return cmdNames, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := cmdNameRegex.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			cmdNames = append(cmdNames, matches[1])
+		}
+	}
+
+	if len(cmdNames) > 0 {
+		logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh)
+	} else {
+		logrus.Debugf("read 0 command names from %s", abraSh)
+	}
+
+	return cmdNames, nil
+}

pkg/app/compose.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strconv"
 
+	"coopcloud.tech/abra/pkg/envfile"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/sirupsen/logrus"
 )
@@ -46,7 +47,7 @@ func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosV
 // SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the
 // auto update process for this app. The default if this variable is not set is to disable
 // the auto update process.
-func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) {
+func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv envfile.AppEnv) {
 	for _, service := range compose.Services {
 		if service.Name == "app" {
 			enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"]

pkg/compose/compose.go

@@ -7,8 +7,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
@@ -30,7 +30,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := app.ReadEnv(envSamplePath)
+		sampleEnv, err := envfile.ReadEnv(envSamplePath)
 		if err != nil {
 			return false, err
 		}
@@ -98,7 +98,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := app.ReadEnv(envSamplePath)
+		sampleEnv, err := envfile.ReadEnv(envSamplePath)
 		if err != nil {
 			return err
 		}

pkg/envfile/envfile.go

@@ -1,19 +1,27 @@
-package app
+package envfile
 
 import (
 	"bufio"
 	"fmt"
 	"os"
-	"path"
 	"regexp"
-	"sort"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/config"
 	"git.coopcloud.tech/coop-cloud/godotenv"
 	"github.com/sirupsen/logrus"
 )
 
+// envVarModifiers is a list of env var modifier strings. These are added to
+// env vars as comments and modify their processing by Abra, e.g. determining
+// how long secrets should be.
+var envVarModifiers = []string{"length"}
+
+// AppEnv is a map of the values in an apps env config
+type AppEnv = map[string]string
+
+// AppModifiers is a map of modifiers in an apps env config
+type AppModifiers = map[string]map[string]string
+
 // ReadEnv loads an app envivornment into a map.
 func ReadEnv(filePath string) (AppEnv, error) {
 	var envVars AppEnv
@@ -87,73 +95,3 @@ type EnvVar struct {
 	Name    string
 	Present bool
 }
-
-func CheckEnv(app App) ([]EnvVar, error) {
-	var envVars []EnvVar
-
-	envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
-	if _, err := os.Stat(envSamplePath); err != nil {
-		if os.IsNotExist(err) {
-			return envVars, fmt.Errorf("%s does not exist?", envSamplePath)
-		}
-		return envVars, err
-	}
-
-	envSample, err := ReadEnv(envSamplePath)
-	if err != nil {
-		return envVars, err
-	}
-
-	var keys []string
-	for key := range envSample {
-		keys = append(keys, key)
-	}
-
-	sort.Strings(keys)
-
-	for _, key := range keys {
-		if _, ok := app.Env[key]; ok {
-			envVars = append(envVars, EnvVar{Name: key, Present: true})
-		} else {
-			envVars = append(envVars, EnvVar{Name: key, Present: false})
-		}
-	}
-
-	return envVars, nil
-}
-
-// ReadAbraShCmdNames reads the names of commands.
-func ReadAbraShCmdNames(abraSh string) ([]string, error) {
-	var cmdNames []string
-
-	file, err := os.Open(abraSh)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return cmdNames, nil
-		}
-		return cmdNames, err
-	}
-	defer file.Close()
-
-	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
-	if err != nil {
-		return cmdNames, err
-	}
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-		matches := cmdNameRegex.FindStringSubmatch(line)
-		if len(matches) > 0 {
-			cmdNames = append(cmdNames, matches[1])
-		}
-	}
-
-	if len(cmdNames) > 0 {
-		logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh)
-	} else {
-		logrus.Debugf("read 0 command names from %s", abraSh)
-	}
-
-	return cmdNames, nil
-}

pkg/envfile/envfile_test.go

@@ -1,4 +1,4 @@
-package app_test
+package envfile_test
 
 import (
 	"fmt"

pkg/lint/recipe.go

@@ -6,8 +6,8 @@ import (
 	"os"
 	"path"
 
-	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
@@ -235,7 +235,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := app.ReadEnv(envSamplePath)
+	sampleEnv, err := envfile.ReadEnv(envSamplePath)
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}

pkg/recipe/recipe.go

@@ -12,10 +12,10 @@ import (
 	"strconv"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/compose"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/limit"
@@ -228,7 +228,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := app.ReadEnv(envSamplePath)
+	sampleEnv, err := envfile.ReadEnv(envSamplePath)
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -258,7 +258,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 
 func (r Recipe) SampleEnv() (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := app.ReadEnv(envSamplePath)
+	sampleEnv, err := envfile.ReadEnv(envSamplePath)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}

pkg/secret/secret.go

@@ -14,6 +14,8 @@ import (
 	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/decentral1se/passgen"
@@ -82,7 +84,7 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
 func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) {
-	appEnv, appModifiers, err := appPkg.ReadEnvWithModifiers(appEnvPath)
+	appEnv, appModifiers, err := envfile.ReadEnvWithModifiers(appEnvPath)
 	if err != nil {
 		return nil, err
 	}
@@ -244,7 +246,7 @@ type secretStatuses []secretStatus
 func PollSecretsStatus(cl *dockerClient.Client, app appPkg.App) (secretStatuses, error) {
 	var secStats secretStatuses
 
-	composeFiles, err := appPkg.GetComposeFiles(app.Recipe, app.Env)
+	composeFiles, err := recipe.GetComposeFiles(app.Recipe, app.Env)
 	if err != nil {
 		return secStats, err
 	}