Fix errors in build-docker-inside.sh

See coop-cloud/organising#484

.dockerignore

@@ -1,4 +1,8 @@
-Dockerfile
-.dockerignore
-*.swp
 *.swo
+*.swp
+.dockerignore
+Dockerfile
+abra
+dist
+kadabra
+tags

.drone.yml

@@ -7,16 +7,13 @@ steps:
     commands:
       - make check
 
-  - name: make build
-    image: golang:1.21
-    commands:
-      - make build
-    depends_on:
-      - make check
-
   - name: make test
     image: golang:1.21
+    environment:
+      ABRA_DIR: "/root/.abra"
     commands:
+      - make build-abra
+      - ./abra help # show version, initialise $ABRA_DIR
       - make test
     depends_on:
       - make check
@@ -27,7 +24,6 @@ steps:
       - git fetch --tags
     depends_on:
       - make check
-      - make build
       - make test
     when:
       event: tag
@@ -47,22 +43,22 @@ steps:
     when:
       event: tag
 
-  # - name: publish image
-  #   image: plugins/docker
-  #   settings:
-  #     auto_tag: true
-  #     username: 3wordchant
-  #     password:
-  #       from_secret: git_coopcloud_tech_token_3wc
-  #     repo: git.coopcloud.tech/coop-cloud/abra
-  #     tags: dev
-  #     registry: git.coopcloud.tech
-  #   when:
-  #     event:
-  #       exclude:
-  #         - pull_request
-  #   depends_on:
-  #     - make check
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: 3wordchant
+      password:
+        from_secret: git_coopcloud_tech_token_3wc
+      repo: git.coopcloud.tech/coop-cloud/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      event:
+        exclude:
+          - pull_request
+    depends_on:
+      - make check
 
 volumes:
   - name: deps

Dockerfile

@@ -2,7 +2,14 @@ FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
-RUN apk add --no-cache make git gcc musl-dev
+RUN apk add --no-cache \
+  ca-certificates \
+  gcc \
+  git \
+  make \
+  musl-dev
+
+RUN update-ca-certificates
 
 COPY . /app
 

Makefile

@@ -7,7 +7,8 @@ DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 
 export GOPRIVATE=coopcloud.tech
 
-all: format check build test
+# NOTE(d1): default `make` optimised for Abra hacking
+all: format check build-abra test
 
 run-abra:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
@@ -18,19 +19,22 @@ run-kadabra:
 install-abra:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
-install-kadaabra:
+install-kadabra:
 	@go install -ldflags=$(LDFLAGS) $(KADABRA)
 
 build-abra:
-	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
 
 build-kadabra:
-	@go build -v -ldflags=$(LDFLAGS) $(KADABRA)
-
-build:
-	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
 	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
+build: build-abra build-kadabra
+
+build-docker-abra:
+	docker run -it -v $(PWD):/abra golang:1.21 bash -c 'cd /abra; ./build-docker-inside.sh'
+
+build-docker: build-docker-abra
+
 clean:
 	@rm '$(GOPATH)/bin/abra'
 	@rm '$(GOPATH)/bin/kadabra'

build-docker-inside.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+if [ ! -f .envrc ]; then
+    . .envrc.sample
+else
+    . .envrc
+fi
+git config --global --add safe.directory /abra  # work around funky file permissions
+
+# fixme for some reason we need to do this
+go get coopcloud.tech/abra/pkg/upstream/commandconn
+go get github.com/sirupsen/logrus@v1.9.3
+go get github.com/cloudflare/circl/dh/x25519@v1.3.3
+go get github.com/mattn/go-runewidth@v0.0.14
+go get github.com/mattn/go-colorable@v0.1.12
+go get coopcloud.tech/abra/pkg/config
+#
+
+make build

cli/app/check.go

@@ -1,13 +1,10 @@
 package app
 
 import (
-	"os"
-	"path"
-	"strings"
-
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
@@ -15,9 +12,21 @@ import (
 )
 
 var appCheckCommand = cli.Command{
-	Name:      "check",
-	Aliases:   []string{"chk"},
-	Usage:     "Check if an app is configured correctly",
+	Name:    "check",
+	Aliases: []string{"chk"},
+	Usage:   "Ensure an app is well configured",
+	Description: `
+This command compares env vars in both the app ".env" and recipe ".env.sample"
+file.
+
+The goal is to ensure that recipe ".env.sample" env vars are defined in your
+app ".env" file. Only env var definitions in the ".env.sample" which are
+uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include
+these env vars, then "check" will complain.
+
+Recipe maintainers may or may not provide defaults for env vars within their
+recipes regardless of commenting or not (e.g. through the use of
+${FOO:<default>} syntax). "check" does not confirm or deny this for you.`,
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
@@ -49,32 +58,23 @@ var appCheckCommand = cli.Command{
 			}
 		}
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
-		if _, err := os.Stat(envSamplePath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
-			}
-			logrus.Fatal(err)
-		}
+		tableCol := []string{"recipe env sample", "app env"}
+		table := formatter.CreateTable(tableCol)
 
-		envSample, err := config.ReadEnv(envSamplePath)
+		envVars, err := config.CheckEnv(app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		var missing []string
-		for k := range envSample {
-			if _, ok := app.Env[k]; !ok {
-				missing = append(missing, k)
+		for _, envVar := range envVars {
+			if envVar.Present {
+				table.Append([]string{envVar.Name, "✅"})
+			} else {
+				table.Append([]string{envVar.Name, "❌"})
 			}
 		}
 
-		if len(missing) > 0 {
-			missingEnvVars := strings.Join(missing, ", ")
-			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
-		}
-
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+		table.Render()
 
 		return nil
 	},

cli/app/cmd.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -22,8 +23,7 @@ var appCmdCommand = cli.Command{
 	Name:    "command",
 	Aliases: []string{"cmd"},
 	Usage:   "Run app commands",
-	Description: `
-Run an app specific command.
+	Description: `Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
@@ -43,8 +43,8 @@ Example:
 		internal.OfflineFlag,
 		internal.ChaosFlag,
 	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
+	Before:      internal.SubCommandBefore,
+	Subcommands: []cli.Command{appCmdListCommand},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -186,3 +186,53 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 
 	return hasCmdArgs, parsedCmdArgs
 }
+
+var appCmdListCommand = cli.Command{
+	Name:      "list",
+	Aliases:   []string{"ls"},
+	Usage:     "List all available commands",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		sort.Strings(cmdNames)
+		for _, cmdName := range cmdNames {
+			fmt.Println(cmdName)
+		}
+
+		return nil
+	},
+}

cli/app/deploy.go

@@ -6,6 +6,7 @@ import (
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/secret"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -50,6 +51,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
@@ -91,6 +97,17 @@ recipes.
 			logrus.Fatal(err)
 		}
 
+		secStats, err := secret.PollSecretsStatus(cl, app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, secStat := range secStats {
+			if !secStat.CreatedOnRemote {
+				logrus.Fatalf("unable to deploy, secrets not generated (%s)?", secStat.LocalName)
+			}
+		}
+
 		if isDeployed {
 			if internal.Force || internal.Chaos {
 				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
@@ -100,7 +117,6 @@ recipes.
 		}
 
 		version := deployedVersion
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			version = specificVersion
 			logrus.Debugf("choosing %s as version to deploy", version)
@@ -188,6 +204,17 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, version)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
 			logrus.Fatal(err)
 		}

cli/app/logs.go

@@ -11,6 +11,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
@@ -86,6 +87,10 @@ var appLogsCommand = cli.Command{
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)

cli/app/new.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"fmt"
+	"path"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -96,7 +97,7 @@ var appNewCommand = cli.Command{
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
-			sampleEnv, err := recipe.SampleEnv()
+			sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -106,7 +107,8 @@ var appNewCommand = cli.Command{
 				logrus.Fatal(err)
 			}
 
-			secretsConfig, err := secret.ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 			if err != nil {
 				return err
 			}

cli/app/rollback.go

@@ -51,6 +51,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
@@ -125,7 +130,6 @@ recipes.
 			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {

cli/app/secret.go

@@ -87,7 +87,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -276,7 +276,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -356,6 +356,7 @@ var appSecretLsCommand = cli.Command{
 		internal.DebugFlag,
 		internal.OfflineFlag,
 		internal.ChaosFlag,
+		internal.MachineReadableFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "List all secrets",
@@ -383,12 +384,7 @@ var appSecretLsCommand = cli.Command{
 			}
 		}
 
-		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -396,42 +392,27 @@ var appSecretLsCommand = cli.Command{
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
 		table := formatter.CreateTable(tableCol)
 
-		cl, err := client.New(app.Server)
+		secStats, err := secret.PollSecretsStatus(cl, app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, false)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteSecretNames := make(map[string]bool)
-		for _, cont := range secretList {
-			remoteSecretNames[cont.Spec.Annotations.Name] = true
-		}
-
-		for secretName, secretValue := range secretsConfig {
-			createdRemote := false
-			val, err := secret.ParseSecretValue(secretValue)
-			if err != nil {
-				logrus.Fatal(err)
+		for _, secStat := range secStats {
+			tableRow := []string{
+				secStat.LocalName,
+				secStat.Version,
+				secStat.RemoteName,
+				strconv.FormatBool(secStat.CreatedOnRemote),
 			}
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
-			if _, ok := remoteSecretNames[secretRemoteName]; ok {
-				createdRemote = true
-			}
-			tableRow := []string{secretName, val.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
 			table.Append(tableRow)
 		}
 
 		if table.NumLines() > 0 {
-			table.Render()
+			if internal.MachineReadable {
+				table.JSONRender()
+			} else {
+				table.Render()
+			}
 		} else {
 			logrus.Warnf("no secrets stored for %s", app.Name)
 		}

cli/app/upgrade.go

@@ -56,6 +56,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if !internal.Chaos {
 			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
@@ -126,7 +131,6 @@ recipes.
 			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {
@@ -254,6 +258,17 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}

cli/catalogue/catalogue.go

@@ -98,11 +98,6 @@ keys configured on your account.
 				continue
 			}
 
-			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
 			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
@@ -173,7 +168,7 @@ keys configured on your account.
 			}
 
 			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
+			if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil {
 				logrus.Fatal(err)
 			}
 

cli/cli.go

@@ -14,8 +14,8 @@ import (
 	"coopcloud.tech/abra/cli/recipe"
 	"coopcloud.tech/abra/cli/server"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	cataloguePkg "coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -184,12 +184,8 @@ func newAbraApp(version, commit string) *cli.App {
 			}
 		}
 
-		if _, err := os.Stat(config.CATALOGUE_DIR); os.IsNotExist(err) {
-			url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
-			logrus.Warnf("local recipe catalogue is missing, retrieving now")
-			if err := git.Clone(config.CATALOGUE_DIR, url); err != nil {
-				logrus.Fatal(err)
-			}
+		if err := cataloguePkg.EnsureCatalogue(); err != nil {
+			logrus.Fatal(err)
 		}
 
 		logrus.Debugf("abra version %s, commit %s", version, commit)

cli/internal/cli.go

@@ -54,7 +54,7 @@ var Chaos bool
 // ChaosFlag turns on/off chaos functionality.
 var ChaosFlag = &cli.BoolFlag{
 	Name:        "chaos, C",
-	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Usage:       "Proceed with uncommitted recipes changes. Use with care!",
 	Destination: &Chaos,
 }
 

cli/recipe/diff.go

@@ -0,0 +1,40 @@
+package recipe
+
+import (
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeDiffCommand = cli.Command{
+	Name:        "diff",
+	Usage:       "Show unstaged changes in recipe config",
+	Description: "Due to limitations in our underlying Git dependency, this command requires /usr/bin/git.",
+	Aliases:     []string{"d"},
+	ArgsUsage:   "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+		if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/lint.go

@@ -68,7 +68,7 @@ var recipeLintCommand = cli.Command{
 
 				skippedOutput := "-"
 				if skipped {
-					skippedOutput = "yes"
+					skippedOutput = "✅"
 				}
 
 				satisfied := false
@@ -87,9 +87,9 @@ var recipeLintCommand = cli.Command{
 					}
 				}
 
-				satisfiedOutput := "yes"
+				satisfiedOutput := "✅"
 				if !satisfied {
-					satisfiedOutput = "NO"
+					satisfiedOutput = "❌"
 					if skipped {
 						satisfiedOutput = "-"
 					}

cli/recipe/recipe.go

@@ -30,5 +30,7 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 		recipeSyncCommand,
 		recipeUpgradeCommand,
 		recipeVersionCommand,
+		recipeResetCommand,
+		recipeDiffCommand,
 	},
 }

cli/recipe/release.go

@@ -106,6 +106,18 @@ your SSH keys configured on your account.
 			}
 		}
 
+		isClean, err := gitPkg.IsClean(recipe.Dir())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if len(tags) > 0 {
 			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
 			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
@@ -244,7 +256,7 @@ func commitRelease(recipe recipe.Recipe, tag string) error {
 
 	msg := fmt.Sprintf("chore: publish %s release", tag)
 	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
+	if err := gitPkg.Commit(repoPath, msg, internal.Dry); err != nil {
 		return err
 	}
 

cli/recipe/reset.go

@@ -0,0 +1,56 @@
+package recipe
+
+import (
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeResetCommand = cli.Command{
+	Name:        "reset",
+	Usage:       "Remove all unstaged changes from recipe config",
+	Description: "WARNING, this will delete your changes. Be Careful.",
+	Aliases:     []string{"rs"},
+	ArgsUsage:   "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		repoPath := path.Join(config.RECIPES_DIR, recipeName)
+		repo, err := git.PlainOpen(repoPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		ref, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		worktree, err := repo.Worktree()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}
+		if err := worktree.Reset(opts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/sync.go

@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -198,6 +199,17 @@ likely to change.
 			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
+		isClean, err := gitPkg.IsClean(recipe.Dir())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		return nil
 	},
 }

cli/recipe/upgrade.go

@@ -14,6 +14,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -326,6 +327,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 
 				fmt.Println(string(jsonstring))
+
 				return nil
 			}
 
@@ -336,6 +338,18 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 		}
+
+		isClean, err := gitPkg.IsClean(recipeDir)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		return nil
 	},
 }

go.mod

@@ -1,22 +1,22 @@
 module coopcloud.tech/abra
 
-go 1.18
+go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v24.0.6+incompatible
-	github.com/docker/distribution v2.8.2+incompatible
-	github.com/docker/docker v24.0.6+incompatible
+	github.com/docker/cli v24.0.7+incompatible
+	github.com/docker/distribution v2.8.3+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.9.0
+	github.com/go-git/go-git/v5 v5.10.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.13.1
+	github.com/schollz/progressbar/v3 v3.14.0
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )
@@ -35,6 +35,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
@@ -77,11 +78,11 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.13.0 // indirect
@@ -115,5 +116,5 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/sys v0.12.0
+	golang.org/x/sys v0.14.0
 )

go.sum

@@ -124,6 +124,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -132,6 +133,7 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -313,6 +315,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
@@ -332,18 +335,20 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
+github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
-github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
+github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
-github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
+github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
-github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -369,6 +374,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -406,13 +412,15 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
-github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
-github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
+github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ=
+github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -663,6 +671,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -782,6 +791,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -881,6 +891,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -1058,8 +1069,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1157,8 +1168,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1304,16 +1315,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1573,6 +1584,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

pkg/catalogue/catalogue.go

@@ -12,50 +12,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                         true,
-	"abra-apps":                    true,
-	"abra-aur":                     true,
-	"abra-bash":                    true,
-	"abra-capsul":                  true,
-	"abra-gandi":                   true,
-	"abra-hetzner":                 true,
-	"abra-integration-test-recipe": true,
-	"apps":                         true,
-	"aur-abra-git":                 true,
-	"auto-mirror":                  true,
-	"auto-recipes-catalogue-json":  true,
-	"backup-bot":                   true,
-	"backup-bot-two":               true,
-	"beta.coopcloud.tech":          true,
-	"comrade-renovate-bot":         true,
-	"coopcloud.tech":               true,
-	"coturn":                       true,
-	"docker-cp-deploy":             true,
-	"docker-dind-bats-kcov":        true,
-	"docs.coopcloud.tech":          true,
-	"drone-abra":                   true,
-	"example":                      true,
-	"gardening":                    true,
-	"go-abra":                      true,
-	"organising":                   true,
-	"pyabra":                       true,
-	"radicle-seed-node":            true,
-	"recipes-catalogue-json":       true,
-	"recipes-wishlist":             true,
-	"recipes.coopcloud.tech":       true,
-	"stack-ssh-deploy":             true,
-	"swarm-cronjob":                true,
-	"tagcmp":                       true,
-	"traefik-cert-dumper":          true,
-	"tyop":                         true,
-}
-
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		logrus.Warnf("local recipe catalogue is missing, retrieving now")
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
 		if err := gitPkg.Clone(catalogueDir, url); err != nil {
 			return err

pkg/compose/compose.go

@@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return err
 		}

pkg/config/app.go

@@ -149,15 +149,15 @@ func (a ByName) Less(i, j int) bool {
 	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
 }
 
-func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path)
+func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
+	env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
 
 	logrus.Debugf("read env %s from %s", env, appFile.Path)
 
-	app, err := newApp(env, name, appFile)
+	app, err := NewApp(env, name, appFile)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
 	}
@@ -165,8 +165,8 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	return app, nil
 }
 
-// newApp creates new App object
-func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+// NewApp creates new App object
+func NewApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	domain := env["DOMAIN"]
 
 	recipe, exists := env["RECIPE"]
@@ -232,7 +232,7 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
 		return App{}, fmt.Errorf("cannot find app with name %s", name)
 	}
 
-	app, err := readAppEnvFile(appFile, name)
+	app, err := ReadAppEnvFile(appFile, name)
 	if err != nil {
 		return App{}, err
 	}
@@ -437,27 +437,56 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
 	return statuses, nil
 }
 
+// ensurePathExists ensures that a path exists.
+func ensurePathExists(path string) error {
+	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
 // GetComposeFiles gets the list of compose files for an app (or recipe if you
 // don't already have an app) which should be merged into a composetypes.Config
 // while respecting the COMPOSE_FILE env var.
 func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
 	var composeFiles []string
 
-	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
-		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
+	composeFileEnvVar, ok := appEnv["COMPOSE_FILE"]
+	if !ok {
 		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path)
 		composeFiles = append(composeFiles, path)
 		return composeFiles, nil
 	}
 
-	composeFileEnvVar := appEnv["COMPOSE_FILE"]
-	envVars := strings.Split(composeFileEnvVar, ":")
-	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
-	for _, file := range strings.Split(composeFileEnvVar, ":") {
+	if !strings.Contains(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("COMPOSE_FILE detected, loading %s", path)
+		composeFiles = append(composeFiles, path)
+		return composeFiles, nil
+	}
+
+	numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1
+	envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles)
+	if len(envVars) != numComposeFiles {
+		return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar)
+	}
+
+	for _, file := range envVars {
 		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
 		composeFiles = append(composeFiles, path)
 	}
 
+	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
 	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
 
 	return composeFiles, nil

pkg/config/app_test.go

@@ -1,36 +1,108 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestNewApp(t *testing.T) {
-	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestReadAppEnvFile(t *testing.T) {
-	app, err := readAppEnvFile(expectedAppFile, appName)
+	app, err := config.ReadAppEnvFile(ExpectedAppFile, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestGetApp(t *testing.T) {
-	app, err := GetApp(expectedAppFiles, appName)
+	app, err := config.GetApp(ExpectedAppFiles, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
+	}
+}
+
+func TestGetComposeFiles(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		appEnv       map[string]string
+		composeFiles []string
+	}{
+		{
+			map[string]string{},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml:compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		composeFiles, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, composeFiles, test.composeFiles)
+	}
+}
+
+func TestGetComposeFilesError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct{ appEnv map[string]string }{
+		{map[string]string{"COMPOSE_FILE": "compose.yml::compose.foo.yml"}},
+		{map[string]string{"COMPOSE_FILE": "doesnt.exist.yml"}},
+	}
+
+	for _, test := range tests {
+		_, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err == nil {
+			t.Fatalf("should have failed: %v", test.appEnv)
+		}
 	}
 }

pkg/config/env.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/Autonomic-Cooperative/godotenv"
@@ -34,6 +36,11 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+// envVarModifiers is a list of env var modifier strings. These are added to
+// env vars as comments and modify their processing by Abra, e.g. determining
+// how long secrets should be.
+var envVarModifiers = []string{"length"}
+
 // GetServers retrieves all servers.
 func GetServers() ([]string, error) {
 	var servers []string
@@ -48,18 +55,43 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
-// ReadEnv loads an app envivornment into a map.
-func ReadEnv(filePath string) (AppEnv, error) {
-	var envFile AppEnv
+// ReadEnvOptions modifies the ReadEnv processing of env vars.
+type ReadEnvOptions struct {
+	IncludeModifiers bool
+}
 
-	envFile, err := godotenv.Read(filePath)
+// ContainsEnvVarModifier determines if an env var contains a modifier.
+func ContainsEnvVarModifier(envVar string) bool {
+	for _, mod := range envVarModifiers {
+		if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
+			return true
+		}
+	}
+	return false
+}
+
+// ReadEnv loads an app envivornment into a map.
+func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
+	var envVars AppEnv
+
+	envVars, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
-	logrus.Debugf("read %s from %s", envFile, filePath)
+	//	for idx, envVar := range envVars {
+	//		if strings.Contains(envVar, "#") {
+	//			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
+	//				continue
+	//			}
+	//			vals := strings.Split(envVar, "#")
+	//			envVars[idx] = strings.TrimSpace(vals[0])
+	//		}
+	//	}
 
-	return envFile, nil
+	logrus.Debugf("read %s from %s", envVars, filePath)
+
+	return envVars, nil
 }
 
 // ReadServerNames retrieves all server names.
@@ -149,22 +181,107 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 		}
 		return envVars, err
 	}
+	defer file.Close()
+
+	exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`)
+	if err != nil {
+		return envVars, err
+	}
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, "export") {
-			splitVals := strings.Split(line, "export ")
+		txt := scanner.Text()
+		if exportRegex.MatchString(txt) {
+			splitVals := strings.Split(txt, "export ")
 			envVarDef := splitVals[len(splitVals)-1]
 			keyVal := strings.Split(envVarDef, "=")
 			if len(keyVal) != 2 {
-				return envVars, fmt.Errorf("couldn't parse %s", line)
+				return envVars, fmt.Errorf("couldn't parse %s", txt)
 			}
 			envVars[keyVal[0]] = keyVal[1]
 		}
 	}
 
-	logrus.Debugf("read %s from %s", envVars, abraSh)
+	if len(envVars) > 0 {
+		logrus.Debugf("read %s from %s", envVars, abraSh)
+	} else {
+		logrus.Debugf("read 0 env var exports from %s", abraSh)
+	}
 
 	return envVars, nil
 }
+
+type EnvVar struct {
+	Name    string
+	Present bool
+}
+
+func CheckEnv(app App) ([]EnvVar, error) {
+	var envVars []EnvVar
+
+	envSamplePath := path.Join(RECIPES_DIR, app.Recipe, ".env.sample")
+	if _, err := os.Stat(envSamplePath); err != nil {
+		if os.IsNotExist(err) {
+			return envVars, fmt.Errorf("%s does not exist?", envSamplePath)
+		}
+		return envVars, err
+	}
+
+	envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
+	if err != nil {
+		return envVars, err
+	}
+
+	var keys []string
+	for key := range envSample {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		if _, ok := app.Env[key]; ok {
+			envVars = append(envVars, EnvVar{Name: key, Present: true})
+		} else {
+			envVars = append(envVars, EnvVar{Name: key, Present: false})
+		}
+	}
+
+	return envVars, nil
+}
+
+// ReadAbraShCmdNames reads the names of commands.
+func ReadAbraShCmdNames(abraSh string) ([]string, error) {
+	var cmdNames []string
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cmdNames, nil
+		}
+		return cmdNames, err
+	}
+	defer file.Close()
+
+	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
+	if err != nil {
+		return cmdNames, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := cmdNameRegex.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			cmdNames = append(cmdNames, matches[1])
+		}
+	}
+
+	if len(cmdNames) > 0 {
+		logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh)
+	} else {
+		logrus.Debugf("read 0 command names from %s", abraSh)
+	}
+
+	return cmdNames, nil
+}

pkg/config/env_test.go

@@ -1,60 +1,63 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
 
 // make sure these are in alphabetical order
-var tFolders = []string{"folder1", "folder2"}
-var tFiles = []string{"bar.env", "foo.env"}
+var TFolders = []string{"folder1", "folder2"}
+var TFiles = []string{"bar.env", "foo.env"}
 
-var appName = "ecloud"
-var serverName = "evil.corp"
+var AppName = "ecloud"
+var ServerName = "evil.corp"
 
-var expectedAppEnv = AppEnv{
+var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
 	"RECIPE": "ecloud",
 }
 
-var expectedApp = App{
-	Name:   appName,
-	Recipe: expectedAppEnv["RECIPE"],
-	Domain: expectedAppEnv["DOMAIN"],
-	Env:    expectedAppEnv,
-	Path:   expectedAppFile.Path,
-	Server: expectedAppFile.Server,
+var ExpectedApp = config.App{
+	Name:   AppName,
+	Recipe: ExpectedAppEnv["RECIPE"],
+	Domain: ExpectedAppEnv["DOMAIN"],
+	Env:    ExpectedAppEnv,
+	Path:   ExpectedAppFile.Path,
+	Server: ExpectedAppFile.Server,
 }
 
-var expectedAppFile = AppFile{
-	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
-	Server: serverName,
+var ExpectedAppFile = config.AppFile{
+	Path:   path.Join(ValidAbraConf, "servers", ServerName, AppName+".env"),
+	Server: ServerName,
 }
 
-var expectedAppFiles = map[string]AppFile{
-	appName: expectedAppFile,
+var ExpectedAppFiles = map[string]config.AppFile{
+	AppName: ExpectedAppFile,
 }
 
-// var expectedServerNames = []string{"evil.corp"}
-
 func TestGetAllFoldersInDirectory(t *testing.T) {
-	folders, err := GetAllFoldersInDirectory(testFolder)
+	folders, err := config.GetAllFoldersInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(folders, tFolders) {
-		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	if !reflect.DeepEqual(folders, TFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(TFolders, ","), strings.Join(folders, ","))
 	}
 }
 
 func TestGetAllFilesInDirectory(t *testing.T) {
-	files, err := GetAllFilesInDirectory(testFolder)
+	files, err := config.GetAllFilesInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,23 +65,208 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 	for _, file := range files {
 		fileNames = append(fileNames, file.Name())
 	}
-	if !reflect.DeepEqual(fileNames, tFiles) {
-		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	if !reflect.DeepEqual(fileNames, TFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(TFiles, ","), strings.Join(fileNames, ","))
 	}
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := ReadEnv(expectedAppFile.Path)
+	env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(env, expectedAppEnv) {
+	if !reflect.DeepEqual(env, ExpectedAppEnv) {
 		t.Fatalf(
 			"did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s",
-			expectedAppEnv["DOMAIN"],
-			expectedAppEnv["RECIPE"],
+			ExpectedAppEnv["DOMAIN"],
+			ExpectedAppEnv["RECIPE"],
 			env["DOMAIN"],
 			env["RECIPE"],
 		)
 	}
 }
+
+func TestReadAbraShEnvVars(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(abraShEnv) == 0 {
+		t.Error("at least one env var should be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_FOO"]; ok {
+		t.Error("INNER_FOO should not be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_BAZ"]; ok {
+		t.Error("INNER_BAZ should not be exported")
+	}
+
+	if _, ok := abraShEnv["OUTER_FOO"]; !ok {
+		t.Error("OUTER_FOO should be exported")
+	}
+}
+
+func TestReadAbraShCmdNames(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(cmdNames) == 0 {
+		t.Error("at least one command name should be found")
+	}
+
+	expectedCmdNames := []string{"test_cmd", "test_cmd_args"}
+	for _, cmdName := range expectedCmdNames {
+		if !slices.Contains(cmdNames, cmdName) {
+			t.Fatalf("%s should have been found in %s", cmdName, abraShPath)
+		}
+	}
+}
+
+func TestCheckEnv(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if !envVar.Present {
+			t.Fatalf("%s should be present", envVar.Name)
+		}
+	}
+}
+
+func TestCheckEnvError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	delete(envSample, "DOMAIN")
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if envVar.Name == "DOMAIN" && envVar.Present {
+			t.Fatalf("%s should not be present", envVar.Name)
+		}
+	}
+}
+
+func TestContainsEnvVarModifier(t *testing.T) {
+	if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
+		t.Fatal("FOO contains no env var modifier")
+	}
+
+	if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
+		t.Fatal("FOO contains an env var modifier (length)")
+	}
+}
+
+func TestEnvVarCommentsRemoved(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envVar, exists := envSample["WITH_COMMENT"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "should be removed") {
+		t.Fatalf("comment from '%s' should be removed", envVar)
+	}
+
+	envVar, exists = envSample["SECRET_TEST_PASS_TWO_VERSION"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should have been removed")
+	}
+}
+
+func TestEnvVarModifiersIncluded(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
+	}
+}

pkg/dns/dns.go

@@ -7,11 +7,16 @@ import (
 
 // EnsureIPv4 ensures that an ipv4 address is set for a domain name
 func EnsureIPv4(domainName string) (string, error) {
-	ipv4, err := net.ResolveIPAddr("ip", domainName)
+	ipv4, err := net.ResolveIPAddr("ip4", domainName)
 	if err != nil {
 		return "", err
 	}
 
+	// NOTE(d1): e.g. when there is only an ipv6 record available
+	if ipv4 == nil {
+		return "", fmt.Errorf("unable to resolve ipv4 address for %s", domainName)
+	}
+
 	return ipv4.String(), nil
 }
 

pkg/dns/dns_test.go

@@ -0,0 +1,64 @@
+package dns
+
+import (
+	"fmt"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
+	tests := []struct {
+		domainName     string
+		serverName     string
+		shouldValidate bool
+	}{
+		// NOTE(d1): DNS records get checked, so use something that is maintained
+		// within the federation. if you're here because of a failing test, try
+		// `dig +short <domain>` to ensure stuff matches first! If flakyness
+		// becomes an issue we can look into mocking
+		{"docs.coopcloud.tech", "coopcloud.tech", true},
+		{"docs.coopcloud.tech", "swarm.autonomic.zone", true},
+
+		// NOTE(d1): special case handling for "--local"
+		{"", "default", true},
+		{"", "local", true},
+
+		{"", "", false},
+		{"123", "", false},
+	}
+
+	for _, test := range tests {
+		_, err := EnsureDomainsResolveSameIPv4(test.domainName, test.serverName)
+		if err != nil && test.shouldValidate {
+			t.Fatal(err)
+		}
+
+		if err == nil && !test.shouldValidate {
+			t.Fatal(fmt.Errorf("should have failed but did not: %v", test))
+		}
+	}
+}
+
+func TestEnsureIpv4(t *testing.T) {
+	// NOTE(d1): DNS records get checked, so use something that is maintained
+	// within the federation. if you're here because of a failing test, try `dig
+	// +short <domain>` to ensure stuff matches first! If flakyness becomes an
+	// issue we can look into mocking
+	domainName := "collabora.ostrom.collective.tools"
+	serverName := "ostrom.collective.tools"
+
+	for i := 0; i < 15; i++ {
+		domainIpv4, err := EnsureIPv4(domainName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		serverIpv4, err := EnsureIPv4(serverName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, domainIpv4, serverIpv4)
+	}
+}

pkg/git/commit.go

@@ -8,7 +8,7 @@ import (
 )
 
 // Commit runs a git commit
-func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
+func Commit(repoPath, commitMessage string, dryRun bool) error {
 	if commitMessage == "" {
 		return fmt.Errorf("no commit message specified?")
 	}
@@ -33,17 +33,8 @@ func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
 	}
 
 	if !dryRun {
-		err = commitWorktree.AddGlob(glob)
-		if err != nil {
-			return err
-		}
-		logrus.Debugf("staged %s for commit", glob)
-	} else {
-		logrus.Debugf("dry run: did not stage %s for commit", glob)
-	}
-
-	if !dryRun {
-		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
+		// NOTE(d1): `All: true` does not include untracked files
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{All: true})
 		if err != nil {
 			return err
 		}

pkg/git/diff.go

@@ -0,0 +1,42 @@
+package git
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+)
+
+// getGitDiffArgs builds the `git diff` invocation args. It removes the usage
+// of a pager and ensures that colours are specified even when Git might detect
+// otherwise.
+func getGitDiffArgs(repoPath string) []string {
+	return []string{
+		"-C",
+		repoPath,
+		"--no-pager",
+		"-c",
+		"color.diff=always",
+		"diff",
+	}
+}
+
+// DiffUnstaged shows a `git diff`. Due to limitations in the underlying go-git
+// library, this implementation requires the /usr/bin/git binary. It gracefully
+// skips if it cannot find the command on the system.
+func DiffUnstaged(path string) error {
+	if _, err := exec.LookPath("git"); err != nil {
+		logrus.Warnf("unable to locate git command, cannot output diff")
+		return nil
+	}
+
+	gitDiffArgs := getGitDiffArgs(path)
+	diff, err := exec.Command("git", gitDiffArgs...).Output()
+	if err != nil {
+		return nil
+	}
+
+	fmt.Print(string(diff))
+
+	return nil
+}

pkg/jsontable/jsontable.go

@@ -3,6 +3,7 @@ package jsontable
 import (
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/olekukonko/tablewriter"
 )
@@ -109,6 +110,9 @@ func (t *JSONTable) _JSONRenderInner() {
 		}
 		writeChar(t.out, '{')
 		for keyidx, key := range t.keys {
+			key := strings.ToLower(key)
+			key = strings.ReplaceAll(key, " ", "-")
+
 			value := "nil"
 			if keyidx < len(row) {
 				value = row[keyidx]
@@ -138,10 +142,8 @@ func (t *JSONTable) JSONRender() {
 
 		if t.hasCaption {
 			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
-
 		}
 		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
-
 	}
 
 	// write list

pkg/lint/recipe.go

@@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}

pkg/recipe/recipe.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -31,7 +32,7 @@ import (
 // RecipeCatalogueURL is the only current recipe catalogue available.
 const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
 
-// ReposMetadataURL is the recipe repository metadata
+// ReposMetadataURL is the recipe repository metadata.
 const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
 
 // tag represents a git tag.
@@ -63,6 +64,11 @@ type RecipeMeta struct {
 	Website       string         `json:"website"`
 }
 
+// TopicMeta represents a list of topics for a repository.
+type TopicMeta struct {
+	Topics []string `json:"topics"`
+}
+
 // LatestVersion returns the latest version of a recipe.
 func (r RecipeMeta) LatestVersion() string {
 	var version string
@@ -221,7 +227,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -249,9 +255,9 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}, nil
 }
 
-func (r Recipe) SampleEnv() (map[string]string, error) {
+func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, opts)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}
@@ -822,7 +828,16 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 		}
 
 		for idx, repo := range reposList {
-			reposMeta[repo.Name] = reposList[idx]
+			var topicMeta TopicMeta
+
+			topicsURL := getReposTopicUrl(repo.Name)
+			if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
+				return reposMeta, err
+			}
+
+			if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
+				reposMeta[repo.Name] = reposList[idx]
+			}
 		}
 
 		pageIdx++
@@ -1002,14 +1017,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 				retrieveBar.Add(1)
 				return
 			}
-			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
 
 			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
 			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
 				logrus.Fatal(err)
 			}
@@ -1025,3 +1034,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 
 	return nil
 }
+
+// getReposTopicUrl retrieves the repository specific topic listing.
+func getReposTopicUrl(repoName string) string {
+	return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
+}

pkg/secret/secret.go

@@ -4,6 +4,7 @@
 package secret
 
 import (
+	"context"
 	"fmt"
 	"slices"
 	"strconv"
@@ -11,9 +12,11 @@ import (
 	"sync"
 
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/decentral1se/passgen"
+	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
@@ -66,9 +69,14 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeName string) (map[string]string, error) {
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
 	secretConfigs := make(map[string]string)
 
+	appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
+	if err != nil {
+		return secretConfigs, err
+	}
+
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
@@ -93,7 +101,7 @@ func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeNa
 		}
 
 		if !(slices.Contains(enabledSecrets, secretId)) {
-			logrus.Warnf("%s not enabled in recipe config, not generating", secretId)
+			logrus.Warnf("%s not enabled in recipe config, skipping", secretId)
 			continue
 		}
 
@@ -209,3 +217,66 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 	return secrets, nil
 }
+
+type secretStatus struct {
+	LocalName       string
+	RemoteName      string
+	Version         string
+	CreatedOnRemote bool
+}
+
+type secretStatuses []secretStatus
+
+// PollSecretsStatus checks status of secrets by comparing the local recipe
+// config and deploymend server state.
+func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, error) {
+	var secStats secretStatuses
+
+	composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+	if err != nil {
+		return secStats, err
+	}
+
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+	if err != nil {
+		return secStats, err
+	}
+
+	remoteSecretNames := make(map[string]bool)
+	for _, cont := range secretList {
+		remoteSecretNames[cont.Spec.Annotations.Name] = true
+	}
+
+	for secretName, secretValue := range secretsConfig {
+		createdRemote := false
+
+		val, err := ParseSecretValue(secretValue)
+		if err != nil {
+			return secStats, err
+		}
+
+		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+		if _, ok := remoteSecretNames[secretRemoteName]; ok {
+			createdRemote = true
+		}
+
+		secStats = append(secStats, secretStatus{
+			LocalName:       secretName,
+			RemoteName:      secretRemoteName,
+			Version:         val.Version,
+			CreatedOnRemote: createdRemote,
+		})
+	}
+
+	return secStats, nil
+}

pkg/secret/secret_test.go

@@ -18,13 +18,14 @@ func TestReadSecretsConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sampleEnv, err := recipe.SampleEnv()
+	sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
-	secretsFromConfig, err := ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/upstream/convert/service.go

@@ -420,7 +420,7 @@ func convertServiceSecrets(
 			return nil, err
 		}
 
-		// NOTE(d1): strip # length=... modifiers
+		// NOTE(d1): strip all comments
 		if strings.Contains(obj.Name, "#") {
 			vals := strings.Split(obj.Name, "#")
 			obj.Name = strings.TrimSpace(vals[0])

scripts/installer/installer

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.8.0-beta"
+ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.0-beta"
+RC_VERSION="0.8.1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do

tests/integration/app_check.bats

@@ -35,7 +35,7 @@ setup(){
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
-  assert_output --partial 'all necessary environment variables defined'
+  refute_output --partial '❌'
 
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 }
@@ -111,12 +111,16 @@ setup(){
 }
 
 @test "error if missing env var" {
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+  refute_output --partial '❌'
+
   run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
   assert_success
 
   run $ABRA app check "$TEST_APP_DOMAIN" --chaos
-  assert_failure
-  assert_output --partial "NEW_VAR"
+  assert_success
+  assert_output --partial '❌'
 
   _checkout_recipe
 }

tests/integration/app_deploy.bats

@@ -272,6 +272,21 @@ teardown(){
   assert_success
 }
 
+@test "ensure domain is checked" {
+  appDomain="custom-html.DOESNTEXIST"
+
+  run $ABRA app new custom-html \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$appDomain"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$appDomain.env"
+
+  run $ABRA app deploy "$appDomain" --no-input
+  assert_failure
+  assert_output --partial 'no such host'
+}
+
 # bats test_tags=slow
 @test "skip domain check when requested" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
@@ -296,3 +311,54 @@ teardown(){
 
   _undeploy_app
 }
+
+@test "bail out if specific version and chaos" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
+# bats test_tags=slow
+@test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
+  _reset_recipe
+
+  run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  # NOTE(d1): --chaos used to bypass versions and access compose.extra_env.yml
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial "compose.yml"
+  assert_output --partial "compose.extra_env.yml"
+
+  _undeploy_app
+  _reset_app
+}
+
+@test "error if no secrets generated" {
+  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'unable to deploy, secrets not generated'
+
+  _reset_app
+}
+
+@test "recipe config comments not present in values" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app env
+  assert_success
+  refute_output --partial 'should be removed'
+}

tests/integration/app_logs.bats

@@ -32,3 +32,15 @@ setup(){
   assert_failure
   assert_output --partial 'is not deployed'
 }
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app logs "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}

tests/integration/app_rollback.bats

@@ -194,6 +194,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app rollback "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "rollback to previous version" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks

tests/integration/app_secret.bats

@@ -26,19 +26,6 @@ teardown(){
   fi
 }
 
-_reset_app(){
-  run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-  assert_success
-  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  run $ABRA app new "$TEST_RECIPE" \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-  assert_success
-  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-}
-
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
@@ -123,10 +110,11 @@ setup(){
   assert_success
   assert_output --partial 'test_pass_one'
 
-  run docker -c "$TEST_SERVER" secret ls
+  run bash -c '$ABRA app secret ls $TEST_APP_DOMAIN --machine | \
+    jq -r ".[] | select(.name==\"test_pass_one\") | .version"'
   assert_success
-  assert_output --regexp ".*_test_pass_one_v2"
-  refute_output --regexp ".*_test_pass_one_v1"
+  assert_output --partial 'v2'
+  refute_output --partial 'v1'
 
   run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
   assert_success
@@ -338,6 +326,22 @@ setup(){
   assert_success
 }
 
+@test "ls: show secrets as machine readable" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN" --machine
+  assert_success
+  assert_output --partial '"created-on-server":"true"'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+}
+
 @test "ls: bail if unstaged changes and no --chaos" {
   run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success

tests/integration/app_upgrade.bats

@@ -47,6 +47,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "no upgrade if lint error" {
   _deploy_app

tests/integration/helpers/app.bash

@@ -40,3 +40,16 @@ _rm_app() {
     run $ABRA app remove "$TEST_APP_DOMAIN" --no-input
   fi
 }
+
+_reset_app(){
+  run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN" \
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+}

tests/integration/helpers/common.bash

@@ -15,5 +15,5 @@ _common_setup() {
 
   export TEST_APP_NAME="$(basename "${BATS_TEST_FILENAME//./_}")"
   export TEST_APP_DOMAIN="$TEST_APP_NAME.$TEST_SERVER"
-  export TEST_RECIPE="abra-integration-test-recipe"
+  export TEST_RECIPE="abra-test-recipe"
 }

tests/integration/recipe_diff.bats

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "show unstaged changes" {
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  refute_output --partial 'traefik.enable'
+
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  assert_output --partial 'traefik.enable'
+
+  _reset_recipe
+}

tests/integration/recipe_release.bats

@@ -84,3 +84,22 @@ setup(){
 
   _reset_recipe "$TEST_RECIPE"
 }
+
+@test "unknown files not committed" {
+  run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch
+  assert_success
+
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA recipe release "$TEST_RECIPE" --no-input --patch
+  assert_success
+  assert_output --partial 'no -p/--publish passed, not publishing'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo
+  assert_failure
+  assert_output --partial "fatal: pathspec 'foo' did not match any files"
+
+  _reset_recipe
+}

tests/integration/recipe_reset.bats

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "reset unstaged changes" {
+  run $ABRA recipe fetch "$TEST_RECIPE"
+  assert_success
+
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  assert_output --partial 'traefik.enable'
+
+  run $ABRA recipe reset "$TEST_RECIPE"
+  assert_success
+
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  refute_output --partial 'traefik.enable'
+}