proper env modifiers support

This implements proper modifier support in the env file using this new fork of the godotenv library. The modifier implementation is quite basic for but can be improved later if needed. See this commit for the actual implementation.

Because we are now using proper modifer parsing, it does not affect the parsing of value, so this is possible again:
```
MY_VAR="#foo"
```
Closes coop-cloud/organising#535

.dockerignore

@@ -1,4 +1,8 @@
-Dockerfile
-.dockerignore
-*.swp
 *.swo
+*.swp
+.dockerignore
+Dockerfile
+abra
+dist
+kadabra
+tags

.drone.yml

@@ -3,20 +3,17 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.20
+    image: golang:1.21
     commands:
       - make check
 
-  - name: make build
-    image: golang:1.20
-    commands:
-      - make build
-    depends_on:
-      - make check
-
   - name: make test
-    image: golang:1.20
+    image: golang:1.21
+    environment:
+      ABRA_DIR: "/root/.abra"
     commands:
+      - make build-abra
+      - ./abra help # show version, initialise $ABRA_DIR
       - make test
     depends_on:
       - make check
@@ -27,13 +24,12 @@ steps:
       - git fetch --tags
     depends_on:
       - make check
-      - make build
       - make test
     when:
       event: tag
 
   - name: release
-    image: golang:1.20
+    image: goreleaser/goreleaser:v1.18.2
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token
@@ -41,7 +37,7 @@ steps:
       - name: deps
         path: /go
     commands:
-      - curl -sL https://git.io/goreleaser | bash
+      - goreleaser release
     depends_on:
       - fetch
     when:

.e2e.env.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

.envrc.sample

@@ -1,6 +1,7 @@
 go env -w GOPRIVATE=coopcloud.tech
 
 # export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
-# export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
-# export CAPSUL_TOKEN=...
-# export GITEA_TOKEN=...
+
+# export ABRA_DIR="$HOME/.abra_test"
+# export ABRA_TEST_DOMAIN=test.example.com
+# export ABRA_SKIP_TEARDOWN=1 # for faster feedback when developing tests

.gitignore

@@ -5,5 +5,5 @@
 /kadabra
 abra
 dist/
-tests/integration/.abra/catalogue
+tests/integration/.bats
 vendor/

AUTHORS.md

@@ -1,7 +1,7 @@
 # authors
 
 > If you're looking at this and you hack on `abra` and you're not listed here,
-> please do add yourself! This is a community project, let's show some :heart:
+> please do add yourself! This is a community project, let's show some 💞
 
 - 3wordchant
 - cassowary
@@ -11,6 +11,8 @@
 - kawaiipunk
 - knoflook
 - moritz
+- p4u1
+- rix
 - roxxers
 - vera
 - yksflip

Dockerfile

@@ -1,8 +1,15 @@
-FROM golang:1.20-alpine AS build
+FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
-RUN apk add --no-cache make git gcc musl-dev
+RUN apk add --no-cache \
+  ca-certificates \
+  gcc \
+  git \
+  make \
+  musl-dev
+
+RUN update-ca-certificates
 
 COPY . /app
 

Makefile

@@ -2,26 +2,41 @@ ABRA         := ./cmd/abra
 KADABRA      := ./cmd/kadabra
 COMMIT       := $(shell git rev-list -1 HEAD)
 GOPATH       := $(shell go env GOPATH)
+GOVERSION    := 1.21
 LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 
 export GOPRIVATE=coopcloud.tech
 
-all: format check build test
+# NOTE(d1): default `make` optimised for Abra hacking
+all: format check build-abra test
 
-run:
+run-abra:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
 
-install:
+run-kadabra:
+	@go run -ldflags=$(LDFLAGS) $(KADABRA)
+
+install-abra:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
-build-dev:
-	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
+install-kadabra:
+	@go install -ldflags=$(LDFLAGS) $(KADABRA)
 
-build:
+build-abra:
 	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
+
+build-kadabra:
 	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
+build: build-abra build-kadabra
+
+build-docker-abra:
+	@docker run -it -v $(PWD):/abra golang:$(GOVERSION) \
+		bash -c 'cd /abra; ./scripts/docker/build.sh'
+
+build-docker: build-docker-abra
+
 clean:
 	@rm '$(GOPATH)/bin/abra'
 	@rm '$(GOPATH)/bin/kadabra'
@@ -38,10 +53,3 @@ test:
 
 loc:
 	@find . -name "*.go" | xargs wc -l
-
-loc-author:
-	@git ls-files -z | \
-		xargs -0rn 1 -P "$$(nproc)" -I{} sh -c 'git blame -w -M -C -C --line-porcelain -- {} | grep -I --line-buffered "^author "' | \
-   	sort -f | \
-   	uniq -ic | \
-   	sort -n

README.md

@@ -2,11 +2,12 @@
 
 [![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
 [![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
+[![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
 
 The Co-op Cloud utility belt 🎩🐇
 
 <a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community 💖
 
 Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

cli/app/backup.go

@@ -15,15 +15,13 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/system"
 	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -42,6 +40,8 @@ var appBackupCommand = cli.Command{
 	ArgsUsage: "<domain> [<service>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
@@ -73,16 +73,26 @@ This single file can be used to restore your app. See "abra app restore" for mor
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		conf := runtime.New()
 
-		cl, err := client.New(app.Server)
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		recipe, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 		backupConfigs := make(map[string]backupConfig)
@@ -114,6 +124,11 @@ This single file can be used to restore your app. See "abra app restore" for mor
 			}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName != "" {
 			backupConfig, ok := backupConfigs[serviceName]
@@ -262,7 +277,7 @@ func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkCo
 }
 
 func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp")
+	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
 	if err != nil {
 		return err
 	}

cli/app/check.go

@@ -1,57 +1,81 @@
 package app
 
 import (
-	"os"
-	"path"
-	"strings"
-
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
 var appCheckCommand = cli.Command{
-	Name:      "check",
-	Aliases:   []string{"chk"},
-	Usage:     "Check if app is configured correctly",
+	Name:    "check",
+	Aliases: []string{"chk"},
+	Usage:   "Ensure an app is well configured",
+	Description: `
+This command compares env vars in both the app ".env" and recipe ".env.sample"
+file.
+
+The goal is to ensure that recipe ".env.sample" env vars are defined in your
+app ".env" file. Only env var definitions in the ".env.sample" which are
+uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include
+these env vars, then "check" will complain.
+
+Recipe maintainers may or may not provide defaults for env vars within their
+recipes regardless of commenting or not (e.g. through the use of
+${FOO:<default>} syntax). "check" does not confirm or deny this for you.`,
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.ChaosFlag,
+		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
-		if _, err := os.Stat(envSamplePath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
-			}
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
-		envSample, err := config.ReadEnv(envSamplePath)
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		tableCol := []string{"recipe env sample", "app env"}
+		table := formatter.CreateTable(tableCol)
+
+		envVars, err := config.CheckEnv(app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		var missing []string
-		for k := range envSample {
-			if _, ok := app.Env[k]; !ok {
-				missing = append(missing, k)
+		for _, envVar := range envVars {
+			if envVar.Present {
+				table.Append([]string{envVar.Name, "✅"})
+			} else {
+				table.Append([]string{envVar.Name, "❌"})
 			}
 		}
 
-		if len(missing) > 0 {
-			missingEnvVars := strings.Join(missing, ", ")
-			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
-		}
-
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+		table.Render()
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cmd.go

@@ -1,27 +1,21 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
+	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -30,8 +24,7 @@ var appCmdCommand = cli.Command{
 	Name:    "command",
 	Aliases: []string{"cmd"},
 	Usage:   "Run app commands",
-	Description: `
-Run an app specific command.
+	Description: `Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
@@ -47,17 +40,46 @@ Example:
 		internal.DebugFlag,
 		internal.LocalCmdFlag,
 		internal.RemoteUserFlag,
+		internal.TtyFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:      internal.SubCommandBefore,
+	Subcommands: []cli.Command{appCmdListCommand},
+	BashComplete: func(ctx *cli.Context) {
+		args := ctx.Args()
+		switch len(args) {
+		case 0:
+			autocomplete.AppNameComplete(ctx)
+		case 1:
+			autocomplete.ServiceNameComplete(args.Get(0))
+		case 2:
+			cmdNameComplete(args.Get(0))
+		}
 	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if internal.LocalCmd && internal.RemoteUser != "" {
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
 		}
@@ -73,8 +95,12 @@ Example:
 		}
 
 		if internal.LocalCmd {
+			if !(len(c.Args()) >= 2) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
 			cmdName := c.Args().Get(1)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
 				logrus.Fatal(err)
 			}
 
@@ -84,6 +110,7 @@ Example:
 			for k, v := range app.Env {
 				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
 			}
+
 			var sourceAndExec string
 			if hasCmdArgs {
 				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
@@ -104,10 +131,14 @@ Example:
 				logrus.Fatal(err)
 			}
 		} else {
+			if !(len(c.Args()) >= 3) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
 			targetServiceName := c.Args().Get(1)
 
 			cmdName := c.Args().Get(2)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
 				logrus.Fatal(err)
 			}
 
@@ -135,7 +166,12 @@ Example:
 				logrus.Debug("did not detect any command arguments")
 			}
 
-			if err := runCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := internal.RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -163,83 +199,75 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 	return hasCmdArgs, parsedCmdArgs
 }
 
-func ensureCommand(abraSh, recipeName, execCmd string) error {
-	bytes, err := ioutil.ReadFile(abraSh)
+func cmdNameComplete(appName string) {
+	app, err := app.Get(appName)
 	if err != nil {
-		return err
+		return
 	}
-
-	if !strings.Contains(string(bytes), execCmd) {
-		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
+	cmdNames, _ := getShCmdNames(app)
+	if err != nil {
+		return
+	}
+	for _, n := range cmdNames {
+		fmt.Println(n)
 	}
-
-	return nil
 }
 
-func runCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+var appCmdListCommand = cli.Command{
+	Name:      "list",
+	Aliases:   []string{"ls"},
+	Usage:     "List all available commands",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 
-	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-	content, err := archive.TarWithOptions(abraSh, toTarOpts)
-	if err != nil {
-		return err
-	}
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
-		return err
-	}
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
+		cmdNames, err := getShCmdNames(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-	shell := "/bin/bash"
-	findShell := []string{"test", "-e", shell}
-	execCreateOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          findShell,
-		Detach:       false,
-		Tty:          false,
-	}
+		for _, cmdName := range cmdNames {
+			fmt.Println(cmdName)
+		}
 
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
-		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
-		shell = "/bin/sh"
-	}
-
-	var cmd []string
-	if cmdArgs != "" {
-		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.Name, app.StackName(), cmdName, cmdArgs)}
-	} else {
-		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.Name, app.StackName(), cmdName)}
-	}
-
-	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
-
-	if internal.RemoteUser != "" {
-		logrus.Debugf("running command with user %s", internal.RemoteUser)
-		execCreateOpts.User = internal.RemoteUser
-	}
-
-	execCreateOpts.Cmd = cmd
-	execCreateOpts.Tty = true
-
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
-		return err
-	}
-
-	return nil
+		return nil
+	},
+}
+
+func getShCmdNames(app config.App) ([]string, error) {
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Strings(cmdNames)
+	return cmdNames, nil
 }

cli/app/config.go

@@ -21,7 +21,8 @@ var appConfigCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		appName := c.Args().First()
 
@@ -60,5 +61,4 @@ var appConfigCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cp.go

@@ -29,7 +29,7 @@ var appCpCommand = cli.Command{
 		internal.NoInputFlag,
 	},
 	Before: internal.SubCommandBefore,
-	Usage:  "Copy files to/from a running app service",
+	Usage:  "Copy files to/from a deployed app service",
 	Description: `
 Copy files to and from any app service file system.
 
@@ -39,16 +39,12 @@ If you want to copy a myfile.txt to the root of the app service:
 
 And if you want to copy that file back to your current working directory locally:
 
-		abra app cp <domain> app:/myfile.txt .
+    abra app cp <domain> app:/myfile.txt .
 `,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		src := c.Args().Get(1)
 		dst := c.Args().Get(2)
 		if src == "" {
@@ -89,19 +85,23 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Debugf("assuming transfer is going TO the container")
 		}
 
-		if !isToContainer {
-			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist locally?", dstPath)
+		if isToContainer {
+			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", srcPath)
 			}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 func configureAndCp(
@@ -123,10 +123,6 @@ func configureAndCp(
 	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 	if isToContainer {
-		if _, err := os.Stat(srcPath); err != nil {
-			logrus.Fatalf("%s does not exist?", srcPath)
-		}
-
 		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
 		content, err := archive.TarWithOptions(srcPath, toTarOpts)
 		if err != nil {

cli/app/deploy.go

@@ -1,8 +1,22 @@
 package app
 
 import (
+	"context"
+	"fmt"
+
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/secret"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
@@ -10,7 +24,7 @@ var appDeployCommand = cli.Command{
 	Name:      "deploy",
 	Aliases:   []string{"d"},
 	Usage:     "Deploy an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -18,6 +32,7 @@ var appDeployCommand = cli.Command{
 		internal.ChaosFlag,
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -27,10 +42,213 @@ for this you need to look at the "abra app upgrade <domain>" command.
 You may pass "--force" to re-deploy the same version again. This can be useful
 if the container runtime has gotten into a weird state.
 
-Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	Action:       internal.DeployAction,
 	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		r, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(r); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secStats, err := secret.PollSecretsStatus(cl, app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, secStat := range secStats {
+			if !secStat.CreatedOnRemote {
+				logrus.Fatalf("unable to deploy, secrets not generated (%s)?", secStat.LocalName)
+			}
+		}
+
+		if isDeployed {
+			if internal.Force || internal.Chaos {
+				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
+			} else {
+				logrus.Fatalf("%s is already deployed", app.Name)
+			}
+		}
+
+		version := deployedVersion
+		if specificVersion != "" {
+			version = specificVersion
+			logrus.Debugf("choosing %s as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if !internal.Chaos && specificVersion == "" {
+			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if len(versions) == 0 && !internal.Chaos {
+				logrus.Warn("no published versions in catalogue, trying local recipe repository")
+				recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+				if err != nil {
+					logrus.Warn(err)
+				}
+				for _, recipeVersion := range recipeVersions {
+					for version := range recipeVersion {
+						versions = append(versions, version)
+					}
+				}
+			}
+
+			if len(versions) > 0 && !internal.Chaos {
+				version = versions[len(versions)-1]
+				logrus.Debugf("choosing %s as version to deploy", version)
+				if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				head, err := git.GetRecipeHead(app.Recipe)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				version = formatter.SmallSHA(head.String())
+				logrus.Warn("no versions detected, using latest commit")
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warnf("chaos mode engaged")
+			var err error
+			version, err = recipe.ChaosVersion(app.Recipe)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, version)
+		config.SetUpdateLabel(compose, stackName, app.Env)
+
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
+		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.NoDomainChecks {
+			domainName, ok := app.Env["DOMAIN"]
+			if ok {
+				if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
+			}
+		} else {
+			logrus.Warn("skipping domain checks as requested")
+		}
+
+		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_DEPLOY_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+			}
+		}
+		return nil
+	},
 }

cli/app/errors.go

@@ -12,7 +12,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -51,12 +50,12 @@ the logs.
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.WatchFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		conf := runtime.New()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -73,25 +72,23 @@ the logs.
 		}
 
 		if !internal.Watch {
-			if err := checkErrors(c, cl, app, conf); err != nil {
+			if err := checkErrors(c, cl, app); err != nil {
 				logrus.Fatal(err)
 			}
 			return nil
 		}
 
 		for {
-			if err := checkErrors(c, cl, app, conf); err != nil {
+			if err := checkErrors(c, cl, app); err != nil {
 				logrus.Fatal(err)
 			}
 			time.Sleep(2 * time.Second)
 		}
-
-		return nil
 	},
 }
 
-func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App, conf *runtime.Config) error {
-	recipe, err := recipe.Get(app.Recipe, conf)
+func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
+	recipe, err := recipe.Get(app.Recipe, internal.Offline)
 	if err != nil {
 		return err
 	}

cli/app/list.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -39,13 +40,16 @@ var listAppServerFlag = &cli.StringFlag{
 }
 
 type appStatus struct {
-	Server  string `json:"server"`
-	Recipe  string `json:"recipe"`
-	AppName string `json:"appName"`
-	Domain  string `json:"domain"`
-	Status  string `json:"status"`
-	Version string `json:"version"`
-	Upgrade string `json:"upgrade"`
+	Server       string `json:"server"`
+	Recipe       string `json:"recipe"`
+	AppName      string `json:"appName"`
+	Domain       string `json:"domain"`
+	Status       string `json:"status"`
+	Chaos        string `json:"chaos"`
+	ChaosVersion string `json:"chaosVersion"`
+	AutoUpdate   string `json:"autoUpdate"`
+	Version      string `json:"version"`
+	Upgrade      string `json:"upgrade"`
 }
 
 type serverStatus struct {
@@ -75,6 +79,7 @@ can take some time.
 		statusFlag,
 		listAppServerFlag,
 		recipeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
@@ -105,7 +110,7 @@ can take some time.
 				logrus.Fatal(err)
 			}
 
-			catl, err = recipe.ReadRecipeCatalogue()
+			catl, err = recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -138,12 +143,24 @@ can take some time.
 				if status {
 					status := "unknown"
 					version := "unknown"
+					chaos := "unknown"
+					chaosVersion := "unknown"
+					autoUpdate := "unknown"
 					if statusMeta, ok := statuses[app.StackName()]; ok {
 						if currentVersion, exists := statusMeta["version"]; exists {
 							if currentVersion != "" {
 								version = currentVersion
 							}
 						}
+						if chaosDeploy, exists := statusMeta["chaos"]; exists {
+							chaos = chaosDeploy
+						}
+						if chaosDeployVersion, exists := statusMeta["chaosVersion"]; exists {
+							chaosVersion = chaosDeployVersion
+						}
+						if autoUpdateState, exists := statusMeta["autoUpdate"]; exists {
+							autoUpdate = autoUpdateState
+						}
 						if statusMeta["status"] != "" {
 							status = statusMeta["status"]
 						}
@@ -153,7 +170,10 @@ can take some time.
 					}
 
 					appStats.Status = status
+					appStats.Chaos = chaos
+					appStats.ChaosVersion = chaosVersion
 					appStats.Version = version
+					appStats.AutoUpdate = autoUpdate
 
 					var newUpdates []string
 					if version != "unknown" {
@@ -223,7 +243,7 @@ can take some time.
 
 			tableCol := []string{"recipe", "domain"}
 			if status {
-				tableCol = append(tableCol, []string{"status", "version", "upgrade"}...)
+				tableCol = append(tableCol, []string{"status", "chaos", "version", "upgrade", "autoupdate"}...)
 			}
 
 			table := formatter.CreateTable(tableCol)
@@ -231,7 +251,17 @@ can take some time.
 			for _, appStat := range serverStat.Apps {
 				tableRow := []string{appStat.Recipe, appStat.Domain}
 				if status {
-					tableRow = append(tableRow, []string{appStat.Status, appStat.Version, appStat.Upgrade}...)
+					chaosStatus := appStat.Chaos
+					if chaosStatus != "unknown" {
+						chaosEnabled, err := strconv.ParseBool(chaosStatus)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+						if chaosEnabled && appStat.ChaosVersion != "unknown" {
+							chaosStatus = appStat.ChaosVersion
+						}
+					}
+					tableRow = append(tableRow, []string{appStat.Status, chaosStatus, appStat.Version, appStat.Upgrade, appStat.AutoUpdate}...)
 				}
 				table.Append(tableRow)
 			}

cli/app/logs.go

@@ -11,8 +11,9 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
@@ -83,13 +84,27 @@ var appLogsCommand = cli.Command{
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c, runtime.WithEnsureRecipeExists(false))
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
 		logOpts.Since = internal.SinceLogs
 
 		serviceName := c.Args().Get(1)

cli/app/new.go

@@ -1,8 +1,21 @@
 package app
 
 import (
+	"fmt"
+	"path"
+
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/jsontable"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
@@ -38,9 +51,210 @@ var appNewCommand = cli.Command{
 		internal.DomainFlag,
 		internal.PassFlag,
 		internal.SecretsFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "[<recipe>]",
-	Action:       internal.NewAction,
 	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if err := ensureServerFlag(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := ensureDomainFlag(recipe, internal.NewAppServer); err != nil {
+			logrus.Fatal(err)
+		}
+
+		sanitisedAppName := config.SanitiseAppName(internal.Domain)
+		logrus.Debugf("%s sanitised as %s for new app", internal.Domain, sanitisedAppName)
+
+		if err := config.TemplateAppEnvSample(
+			recipe.Name,
+			internal.Domain,
+			internal.NewAppServer,
+			internal.Domain,
+		); err != nil {
+			logrus.Fatal(err)
+		}
+
+		var secrets AppSecrets
+		var secretTable *jsontable.JSONTable
+		if internal.Secrets {
+			sampleEnv, err := recipe.SampleEnv()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			composeFiles, err := config.GetComposeFiles(recipe.Name, sampleEnv)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
+			if err != nil {
+				return err
+			}
+
+			if err := promptForSecrets(recipe.Name, secretsConfig); err != nil {
+				logrus.Fatal(err)
+			}
+
+			cl, err := client.New(internal.NewAppServer)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secretCols := []string{"Name", "Value"}
+			secretTable = formatter.CreateTable(secretCols)
+			for name, val := range secrets {
+				secretTable.Append([]string{name, val})
+			}
+		}
+
+		if internal.NewAppServer == "default" {
+			internal.NewAppServer = "local"
+		}
+
+		tableCol := []string{"server", "recipe", "domain"}
+		table := formatter.CreateTable(tableCol)
+		table.Append([]string{internal.NewAppServer, recipe.Name, internal.Domain})
+
+		fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
+		fmt.Println("")
+		table.Render()
+		fmt.Println("")
+		fmt.Println("You can configure this app by running the following:")
+		fmt.Println(fmt.Sprintf("\n    abra app config %s", internal.Domain))
+		fmt.Println("")
+		fmt.Println("You can deploy this app by running the following:")
+		fmt.Println(fmt.Sprintf("\n    abra app deploy %s", internal.Domain))
+
+		if len(secrets) > 0 {
+			fmt.Println("")
+			fmt.Println("Here are your generated secrets:")
+			fmt.Println("")
+			secretTable.Render()
+			logrus.Warn("generated secrets are not shown again, please take note of them NOW")
+		}
+
+		return nil
+	},
+}
+
+// AppSecrets represents all app secrest
+type AppSecrets map[string]string
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
+	// NOTE(d1): trim to match app.StackName() implementation
+	if len(sanitisedAppName) > 45 {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
+		sanitisedAppName = sanitisedAppName[:45]
+	}
+
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if internal.Pass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(
+				secretValue,
+				secretName,
+				internal.Domain,
+				internal.NewAppServer,
+			); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return secrets, nil
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag(recipe recipe.Recipe, server string) error {
+	if internal.Domain == "" && !internal.NoInput {
+		prompt := &survey.Input{
+			Message: "Specify app domain",
+			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
+		}
+		if err := survey.AskOne(prompt, &internal.Domain); err != nil {
+			return err
+		}
+	}
+
+	if internal.Domain == "" {
+		return fmt.Errorf("no domain provided")
+	}
+
+	return nil
+}
+
+// promptForSecrets asks if we should generate secrets for a new app.
+func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
+	if len(secretsConfig) == 0 {
+		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
+		return nil
+	}
+
+	if !internal.Secrets && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: "Generate app secrets?",
+		}
+		if err := survey.AskOne(prompt, &internal.Secrets); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	servers, err := config.GetServers()
+	if err != nil {
+		return err
+	}
+
+	if internal.NewAppServer == "" && !internal.NoInput {
+		prompt := &survey.Select{
+			Message: "Select app server:",
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &internal.NewAppServer); err != nil {
+			return err
+		}
+	}
+
+	if internal.NewAppServer == "" {
+		return fmt.Errorf("no server provided")
+	}
+
+	return nil
 }

cli/app/remove.go

@@ -11,6 +11,7 @@ import (
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -42,6 +43,7 @@ flag.
 		internal.ForceFlag,
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
 	BashComplete: autocomplete.AppNameComplete,
 	Before:       internal.SubCommandBefore,
@@ -108,7 +110,8 @@ flag.
 			logrus.Fatal(err)
 		}
 
-		volumeListOKBody, err := cl.VolumeList(context.Background(), fs)
+		volumeListOptions := volume.ListOptions{fs}
+		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
 		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
@@ -120,8 +123,7 @@ flag.
 		}
 
 		if len(vols) > 0 {
-			var removeVols []string
-			for _, vol := range removeVols {
+			for _, vol := range vols {
 				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
 				if err != nil {
 					logrus.Fatal(err)

cli/app/restart.go

@@ -21,6 +21,7 @@ var appRestartCommand = cli.Command{
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Description:  `This command restarts a service within a deployed app.`,
@@ -39,6 +40,15 @@ var appRestartCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
 		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
 
 		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)

cli/app/restore.go

@@ -12,7 +12,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -35,6 +35,8 @@ var appRestoreCommand = cli.Command{
 	ArgsUsage: "<domain> <service> <file>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
@@ -48,7 +50,8 @@ restoring the backup.
 Unlike "abra app backup", restore must be run on a per-service basis. You can
 not restore all services in one go. Backup files produced by Abra are
 compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic.
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
 
 Example:
 
@@ -56,13 +59,28 @@ Example:
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		conf := runtime.New()
 
-		cl, err := client.New(app.Server)
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
@@ -79,11 +97,6 @@ Example:
 			}
 		}
 
-		recipe, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		restoreConfigs := make(map[string]restoreConfig)
 		for _, service := range recipe.Config.Services {
 			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
@@ -113,6 +126,11 @@ Example:
 			rsConfig = restoreConfig{}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
 			logrus.Fatal(err)
 		}
@@ -169,8 +187,8 @@ func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName
 		return err
 	}
 
-	// we use absolute paths so tar knows what to do. it will restore files
-	// according to the paths set in the compresed archive
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
 	restorePath := "/"
 
 	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}

cli/app/rollback.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 
@@ -23,7 +22,7 @@ var appRollbackCommand = cli.Command{
 	Name:      "rollback",
 	Aliases:   []string{"rl"},
 	Usage:     "Roll an app back to a previous version",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -31,6 +30,7 @@ var appRollbackCommand = cli.Command{
 		internal.ChaosFlag,
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -42,7 +42,7 @@ useful if the container runtime has gotten into a weird state.
 This action could be destructive, please ensure you have a copy of your app
 data beforehand.
 
-Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
@@ -50,15 +50,33 @@ recipes.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
-		conf := runtime.New()
+
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe, conf); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		r, err := recipe.Get(app.Recipe, conf)
+		r, err := recipe.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -83,7 +101,7 @@ recipes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -94,16 +112,40 @@ recipes.
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
 		}
 
 		var availableDowngrades []string
 		if deployedVersion == "unknown" {
 			availableDowngrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		if deployedVersion != "unknown" && !internal.Chaos {
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsGreaterThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not a downgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableDowngrades = append(availableDowngrades, specificVersion)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
 			for _, version := range versions {
 				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 				if err != nil {
@@ -113,28 +155,26 @@ recipes.
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				if parsedVersion != parsedDeployedVersion && parsedVersion.IsLessThan(parsedDeployedVersion) {
+				if parsedVersion.IsLessThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
 					availableDowngrades = append(availableDowngrades, version)
 				}
 			}
 
-			if len(availableDowngrades) == 0 {
+			if len(availableDowngrades) == 0 && !internal.Force {
 				logrus.Info("no available downgrades, you're on oldest ✌️")
 				return nil
 			}
 		}
 
-		availableDowngrades = internal.ReverseStringList(availableDowngrades)
-
 		var chosenDowngrade string
-		if !internal.Chaos {
-			if internal.Force || internal.NoInput {
-				chosenDowngrade = availableDowngrades[0]
-				logrus.Debugf("choosing %s as version to downgrade to (--force)", chosenDowngrade)
+		if len(availableDowngrades) > 0 && !internal.Chaos {
+			if internal.Force || internal.NoInput || specificVersion != "" {
+				chosenDowngrade = availableDowngrades[len(availableDowngrades)-1]
+				logrus.Debugf("choosing %s as version to downgrade to (--force/--no-input)", chosenDowngrade)
 			} else {
 				prompt := &survey.Select{
 					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
-					Options: availableDowngrades,
+					Options: internal.ReverseStringList(availableDowngrades),
 				}
 				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
 					return err
@@ -166,7 +206,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -183,15 +223,15 @@ recipes.
 		config.ExposeAllEnv(stackName, compose, app.Env)
 		config.SetRecipeLabel(compose, stackName, app.Recipe)
 		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, chosenDowngrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
-		if !internal.Force {
-			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
-				logrus.Fatal(err)
-			}
+		// NOTE(d1): no release notes implemeneted for rolling back
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
+			logrus.Fatal(err)
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose, stackName, internal.DontWaitConverge); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/secret.go

@@ -12,6 +12,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
@@ -19,19 +20,23 @@ import (
 	"github.com/urfave/cli"
 )
 
-var allSecrets bool
-var allSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &allSecrets,
-	Usage:       "Generate all secrets",
-}
+var (
+	allSecrets     bool
+	allSecretsFlag = &cli.BoolFlag{
+		Name:        "all, a",
+		Destination: &allSecrets,
+		Usage:       "Generate all secrets",
+	}
+)
 
-var rmAllSecrets bool
-var rmAllSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &rmAllSecrets,
-	Usage:       "Remove all secrets",
-}
+var (
+	rmAllSecrets     bool
+	rmAllSecretsFlag = &cli.BoolFlag{
+		Name:        "all, a",
+		Destination: &rmAllSecrets,
+		Usage:       "Remove all secrets",
+	}
+)
 
 var appSecretGenerateCommand = cli.Command{
 	Name:      "generate",
@@ -42,17 +47,35 @@ var appSecretGenerateCommand = cli.Command{
 		internal.DebugFlag,
 		allSecretsFlag,
 		internal.PassFlag,
+		internal.MachineReadableFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if len(c.Args()) == 1 && !allSecrets {
 			err := errors.New("missing arguments <secret>/<version> or '--all'")
 			internal.ShowSubcommandHelpAndError(c, err)
@@ -63,28 +86,35 @@ var appSecretGenerateCommand = cli.Command{
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
-		secretsToCreate := make(map[string]string)
-		secretEnvVars := secret.ReadSecretEnvVars(app.Env)
-		if allSecrets {
-			secretsToCreate = secretEnvVars
-		} else {
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !allSecrets {
 			secretName := c.Args().Get(1)
 			secretVersion := c.Args().Get(2)
-			matches := false
-			for sec := range secretEnvVars {
-				parsed := secret.ParseSecretEnvVarName(sec)
-				if secretName == parsed {
-					secretsToCreate[sec] = secretVersion
-					matches = true
-				}
-			}
-
-			if !matches {
+			s, ok := secrets[secretName]
+			if !ok {
 				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
+			s.Version = secretVersion
+			secrets = map[string]secret.SecretValue{
+				secretName: s,
+			}
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -107,8 +137,13 @@ var appSecretGenerateCommand = cli.Command{
 		for name, val := range secretVals {
 			table.Append([]string{name, val})
 		}
-		table.Render()
-		logrus.Warn("generated secrets are not shown again, please take note of them *now*")
+
+		if internal.MachineReadable {
+			table.JSONRender()
+		} else {
+			table.Render()
+		}
+		logrus.Warn("generated secrets are not shown again, please take note of them NOW")
 
 		return nil
 	},
@@ -140,15 +175,15 @@ Example:
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		if len(c.Args()) != 4 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if len(c.Args()) != 4 {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
-		}
-
 		name := c.Args().Get(1)
 		version := c.Args().Get(2)
 		data := c.Args().Get(3)
@@ -198,6 +233,8 @@ var appSecretRmCommand = cli.Command{
 		internal.NoInputFlag,
 		rmAllSecretsFlag,
 		internal.PassRemoveFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "<domain> [<secret-name>]",
@@ -211,7 +248,36 @@ Example:
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		secrets := secret.ReadSecretEnvVars(app.Env)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
 		if c.Args().Get(1) != "" && rmAllSecrets {
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret-name>' and '--all' together"))
@@ -243,15 +309,8 @@ Example:
 
 		match := false
 		secretToRm := c.Args().Get(1)
-		for sec := range secrets {
-			secretName := secret.ParseSecretEnvVarName(sec)
-
-			secVal, err := secret.ParseSecretEnvVarValue(secrets[sec])
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version)
+		for secretName, val := range secrets {
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 			if _, ok := remoteSecretNames[secretRemoteName]; ok {
 				if secretToRm != "" {
 					if secretName == secretToRm {
@@ -288,60 +347,71 @@ var appSecretLsCommand = cli.Command{
 	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+		internal.MachineReadableFlag,
 	},
-	Before: internal.SubCommandBefore,
-	Usage:  "List all secrets",
+	Before:       internal.SubCommandBefore,
+	Usage:        "List all secrets",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		secrets := secret.ReadSecretEnvVars(app.Env)
 
-		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
-		table := formatter.CreateTable(tableCol)
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, false)
+		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
+		table := formatter.CreateTable(tableCol)
+
+		secStats, err := secret.PollSecretsStatus(cl, app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteSecretNames := make(map[string]bool)
-		for _, cont := range secretList {
-			remoteSecretNames[cont.Spec.Annotations.Name] = true
-		}
-
-		for sec := range secrets {
-			createdRemote := false
-			secretName := secret.ParseSecretEnvVarName(sec)
-			secVal, err := secret.ParseSecretEnvVarValue(secrets[sec])
-			if err != nil {
-				logrus.Fatal(err)
+		for _, secStat := range secStats {
+			tableRow := []string{
+				secStat.LocalName,
+				secStat.Version,
+				secStat.RemoteName,
+				strconv.FormatBool(secStat.CreatedOnRemote),
 			}
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version)
-			if _, ok := remoteSecretNames[secretRemoteName]; ok {
-				createdRemote = true
-			}
-			tableRow := []string{secretName, secVal.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
 			table.Append(tableRow)
 		}
 
 		if table.NumLines() > 0 {
-			table.Render()
+			if internal.MachineReadable {
+				table.JSONRender()
+			} else {
+				table.Render()
+			}
 		} else {
 			logrus.Warnf("no secrets stored for %s", app.Name)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appSecretCommand = cli.Command{

cli/app/undeploy.go

@@ -101,6 +101,7 @@ Passing "-p/--prune" does not remove those volumes.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)

cli/app/upgrade.go

@@ -10,7 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -22,7 +22,7 @@ var appUpgradeCommand = cli.Command{
 	Name:      "upgrade",
 	Aliases:   []string{"up"},
 	Usage:     "Upgrade an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -30,6 +30,7 @@ var appUpgradeCommand = cli.Command{
 		internal.ChaosFlag,
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -46,37 +47,52 @@ useful if the container runtime has gotten into a weird state.
 This action could be destructive, please ensure you have a copy of your app
 data beforehand.
 
-Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
-		conf := runtime.New()
+
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe, conf); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		r, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("checking whether %s is already deployed", stackName)
-
 		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
@@ -86,37 +102,62 @@ recipes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		versions, err := recipePkg.GetRecipeCatalogueVersions(app.Recipe, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipePkg.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
 		}
 
 		var availableUpgrades []string
 		if deployedVersion == "unknown" {
 			availableUpgrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		if deployedVersion != "unknown" && !internal.Chaos {
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsLessThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not an upgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableUpgrades = append(availableUpgrades, specificVersion)
+		}
+
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
 			for _, version := range versions {
-				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
-				if err != nil {
-					logrus.Fatal(err)
-				}
 				parsedVersion, err := tagcmp.Parse(version)
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				if parsedVersion.IsGreaterThan(parsedDeployedVersion) {
+				if parsedVersion.IsGreaterThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
 					availableUpgrades = append(availableUpgrades, version)
 				}
 			}
@@ -127,17 +168,15 @@ recipes.
 			}
 		}
 
-		availableUpgrades = internal.ReverseStringList(availableUpgrades)
-
 		var chosenUpgrade string
 		if len(availableUpgrades) > 0 && !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force || internal.NoInput || specificVersion != "" {
 				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
 				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
 			} else {
 				prompt := &survey.Select{
 					Message: fmt.Sprintf("Please select an upgrade (current version: %s):", deployedVersion),
-					Options: availableUpgrades,
+					Options: internal.ReverseStringList(availableUpgrades),
 				}
 				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
 					return err
@@ -145,16 +184,38 @@ recipes.
 			}
 		}
 
+		if internal.Force && chosenUpgrade == "" {
+			logrus.Warnf("%s is already upgraded to latest but continuing (--force/--chaos)", app.Name)
+			chosenUpgrade = deployedVersion
+		}
+
 		// if release notes written after git tag published, read them before we
 		// check out the tag and then they'll appear to be missing. this covers
 		// when we obviously will forget to write release notes before publishing
-		releaseNotes, err := internal.GetReleaseNotes(app.Recipe, chosenUpgrade)
-		if err != nil {
-			return err
+		var releaseNotes string
+		for _, version := range versions {
+			parsedVersion, err := tagcmp.Parse(version)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !(parsedVersion.Equals(parsedDeployedVersion)) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
+				note, err := internal.GetReleaseNotes(app.Recipe, version)
+				if err != nil {
+					return err
+				}
+				if note != "" {
+					releaseNotes += fmt.Sprintf("%s\n", note)
+				}
+			}
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
+			if err := recipePkg.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -162,7 +223,7 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenUpgrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenUpgrade, err = recipePkg.ChaosVersion(app.Recipe)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -177,7 +238,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -194,17 +255,42 @@ recipes.
 		config.ExposeAllEnv(stackName, compose, app.Env)
 		config.SetRecipeLabel(compose, stackName, app.Recipe)
 		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
 			logrus.Fatal(err)
 		}
+		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, stackName, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_UPGRADE_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+			}
+		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/version.go

@@ -2,19 +2,30 @@ package app
 
 import (
 	"context"
+	"sort"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
+	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
 // getImagePath returns the image name
 func getImagePath(image string) (string, error) {
 	img, err := reference.ParseNormalizedNamed(image)
@@ -38,18 +49,14 @@ var appVersionCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Show app versions",
-	Description: `
-Show all information about versioning related to a deployed app. This includes
-the individual image names, tags and digests. But also the Co-op Cloud recipe
-version.
-`,
+	Before:       internal.SubCommandBefore,
+	Usage:        "Show version info of a deployed app",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
-		conf := runtime.New()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -63,15 +70,15 @@ version.
 			logrus.Fatal(err)
 		}
 
-		if deployedVersion == "unknown" {
-			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
-		}
-
 		if !isDeployed {
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, conf)
+		if deployedVersion == "unknown" {
+			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
+		}
+
+		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -87,17 +94,24 @@ version.
 			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"version", "service", "image"}
+		tableCol := []string{"version", "service", "image", "tag"}
 		table := formatter.CreateTable(tableCol)
-		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
+		var versions [][]string
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{deployedVersion, serviceName, versionMeta.Image})
+			versions = append(versions, []string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Tag})
 		}
 
+		sort.Slice(versions, sortServiceByName(versions))
+
+		for _, version := range versions {
+			table.Append(version)
+		}
+
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+		table.SetAlignment(tablewriter.ALIGN_LEFT)
 		table.Render()
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/volume.go

@@ -7,6 +7,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -81,7 +82,8 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		internal.NoInputFlag,
 		internal.ForceFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -90,6 +92,15 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+		}
+
 		filters, err := app.Filters(false, true)
 		if err != nil {
 			logrus.Fatal(err)
@@ -119,16 +130,19 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 			volumesToRemove = volumeNames
 		}
 
-		err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
-		if err != nil {
-			logrus.Fatal(err)
-		}
+		if len(volumesToRemove) > 0 {
+			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
+			if err != nil {
+				logrus.Fatal(err)
+			}
 
-		logrus.Info("volumes removed successfully")
+			logrus.Info("volumes removed successfully")
+		} else {
+			logrus.Info("no volumes removed")
+		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appVolumeCommand = cli.Command{

cli/catalogue/catalogue.go

@@ -13,7 +13,6 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -29,6 +28,8 @@ var catalogueGenerateCommand = cli.Command{
 		internal.PublishFlag,
 		internal.DryFlag,
 		internal.SkipUpdatesFlag,
+		internal.ChaosFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -52,17 +53,19 @@ Push your new release to git.coopcloud.tech with "-p/--publish". This requires
 that you have permission to git push to these repositories and have your SSH
 keys configured on your account.
 `,
-	ArgsUsage: "[<recipe>]",
+	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
-		conf := runtime.New()
 
 		if recipeName != "" {
 			internal.ValidateRecipe(c)
 		}
 
-		if err := catalogue.EnsureUpToDate(); err != nil {
-			logrus.Fatal(err)
+		if !internal.Chaos {
+			if err := catalogue.EnsureIsClean(); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 		repos, err := recipe.ReadReposMetadata()
@@ -82,7 +85,7 @@ keys configured on your account.
 
 		if !internal.SkipUpdates {
 			logrus.Warn(logMsg)
-			if err := recipe.UpdateRepositories(repos, recipeName, conf); err != nil {
+			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -95,12 +98,7 @@ keys configured on your account.
 				continue
 			}
 
-			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
-			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, conf)
+			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
 			}
@@ -136,7 +134,7 @@ keys configured on your account.
 				logrus.Fatal(err)
 			}
 		} else {
-			catlFS, err := recipe.ReadRecipeCatalogue()
+			catlFS, err := recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -170,7 +168,7 @@ keys configured on your account.
 			}
 
 			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
+			if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil {
 				logrus.Fatal(err)
 			}
 
@@ -210,7 +208,6 @@ keys configured on your account.
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }
 
 // CatalogueCommand defines the `abra catalogue` command and sub-commands.

cli/cli.go

@@ -12,9 +12,9 @@ import (
 	"coopcloud.tech/abra/cli/catalogue"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/cli/recipe"
-	"coopcloud.tech/abra/cli/record"
 	"coopcloud.tech/abra/cli/server"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	cataloguePkg "coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/sirupsen/logrus"
@@ -158,7 +158,6 @@ func newAbraApp(version, commit string) *cli.App {
 			server.ServerCommand,
 			recipe.RecipeCommand,
 			catalogue.CatalogueCommand,
-			record.RecordCommand,
 			UpgradeCommand,
 			AutoCompleteCommand,
 		},
@@ -170,10 +169,10 @@ func newAbraApp(version, commit string) *cli.App {
 	app.Before = func(c *cli.Context) error {
 		paths := []string{
 			config.ABRA_DIR,
-			path.Join(config.SERVERS_DIR),
-			path.Join(config.RECIPES_DIR),
-			path.Join(config.VENDOR_DIR),
-			path.Join(config.BACKUP_DIR),
+			config.SERVERS_DIR,
+			config.RECIPES_DIR,
+			config.VENDOR_DIR,
+			config.BACKUP_DIR,
 		}
 
 		for _, path := range paths {
@@ -185,6 +184,10 @@ func newAbraApp(version, commit string) *cli.App {
 			}
 		}
 
+		if err := cataloguePkg.EnsureCatalogue(); err != nil {
+			logrus.Fatal(err)
+		}
+
 		logrus.Debugf("abra version %s, commit %s", version, commit)
 
 		return nil

cli/internal/cli.go

@@ -54,19 +54,18 @@ var Chaos bool
 // ChaosFlag turns on/off chaos functionality.
 var ChaosFlag = &cli.BoolFlag{
 	Name:        "chaos, C",
-	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Usage:       "Proceed with uncommitted recipes changes. Use with care!",
 	Destination: &Chaos,
 }
 
-// DNSProvider specifies a DNS provider.
-var DNSProvider string
+// Disable tty to run commands from script
+var Tty bool
 
-// DNSProviderFlag selects a DNS provider.
-var DNSProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Value:       "",
-	Usage:       "DNS provider",
-	Destination: &DNSProvider,
+// TtyFlag turns on/off tty mode.
+var TtyFlag = &cli.BoolFlag{
+	Name:        "tty, T",
+	Usage:       "Disables TTY mode to run this command from a script.",
+	Destination: &Tty,
 }
 
 var NoInput bool
@@ -76,163 +75,6 @@ var NoInputFlag = &cli.BoolFlag{
 	Destination: &NoInput,
 }
 
-var DNSType string
-
-var DNSTypeFlag = &cli.StringFlag{
-	Name:        "record-type, rt",
-	Value:       "",
-	Usage:       "Domain name record type (e.g. A)",
-	Destination: &DNSType,
-}
-
-var DNSName string
-
-var DNSNameFlag = &cli.StringFlag{
-	Name:        "record-name, rn",
-	Value:       "",
-	Usage:       "Domain name record name (e.g. mysubdomain)",
-	Destination: &DNSName,
-}
-
-var DNSValue string
-
-var DNSValueFlag = &cli.StringFlag{
-	Name:        "record-value, rv",
-	Value:       "",
-	Usage:       "Domain name record value (e.g. 192.168.1.1)",
-	Destination: &DNSValue,
-}
-
-var DNSTTL string
-var DNSTTLFlag = &cli.StringFlag{
-	Name:        "record-ttl, rl",
-	Value:       "600s",
-	Usage:       "Domain name TTL value (seconds)",
-	Destination: &DNSTTL,
-}
-
-var DNSPriority int
-
-var DNSPriorityFlag = &cli.IntFlag{
-	Name:        "record-priority, rp",
-	Value:       10,
-	Usage:       "Domain name priority value",
-	Destination: &DNSPriority,
-}
-
-var ServerProvider string
-
-var ServerProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Usage:       "3rd party server provider",
-	Destination: &ServerProvider,
-}
-
-var CapsulInstanceURL string
-
-var CapsulInstanceURLFlag = &cli.StringFlag{
-	Name:        "capsul-url, cu",
-	Value:       "yolo.servers.coop",
-	Usage:       "capsul instance URL",
-	Destination: &CapsulInstanceURL,
-}
-
-var CapsulName string
-
-var CapsulNameFlag = &cli.StringFlag{
-	Name:        "capsul-name, cn",
-	Value:       "",
-	Usage:       "capsul name",
-	Destination: &CapsulName,
-}
-
-var CapsulType string
-
-var CapsulTypeFlag = &cli.StringFlag{
-	Name:        "capsul-type, ct",
-	Value:       "f1-xs",
-	Usage:       "capsul type",
-	Destination: &CapsulType,
-}
-
-var CapsulImage string
-
-var CapsulImageFlag = &cli.StringFlag{
-	Name:        "capsul-image, ci",
-	Value:       "debian10",
-	Usage:       "capsul image",
-	Destination: &CapsulImage,
-}
-
-var CapsulSSHKeys cli.StringSlice
-var CapsulSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "capsul-ssh-keys, cs",
-	Usage: "capsul SSH key",
-	Value: &CapsulSSHKeys,
-}
-
-var CapsulAPIToken string
-
-var CapsulAPITokenFlag = &cli.StringFlag{
-	Name:        "capsul-token, ca",
-	Usage:       "capsul API token",
-	EnvVar:      "CAPSUL_TOKEN",
-	Destination: &CapsulAPIToken,
-}
-
-var HetznerCloudName string
-
-var HetznerCloudNameFlag = &cli.StringFlag{
-	Name:        "hetzner-name, hn",
-	Value:       "",
-	Usage:       "hetzner cloud name",
-	Destination: &HetznerCloudName,
-}
-
-var HetznerCloudType string
-
-var HetznerCloudTypeFlag = &cli.StringFlag{
-	Name:        "hetzner-type, ht",
-	Usage:       "hetzner cloud type",
-	Destination: &HetznerCloudType,
-	Value:       "cx11",
-}
-
-var HetznerCloudImage string
-
-var HetznerCloudImageFlag = &cli.StringFlag{
-	Name:        "hetzner-image, hi",
-	Usage:       "hetzner cloud image",
-	Value:       "debian-10",
-	Destination: &HetznerCloudImage,
-}
-
-var HetznerCloudSSHKeys cli.StringSlice
-
-var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "hetzner-ssh-keys, hs",
-	Usage: "hetzner cloud SSH keys (e.g. me@foo.com)",
-	Value: &HetznerCloudSSHKeys,
-}
-
-var HetznerCloudLocation string
-
-var HetznerCloudLocationFlag = &cli.StringFlag{
-	Name:        "hetzner-location, hl",
-	Usage:       "hetzner cloud server location",
-	Value:       "hel1",
-	Destination: &HetznerCloudLocation,
-}
-
-var HetznerCloudAPIToken string
-
-var HetznerCloudAPITokenFlag = &cli.StringFlag{
-	Name:        "hetzner-token, ha",
-	Usage:       "hetzner cloud API token",
-	EnvVar:      "HCLOUD_TOKEN",
-	Destination: &HetznerCloudAPIToken,
-}
-
 // Debug stores the variable from DebugFlag.
 var Debug bool
 
@@ -243,6 +85,16 @@ var DebugFlag = &cli.BoolFlag{
 	Usage:       "Show DEBUG messages",
 }
 
+// Offline stores the variable from OfflineFlag.
+var Offline bool
+
+// DebugFlag turns on/off offline mode.
+var OfflineFlag = &cli.BoolFlag{
+	Name:        "offline, o",
+	Destination: &Offline,
+	Usage:       "Prefer offline & filesystem access when possible",
+}
+
 // MachineReadable stores the variable from MachineReadableFlag
 var MachineReadable bool
 
@@ -260,7 +112,7 @@ var RC bool
 var RCFlag = &cli.BoolFlag{
 	Name:        "rc, r",
 	Destination: &RC,
-	Usage:       "Insatll the latest release candidate",
+	Usage:       "Install the latest release candidate",
 }
 
 var Major bool

cli/internal/command.go

@@ -2,10 +2,109 @@ package internal
 
 import (
 	"bufio"
+	"context"
 	"fmt"
+	"io/ioutil"
 	"os/exec"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
 )
 
+// RunCmdRemote executes an abra.sh command in the target service
+func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
+
+	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(abraSh, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
+		return err
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	shell := "/bin/bash"
+	findShell := []string{"test", "-e", shell}
+	execCreateOpts := types.ExecConfig{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          findShell,
+		Detach:       false,
+		Tty:          false,
+	}
+
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
+		shell = "/bin/sh"
+	}
+
+	var cmd []string
+	if cmdArgs != "" {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.Name, app.StackName(), cmdName, cmdArgs)}
+	} else {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.Name, app.StackName(), cmdName)}
+	}
+
+	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
+
+	if RemoteUser != "" {
+		logrus.Debugf("running command with user %s", RemoteUser)
+		execCreateOpts.User = RemoteUser
+	}
+
+	execCreateOpts.Cmd = cmd
+	execCreateOpts.Tty = true
+	if Tty {
+		execCreateOpts.Tty = false
+	}
+
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func EnsureCommand(abraSh, recipeName, execCmd string) error {
+	bytes, err := ioutil.ReadFile(abraSh)
+	if err != nil {
+		return err
+	}
+
+	if !strings.Contains(string(bytes), execCmd) {
+		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
+	}
+
+	return nil
+}
+
 // RunCmd runs a shell command and streams stdout/stderr in real-time.
 func RunCmd(cmd *exec.Cmd) error {
 	r, err := cmd.StdoutPipe()

cli/internal/deploy.go

@@ -1,167 +1,136 @@
 package internal
 
 import (
-	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/lint"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
-// DeployAction is the main command-line action for this package
-func DeployAction(c *cli.Context) error {
-	app := ValidateApp(c)
-	conf := runtime.New()
+// NewVersionOverview shows an upgrade or downgrade overview
+func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
+	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
+	table := formatter.CreateTable(tableCol)
 
-	cl, err := client.New(app.Server)
-	if err != nil {
-		logrus.Fatal(err)
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
 	}
 
-	if !Chaos {
-		if err := recipe.EnsureUpToDate(app.Recipe, conf); err != nil {
-			logrus.Fatal(err)
-		}
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
 	}
 
-	r, err := recipe.Get(app.Recipe, conf)
-	if err != nil {
-		logrus.Fatal(err)
-	}
+	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
+	table.Render()
 
-	if err := lint.LintForErrors(r); err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("checking whether %s is already deployed", app.StackName())
-
-	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if isDeployed {
-		if Force || Chaos {
-			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
-		} else {
-			logrus.Fatalf("%s is already deployed", app.Name)
-		}
-	}
-
-	version := deployedVersion
-	if version == "unknown" && !Chaos {
-		catl, err := recipe.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if len(versions) > 0 {
-			version = versions[len(versions)-1]
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			head, err := git.GetRecipeHead(app.Recipe)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			version = formatter.SmallSHA(head.String())
-			logrus.Warn("no versions detected, using latest commit")
-			if err := recipe.EnsureLatest(app.Recipe, conf); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-	}
-
-	if version == "unknown" && !Chaos {
-		logrus.Debugf("choosing %s as version to deploy", version)
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	if version != "unknown" && !Chaos {
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	if Chaos {
-		logrus.Warnf("chaos mode engaged")
-		var err error
-		version, err = recipe.ChaosVersion(app.Recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	for k, v := range abraShEnv {
-		app.Env[k] = v
-	}
-
-	composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	deployOpts := stack.Deploy{
-		Composefiles: composeFiles,
-		Namespace:    app.StackName(),
-		Prune:        false,
-		ResolveImage: stack.ResolveImageAlways,
-	}
-	compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	config.ExposeAllEnv(app.StackName(), compose, app.Env)
-	config.SetRecipeLabel(compose, app.StackName(), app.Recipe)
-	config.SetChaosLabel(compose, app.StackName(), Chaos)
-	config.SetUpdateLabel(compose, app.StackName(), app.Env)
-
-	if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if !NoDomainChecks {
-		domainName, ok := app.Env["DOMAIN"]
-		if ok {
-			if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
-		}
+	if releaseNotes != "" && newVersion != "" {
+		fmt.Println()
+		fmt.Print(releaseNotes)
 	} else {
-		logrus.Warn("skipping domain checks as requested")
+		logrus.Warnf("no release notes available for %s", newVersion)
 	}
 
-	if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, DontWaitConverge); err != nil {
-		logrus.Fatal(err)
+	if NoInput {
+		return nil
 	}
 
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with deployment?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}
+
+// GetReleaseNotes prints release notes for a recipe version
+func GetReleaseNotes(recipeName, version string) (string, error) {
+	if version == "" {
+		return "", nil
+	}
+
+	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
+
+	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
+		releaseNotes, err := ioutil.ReadFile(fpath)
+		if err != nil {
+			return "", err
+		}
+		withTitle := fmt.Sprintf("%s release notes:\n%s", version, string(releaseNotes))
+		return withTitle, nil
+	}
+
+	return "", nil
+}
+
+// PostCmds parses a string of commands and executes them inside of the respective services
+// the commands string must have the following format:
+// "<service> <command> <arguments>|<service> <command> <arguments>|... "
+func PostCmds(cl *dockerClient.Client, app config.App, commands string) error {
+	abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
+	if _, err := os.Stat(abraSh); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf(fmt.Sprintf("%s does not exist for %s?", abraSh, app.Name))
+		}
+		return err
+	}
+
+	for _, command := range strings.Split(commands, "|") {
+		commandParts := strings.Split(command, " ")
+		if len(commandParts) < 2 {
+			return fmt.Errorf(fmt.Sprintf("not enough arguments: %s", command))
+		}
+		targetServiceName := commandParts[0]
+		cmdName := commandParts[1]
+		parsedCmdArgs := ""
+		if len(commandParts) > 2 {
+			parsedCmdArgs = fmt.Sprintf("%s ", strings.Join(commandParts[2:], " "))
+		}
+		logrus.Infof("running post-command '%s %s' in container %s", cmdName, parsedCmdArgs, targetServiceName)
+
+		if err := EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			return err
+		}
+
+		serviceNames, err := config.GetAppServiceNames(app.Name)
+		if err != nil {
+			return err
+		}
+
+		matchingServiceName := false
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
+			}
+		}
+
+		if !matchingServiceName {
+			return fmt.Errorf(fmt.Sprintf("no service %s for %s?", targetServiceName, app.Name))
+		}
+
+		logrus.Debugf("running command %s %s within the context of %s_%s", cmdName, parsedCmdArgs, app.StackName(), targetServiceName)
+
+		Tty = true
+		if err := RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -202,75 +171,3 @@ func DeployOverview(app config.App, version, message string) error {
 
 	return nil
 }
-
-// NewVersionOverview shows an upgrade or downgrade overview
-func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
-	table := formatter.CreateTable(tableCol)
-
-	deployConfig := "compose.yml"
-	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
-		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
-	}
-
-	server := app.Server
-	if app.Server == "default" {
-		server = "local"
-	}
-
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
-	table.Render()
-
-	if releaseNotes == "" {
-		var err error
-		releaseNotes, err = GetReleaseNotes(app.Recipe, newVersion)
-		if err != nil {
-			return err
-		}
-	}
-
-	if releaseNotes != "" && newVersion != "" {
-		fmt.Println()
-		fmt.Println(fmt.Sprintf("%s release notes:\n\n%s", newVersion, releaseNotes))
-	} else {
-		logrus.Warnf("no release notes available for %s", newVersion)
-	}
-
-	if NoInput {
-		return nil
-	}
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with deployment?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	return nil
-}
-
-// GetReleaseNotes prints release notes for a recipe version
-func GetReleaseNotes(recipeName, version string) (string, error) {
-	if version == "" {
-		return "", nil
-	}
-
-	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
-
-	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
-		releaseNotes, err := ioutil.ReadFile(fpath)
-		if err != nil {
-			return "", err
-		}
-		return string(releaseNotes), nil
-	}
-
-	return "", nil
-}

cli/internal/new.go

@@ -1,199 +0,0 @@
-package internal
-
-import (
-	"fmt"
-	"path"
-
-	"coopcloud.tech/abra/pkg/app"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/jsontable"
-	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/secret"
-	"github.com/AlecAivazis/survey/v2"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// AppSecrets represents all app secrest
-type AppSecrets map[string]string
-
-// RecipeName is used for configuring recipe name programmatically
-var RecipeName string
-
-// createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets, error) {
-	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", Domain))
-	appEnv, err := config.ReadEnv(appEnvPath)
-	if err != nil {
-		return nil, err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
-	secrets, err := secret.GenerateSecrets(cl, secretEnvVars, sanitisedAppName, NewAppServer)
-	if err != nil {
-		return nil, err
-	}
-
-	if Pass {
-		for secretName := range secrets {
-			secretValue := secrets[secretName]
-			if err := secret.PassInsertSecret(secretValue, secretName, Domain, NewAppServer); err != nil {
-				return nil, err
-			}
-		}
-	}
-	return secrets, nil
-}
-
-// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
-func ensureDomainFlag(recipe recipe.Recipe, server string) error {
-	if Domain == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify app domain",
-			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
-		}
-		if err := survey.AskOne(prompt, &Domain); err != nil {
-			return err
-		}
-	}
-
-	if Domain == "" {
-		return fmt.Errorf("no domain provided")
-	}
-
-	return nil
-}
-
-// promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(appName string) error {
-	app, err := app.Get(appName)
-	if err != nil {
-		return err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(app.Env)
-	if len(secretEnvVars) == 0 {
-		logrus.Debugf("%s has no secrets to generate, skipping...", app.Recipe)
-		return nil
-	}
-
-	if !Secrets && !NoInput {
-		prompt := &survey.Confirm{
-			Message: "Generate app secrets?",
-		}
-		if err := survey.AskOne(prompt, &Secrets); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
-func ensureServerFlag() error {
-	servers, err := config.GetServers()
-	if err != nil {
-		return err
-	}
-
-	if NewAppServer == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select app server:",
-			Options: servers,
-		}
-		if err := survey.AskOne(prompt, &NewAppServer); err != nil {
-			return err
-		}
-	}
-
-	if NewAppServer == "" {
-		return fmt.Errorf("no server provided")
-	}
-
-	return nil
-}
-
-// NewAction is the new app creation logic
-func NewAction(c *cli.Context) error {
-	recipe := ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
-
-	conf := runtime.New(runtime.WithEnsureRecipeLatest(false))
-	if err := recipePkg.EnsureUpToDate(recipe.Name, conf); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ensureServerFlag(); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ensureDomainFlag(recipe, NewAppServer); err != nil {
-		logrus.Fatal(err)
-	}
-
-	sanitisedAppName := config.SanitiseAppName(Domain)
-	logrus.Debugf("%s sanitised as %s for new app", Domain, sanitisedAppName)
-
-	if err := config.TemplateAppEnvSample(recipe.Name, Domain, NewAppServer, Domain); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := promptForSecrets(Domain); err != nil {
-		logrus.Fatal(err)
-	}
-
-	cl, err := client.New(NewAppServer)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	var secrets AppSecrets
-	var secretTable *jsontable.JSONTable
-	if Secrets {
-		secrets, err := createSecrets(cl, sanitisedAppName)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretCols := []string{"Name", "Value"}
-		secretTable = formatter.CreateTable(secretCols)
-		for secret := range secrets {
-			secretTable.Append([]string{secret, secrets[secret]})
-		}
-
-	}
-
-	if NewAppServer == "default" {
-		NewAppServer = "local"
-	}
-
-	tableCol := []string{"server", "recipe", "domain"}
-	table := formatter.CreateTable(tableCol)
-	table.Append([]string{NewAppServer, recipe.Name, Domain})
-
-	fmt.Println("")
-	fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
-	fmt.Println("")
-	table.Render()
-	fmt.Println("")
-	fmt.Println("You can configure this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app config %s", Domain))
-	fmt.Println("")
-	fmt.Println("You can deploy this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", Domain))
-	fmt.Println("")
-
-	if len(secrets) > 0 {
-		fmt.Println("Here are your generated secrets:")
-		fmt.Println("")
-		secretTable.Render()
-		fmt.Println("")
-		logrus.Warn("generated secrets are not shown again, please take note of them *now*")
-	}
-
-	return nil
-}

cli/internal/validate.go

@@ -2,65 +2,24 @@ package internal
 
 import (
 	"errors"
-	"fmt"
-	"os"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-// AppName is used for configuring app name programmatically
-var AppName string
-
 // ValidateRecipe ensures the recipe arg is valid.
-func ValidateRecipe(c *cli.Context, opts ...runtime.Option) recipe.Recipe {
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
-	conf := runtime.New(opts...)
-
-	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
-	}
-
-	chosenRecipe, err := recipe.Get(recipeName, conf)
-	if err != nil {
-		if c.Command.Name == "generate" {
-			if strings.Contains(err.Error(), "missing a compose") {
-				logrus.Fatal(err)
-			}
-			logrus.Warn(err)
-		} else {
-			if strings.Contains(err.Error(), "template_driver is not allowed") {
-				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
-			}
-			logrus.Fatalf("unable to validate recipe: %s", err)
-		}
-	}
-
-	if err := recipe.EnsureLatest(recipeName, conf); err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("validated %s as recipe argument", recipeName)
-
-	return chosenRecipe
-}
-
-// ValidateRecipeWithPrompt ensures a recipe argument is present before
-// validating, asking for input if required.
-func ValidateRecipeWithPrompt(c *cli.Context, opts ...runtime.Option) recipe.Recipe {
-	recipeName := c.Args().First()
-	conf := runtime.New(opts...)
 
 	if recipeName == "" && !NoInput {
 		var recipes []string
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipe.ReadRecipeCatalogue(Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -94,22 +53,23 @@ func ValidateRecipeWithPrompt(c *cli.Context, opts ...runtime.Option) recipe.Rec
 		}
 	}
 
-	if RecipeName != "" {
-		recipeName = RecipeName
-		logrus.Debugf("programmatically setting recipe name to %s", recipeName)
-	}
-
 	if recipeName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName, conf)
+	chosenRecipe, err := recipe.Get(recipeName, Offline)
 	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := recipe.EnsureLatest(recipeName, conf); err != nil {
-		logrus.Fatal(err)
+		if c.Command.Name == "generate" {
+			if strings.Contains(err.Error(), "missing a compose") {
+				logrus.Fatal(err)
+			}
+			logrus.Warn(err)
+		} else {
+			if strings.Contains(err.Error(), "template_driver is not allowed") {
+				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
+			}
+			logrus.Fatalf("unable to validate recipe: %s", err)
+		}
 	}
 
 	logrus.Debugf("validated %s as recipe argument", recipeName)
@@ -118,14 +78,8 @@ func ValidateRecipeWithPrompt(c *cli.Context, opts ...runtime.Option) recipe.Rec
 }
 
 // ValidateApp ensures the app name arg is valid.
-func ValidateApp(c *cli.Context, opts ...runtime.Option) config.App {
+func ValidateApp(c *cli.Context) config.App {
 	appName := c.Args().First()
-	conf := runtime.New(opts...)
-
-	if AppName != "" {
-		appName = AppName
-		logrus.Debugf("programmatically setting app name to %s", appName)
-	}
 
 	if appName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
@@ -136,10 +90,6 @@ func ValidateApp(c *cli.Context, opts ...runtime.Option) config.App {
 		logrus.Fatal(err)
 	}
 
-	if err := recipe.EnsureExists(app.Recipe, conf); err != nil {
-		logrus.Fatal(err)
-	}
-
 	logrus.Debugf("validated %s as app argument", appName)
 
 	return app
@@ -208,309 +158,15 @@ func ValidateServer(c *cli.Context) string {
 		}
 	}
 
-	if !matched {
-		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
-	}
-
 	if serverName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
 	}
 
+	if !matched {
+		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
+	}
+
 	logrus.Debugf("validated %s as server argument", serverName)
 
 	return serverName
 }
-
-// EnsureDNSProvider ensures a DNS provider is chosen.
-func EnsureDNSProvider() error {
-	if DNSProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select DNS provider",
-			Options: []string{"gandi"},
-		}
-
-		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
-			return err
-		}
-	}
-
-	if DNSProvider == "" {
-		return fmt.Errorf("missing DNS provider?")
-	}
-
-	return nil
-}
-
-// EnsureDNSTypeFlag ensures a DNS type flag is present.
-func EnsureDNSTypeFlag(c *cli.Context) error {
-	if DNSType == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record type",
-			Default: "A",
-		}
-		if err := survey.AskOne(prompt, &DNSType); err != nil {
-			return err
-		}
-	}
-
-	if DNSType == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSNameFlag ensures a DNS name flag is present.
-func EnsureDNSNameFlag(c *cli.Context) error {
-	if DNSName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record name",
-			Default: "mysubdomain",
-		}
-		if err := survey.AskOne(prompt, &DNSName); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSValueFlag ensures a DNS value flag is present.
-func EnsureDNSValueFlag(c *cli.Context) error {
-	if DNSValue == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record value",
-			Default: "192.168.1.2",
-		}
-		if err := survey.AskOne(prompt, &DNSValue); err != nil {
-			return err
-		}
-	}
-
-	if DNSValue == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
-	}
-
-	return nil
-}
-
-// EnsureZoneArgument ensures a zone argument is present.
-func EnsureZoneArgument(c *cli.Context) (string, error) {
-	zone := c.Args().First()
-
-	if zone == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name zone",
-			Default: "example.com",
-		}
-
-		if err := survey.AskOne(prompt, &zone); err != nil {
-			return zone, err
-		}
-	}
-
-	if zone == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
-	}
-
-	return zone, nil
-}
-
-// EnsureServerProvider ensures a 3rd party server provider is chosen.
-func EnsureServerProvider() error {
-	if ServerProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select server provider",
-			Options: []string{"capsul", "hetzner-cloud"},
-		}
-
-		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
-			return err
-		}
-	}
-
-	if ServerProvider == "" {
-		return fmt.Errorf("missing server provider?")
-	}
-
-	return nil
-}
-
-// EnsureNewCapsulVPSFlags ensure all flags are present.
-func EnsureNewCapsulVPSFlags(c *cli.Context) error {
-	if CapsulName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul name",
-		}
-		if err := survey.AskOne(prompt, &CapsulName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul instance URL",
-			Default: CapsulInstanceURL,
-		}
-		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul type",
-			Default: CapsulType,
-		}
-		if err := survey.AskOne(prompt, &CapsulType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul image",
-			Default: CapsulImage,
-		}
-		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
-			return err
-		}
-	}
-
-	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify capsul SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
-			return err
-		}
-		CapsulSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if CapsulAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("CAPSUL_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify capsul API token",
-			}
-			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
-				return err
-			}
-		} else {
-			CapsulAPIToken = token
-		}
-	}
-
-	if CapsulName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
-	}
-	if CapsulInstanceURL == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
-	}
-	if CapsulType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
-	}
-	if CapsulImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
-	}
-	if len(CapsulSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
-	}
-	if CapsulAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
-	}
-
-	return nil
-}
-
-// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
-func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
-	if HetznerCloudName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS type",
-			Default: HetznerCloudType,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS image",
-			Default: HetznerCloudImage,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
-			return err
-		}
-	}
-
-	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &sshKeys); err != nil {
-			return err
-		}
-		HetznerCloudSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS location",
-			Default: HetznerCloudLocation,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
-			return err
-		}
-	}
-
-	if HetznerCloudAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			HetznerCloudAPIToken = token
-		}
-	}
-
-	if HetznerCloudName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
-	}
-	if HetznerCloudType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
-	}
-	if HetznerCloudImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
-	}
-	if HetznerCloudLocation == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
-	}
-	if HetznerCloudAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
-	}
-
-	return nil
-}

cli/recipe/diff.go

@@ -0,0 +1,40 @@
+package recipe
+
+import (
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeDiffCommand = cli.Command{
+	Name:        "diff",
+	Usage:       "Show unstaged changes in recipe config",
+	Description: "Due to limitations in our underlying Git dependency, this command requires /usr/bin/git.",
+	Aliases:     []string{"d"},
+	ArgsUsage:   "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+		if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/fetch.go

@@ -4,37 +4,38 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
 var recipeFetchCommand = cli.Command{
 	Name:        "fetch",
-	Usage:       "Fetch recipe local copies",
+	Usage:       "Fetch recipe(s)",
 	Aliases:     []string{"f"},
 	ArgsUsage:   "[<recipe>]",
-	Description: "Fetchs all recipes without arguments.",
+	Description: "Retrieves all recipes if no <recipe> argument is passed",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.NoInputFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
-		conf := runtime.New()
 
 		if recipeName != "" {
 			internal.ValidateRecipe(c)
-			return nil // ValidateRecipe ensures latest checkout
 		}
 
-		repos, err := recipe.ReadReposMetadata()
-		if err != nil {
+		if err := recipe.EnsureExists(recipeName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := recipe.UpdateRepositories(repos, recipeName, conf); err != nil {
+		if err := recipe.EnsureUpToDate(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.EnsureLatest(recipeName); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/recipe/lint.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/lint"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -21,17 +20,35 @@ var recipeLintCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OnlyErrorFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
-		conf := runtime.New()
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name, conf); err != nil {
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		tableCol := []string{"ref", "rule", "severity", "satisfied", "skipped", "resolve"}
 		table := formatter.CreateTable(tableCol)
 
@@ -51,7 +68,7 @@ var recipeLintCommand = cli.Command{
 
 				skippedOutput := "-"
 				if skipped {
-					skippedOutput = "yes"
+					skippedOutput = "✅"
 				}
 
 				satisfied := false
@@ -70,9 +87,9 @@ var recipeLintCommand = cli.Command{
 					}
 				}
 
-				satisfiedOutput := "yes"
+				satisfiedOutput := "✅"
 				if !satisfied {
-					satisfiedOutput = "NO"
+					satisfiedOutput = "❌"
 					if skipped {
 						satisfiedOutput = "-"
 					}

cli/recipe/list.go

@@ -29,10 +29,11 @@ var recipeListCommand = cli.Command{
 		internal.DebugFlag,
 		internal.MachineReadableFlag,
 		patternFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err.Error())
 		}

cli/recipe/new.go

@@ -36,6 +36,7 @@ var recipeNewCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
 	Before:    internal.SubCommandBefore,
 	Usage:     "Create a new recipe",

cli/recipe/recipe.go

@@ -30,5 +30,7 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 		recipeSyncCommand,
 		recipeUpgradeCommand,
 		recipeVersionCommand,
+		recipeResetCommand,
+		recipeDiffCommand,
 	},
 }

cli/recipe/release.go

@@ -13,7 +13,6 @@ import (
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
@@ -55,11 +54,12 @@ your SSH keys configured on your account.
 		internal.MinorFlag,
 		internal.PatchFlag,
 		internal.PublishFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
+		recipe := internal.ValidateRecipe(c)
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
@@ -106,6 +106,18 @@ your SSH keys configured on your account.
 			}
 		}
 
+		isClean, err := gitPkg.IsClean(recipe.Dir())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if len(tags) > 0 {
 			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
 			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
@@ -244,7 +256,7 @@ func commitRelease(recipe recipe.Recipe, tag string) error {
 
 	msg := fmt.Sprintf("chore: publish %s release", tag)
 	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
+	if err := gitPkg.Commit(repoPath, msg, internal.Dry); err != nil {
 		return err
 	}
 

cli/recipe/reset.go

@@ -0,0 +1,56 @@
+package recipe
+
+import (
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeResetCommand = cli.Command{
+	Name:        "reset",
+	Usage:       "Remove all unstaged changes from recipe config",
+	Description: "WARNING, this will delete your changes. Be Careful.",
+	Aliases:     []string{"rs"},
+	ArgsUsage:   "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		repoPath := path.Join(config.RECIPES_DIR, recipeName)
+		repo, err := git.PlainOpen(repoPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		ref, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		worktree, err := repo.Worktree()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}
+		if err := worktree.Reset(opts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/sync.go

@@ -8,7 +8,7 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -41,8 +41,9 @@ Where <version> can be specifed on the command-line or Abra can attempt to
 auto-generate it for you. The <recipe> configuration will be updated on the
 local file system.
 `,
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
+		recipe := internal.ValidateRecipe(c)
 
 		mainApp, err := internal.GetMainAppImage(recipe)
 		if err != nil {
@@ -64,6 +65,9 @@ local file system.
 		nextTag := c.Args().Get(1)
 		if len(tags) == 0 && nextTag == "" {
 			logrus.Warnf("no git tags found for %s", recipe.Name)
+			if internal.NoInput {
+				logrus.Fatalf("unable to continue, input required for initial version")
+			}
 			fmt.Println(fmt.Sprintf(`
 The following options are two types of initial semantic version that you can
 pick for %s that will be published in the recipe catalogue. This follows the
@@ -118,6 +122,7 @@ likely to change.
 			if err := iter.ForEach(func(ref *plumbing.Reference) error {
 				obj, err := repo.TagObject(ref.Hash())
 				if err != nil {
+					logrus.Fatal("Tag at commit ", ref.Hash(), " is unannotated or otherwise broken. Please fix it.")
 					return err
 				}
 
@@ -194,7 +199,17 @@ likely to change.
 			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
+		isClean, err := gitPkg.IsClean(recipe.Dir())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }

cli/recipe/upgrade.go

@@ -2,6 +2,7 @@ package recipe
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -13,8 +14,8 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
@@ -27,6 +28,15 @@ type imgPin struct {
 	version tagcmp.Tag
 }
 
+// anUpgrade represents a single service upgrade (as within a recipe), and the list of tags that it can be upgraded to,
+// for serialization purposes.
+type anUpgrade struct {
+	Service     string   `json:"service"`
+	Image       string   `json:"image"`
+	Tag         string   `json:"tag"`
+	UpgradeTags []string `json:"upgrades"`
+}
+
 var recipeUpgradeCommand = cli.Command{
 	Name:    "upgrade",
 	Aliases: []string{"u"},
@@ -48,22 +58,34 @@ You may invoke this command in "wizard" mode and be prompted for input:
 
     abra recipe upgrade
 `,
-	BashComplete: autocomplete.RecipeNameComplete,
-	ArgsUsage:    "<recipe>",
+	ArgsUsage: "<recipe>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
 		internal.PatchFlag,
 		internal.MinorFlag,
 		internal.MajorFlag,
+		internal.MachineReadableFlag,
 		internal.AllTagsFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c)
-		conf := runtime.New()
+		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name, conf); err != nil {
+		if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -75,6 +97,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
 			}
 		}
 
+		if internal.MachineReadable {
+			// -m implies -n in this case
+			internal.NoInput = true
+		}
+
+		upgradeList := make(map[string]anUpgrade)
+
 		// check for versions file and load pinned versions
 		versionsPresent := false
 		recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
@@ -166,7 +195,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				continue // skip on to the next tag and don't update any compose files
 			}
 
-			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name)
+			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -241,14 +270,37 @@ You may invoke this command in "wizard" mode and be prompted for input:
 						}
 					}
 
-					prompt := &survey.Select{
-						Message: msg,
-						Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
-						VimMode: true,
-						Options: compatibleStrings,
+					// there is always at least the item "skip" in compatibleStrings (a list of
+					// possible upgradable tags) and at least one other tag.
+					upgradableTags := compatibleStrings[1:]
+					upgrade := anUpgrade{
+						Service:     service.Name,
+						Image:       image,
+						Tag:         tag.String(),
+						UpgradeTags: make([]string, len(upgradableTags)),
 					}
-					if err := survey.AskOne(prompt, &upgradeTag); err != nil {
-						logrus.Fatal(err)
+
+					for n, s := range upgradableTags {
+						var sb strings.Builder
+						if _, err := sb.WriteString(s); err != nil {
+						}
+						upgrade.UpgradeTags[n] = sb.String()
+					}
+
+					upgradeList[upgrade.Service] = upgrade
+
+					if internal.NoInput {
+						upgradeTag = "skip"
+					} else {
+						prompt := &survey.Select{
+							Message: msg,
+							Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
+							VimMode: true,
+							Options: compatibleStrings,
+						}
+						if err := survey.AskOne(prompt, &upgradeTag); err != nil {
+							logrus.Fatal(err)
+						}
 					}
 				}
 			}
@@ -261,7 +313,40 @@ You may invoke this command in "wizard" mode and be prompted for input:
 					logrus.Infof("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)
 				}
 			} else {
-				logrus.Warnf("not upgrading %s, skipping as requested", image)
+				if !internal.NoInput {
+					logrus.Warnf("not upgrading %s, skipping as requested", image)
+				}
+			}
+		}
+
+		if internal.NoInput {
+			if internal.MachineReadable {
+				jsonstring, err := json.Marshal(upgradeList)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				fmt.Println(string(jsonstring))
+
+				return nil
+			}
+
+			for _, upgrade := range upgradeList {
+				logrus.Infof("can upgrade service: %s, image: %s, tag: %s ::\n", upgrade.Service, upgrade.Image, upgrade.Tag)
+				for _, utag := range upgrade.UpgradeTags {
+					logrus.Infof("    %s\n", utag)
+				}
+			}
+		}
+
+		isClean, err := gitPkg.IsClean(recipeDir)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
+				logrus.Fatal(err)
 			}
 		}
 

cli/recipe/version.go

@@ -1,57 +1,85 @@
 package recipe
 
 import (
+	"fmt"
+	"sort"
+
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/formatter"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
 var recipeVersionCommand = cli.Command{
-	Name:      "versions",
-	Aliases:   []string{"v"},
-	Usage:     "List recipe versions",
-	ArgsUsage: "<recipe>",
+	Name:        "versions",
+	Aliases:     []string{"v"},
+	Usage:       "List recipe versions",
+	ArgsUsage:   "<recipe>",
+	Description: "Versions are read from the recipe catalogue.",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.MachineReadableFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, runtime.WithEnsureRecipeLatest(false))
+		recipe := internal.ValidateRecipe(c)
 
-		catalogue, err := recipePkg.ReadRecipeCatalogue()
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		recipeMeta, ok := catalogue[recipe.Name]
+		recipeMeta, ok := catl[recipe.Name]
 		if !ok {
-			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
+			logrus.Fatalf("%s is not published on the catalogue?", recipe.Name)
 		}
 
-		tableCol := []string{"Version", "Service", "Image", "Tag"}
-		table := formatter.CreateTable(tableCol)
+		if len(recipeMeta.Versions) == 0 {
+			logrus.Fatalf("%s has no catalogue published versions?", recipe.Name)
+		}
 
 		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
-			for tag, meta := range recipeMeta.Versions[i] {
+			tableCols := []string{"version", "service", "image", "tag"}
+			table := formatter.CreateTable(tableCols)
+			for version, meta := range recipeMeta.Versions[i] {
+				var versions [][]string
 				for service, serviceMeta := range meta {
-					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag})
+					versions = append(versions, []string{version, service, serviceMeta.Image, serviceMeta.Tag})
+				}
+
+				sort.Slice(versions, sortServiceByName(versions))
+
+				for _, version := range versions {
+					table.Append(version)
+				}
+
+				if internal.MachineReadable {
+					table.JSONRender()
+				} else {
+					table.SetAutoMergeCellsByColumnIndex([]int{0})
+					table.SetAlignment(tablewriter.ALIGN_LEFT)
+					table.Render()
+					fmt.Println()
 				}
 			}
 		}
 
-		table.SetAutoMergeCells(true)
-
-		if table.NumLines() > 0 {
-			table.Render()
-		} else {
-			logrus.Fatalf("%s has no published versions?", recipe.Name)
-		}
-
 		return nil
 	},
 }

cli/record/list.go

@@ -1,82 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordListCommand lists domains.
-var RecordListCommand = cli.Command{
-	Name:      "list",
-	Usage:     "List domain name records",
-	Aliases:   []string{"ls"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.DNSProviderFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-List all domain name records managed by a 3rd party provider for a specific
-zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-`,
-	Action: func(c *cli.Context) error {
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		for _, record := range records {
-			value := record.Value
-			if len(record.Value) > 30 {
-				value = fmt.Sprintf("%s...", record.Value[:30])
-			}
-
-			table.Append([]string{
-				record.Type,
-				record.Name,
-				value,
-				record.TTL.String(),
-				strconv.Itoa(record.Priority),
-			})
-		}
-
-		table.Render()
-
-		return nil
-	},
-}

cli/record/new.go

@@ -1,148 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/dns"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordNewCommand creates a new domain name record.
-var RecordNewCommand = cli.Command{
-	Name:      "new",
-	Usage:     "Create a new domain record",
-	Aliases:   []string{"n"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-		internal.DNSValueFlag,
-		internal.DNSTTLFlag,
-		internal.DNSPriorityFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Create a new domain name record for a specific zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-
-Example:
-
-    abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSValueFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		ttl, err := dns.GetTTL(internal.DNSTTL)
-		if err != nil {
-			return err
-		}
-
-		record := libdns.Record{
-			Type:  internal.DNSType,
-			Name:  internal.DNSName,
-			Value: internal.DNSValue,
-			TTL:   ttl,
-		}
-
-		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
-			record.Priority = internal.DNSPriority
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, existingRecord := range records {
-			if existingRecord.Type == record.Type &&
-				existingRecord.Name == record.Name &&
-				existingRecord.Value == record.Value {
-				logrus.Fatalf("%s record for %s already exists?", record.Type, zone)
-			}
-		}
-
-		createdRecords, err := provider.SetRecords(
-			context.Background(),
-			zone,
-			[]libdns.Record{record},
-		)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(createdRecords) == 0 {
-			logrus.Fatal("provider library reports that no record was created?")
-		}
-
-		createdRecord := createdRecords[0]
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := createdRecord.Value
-		if len(createdRecord.Value) > 30 {
-			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
-		}
-
-		table.Append([]string{
-			createdRecord.Type,
-			createdRecord.Name,
-			value,
-			createdRecord.TTL.String(),
-			strconv.Itoa(createdRecord.Priority),
-		})
-
-		table.Render()
-
-		logrus.Info("record created")
-
-		return nil
-	},
-}

cli/record/record.go

@@ -1,37 +0,0 @@
-package record
-
-import (
-	"github.com/urfave/cli"
-)
-
-// RecordCommand supports managing DNS entries.
-var RecordCommand = cli.Command{
-	Name:      "record",
-	Usage:     "Manage domain name records",
-	Aliases:   []string{"rc"},
-	ArgsUsage: "<record>",
-	Description: `
-Manage domain name records via 3rd party providers such as Gandi DNS. It
-supports listing, creating and removing all types of records that you might
-need for managing Co-op Cloud apps.
-
-The following providers are supported:
-
-    Gandi DNS https://www.gandi.net
-
-You need an account with such a provider already. Typically, you need to
-provide an API token on the Abra command-line when using these commands so that
-you can authenticate with your provider account.
-
-New providers can be integrated, we welcome change sets. See the underlying DNS
-library documentation for more. It supports many existing providers and allows
-to implement new provider support easily.
-
-    https://pkg.go.dev/github.com/libdns/libdns
-`,
-	Subcommands: []cli.Command{
-		RecordListCommand,
-		RecordNewCommand,
-		RecordRemoveCommand,
-	},
-}

cli/record/remove.go

@@ -1,136 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordRemoveCommand lists domains.
-var RecordRemoveCommand = cli.Command{
-	Name:      "remove",
-	Usage:     "Remove a domain name record",
-	Aliases:   []string{"rm"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Remove a domain name record for a specific zone.
-
-It uses the type of record and name to match existing records and choose one
-for deletion. You must specify a zone (e.g. example.com) under which your
-domain name records are listed. This zone must already be created on your
-provider account.
-
-Example:
-
-    abra record remove foo.com -p gandi -t A -n myapp
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record rm
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var toDelete libdns.Record
-		for _, record := range records {
-			if record.Type == internal.DNSType && record.Name == internal.DNSName {
-				toDelete = record
-				break
-			}
-		}
-
-		if (libdns.Record{}) == toDelete {
-			logrus.Fatal("provider library reports no matching record?")
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := toDelete.Value
-		if len(toDelete.Value) > 30 {
-			value = fmt.Sprintf("%s...", toDelete.Value[:30])
-		}
-
-		table.Append([]string{
-			toDelete.Type,
-			toDelete.Name,
-			value,
-			toDelete.TTL.String(),
-			strconv.Itoa(toDelete.Priority),
-		})
-
-		table.Render()
-
-		if !internal.NoInput {
-			response := false
-			prompt := &survey.Confirm{
-				Message: "continue with record deletion?",
-			}
-
-			if err := survey.AskOne(prompt, &response); err != nil {
-				return err
-			}
-
-			if !response {
-				logrus.Fatal("exiting as requested")
-			}
-		}
-
-		_, err = provider.DeleteRecords(context.Background(), zone, []libdns.Record{toDelete})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Info("record successfully deleted")
-
-		return nil
-	},
-}

cli/server/list.go

@@ -28,6 +28,7 @@ var serverListCommand = cli.Command{
 		problemsFilterFlag,
 		internal.DebugFlag,
 		internal.MachineReadableFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
@@ -59,6 +60,17 @@ var serverListCommand = cli.Command{
 					if err != nil {
 						logrus.Fatal(err)
 					}
+
+					if sp.Host == "" {
+						sp.Host = "unknown"
+					}
+					if sp.User == "" {
+						sp.User = "unknown"
+					}
+					if sp.Port == "" {
+						sp.Port = "unknown"
+					}
+
 					row = []string{serverName, sp.Host, sp.User, sp.Port}
 				}
 			}
@@ -72,8 +84,11 @@ var serverListCommand = cli.Command{
 			}
 
 			if problemsFilter {
-				if row[1] == "unknown" {
-					table.Append(row)
+				for _, val := range row {
+					if val == "unknown" {
+						table.Append(row)
+						break
+					}
 				}
 			} else {
 				table.Append(row)

cli/server/new.go

@@ -1,260 +0,0 @@
-package server
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/libcapsul"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-func newHetznerCloudVPS(c *cli.Context) error {
-	if err := internal.EnsureNewHetznerCloudVPSFlags(c); err != nil {
-		return err
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	var sshKeysRaw []string
-	var sshKeys []*hcloud.SSHKey
-	for _, sshKey := range c.StringSlice("hetzner-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-
-		sshKey, _, err := client.SSHKey.GetByName(context.Background(), sshKey)
-		if err != nil {
-			return err
-		}
-		sshKeys = append(sshKeys, sshKey)
-		sshKeysRaw = append(sshKeysRaw, sshKey.Name)
-	}
-
-	serverOpts := hcloud.ServerCreateOpts{
-		Name:       internal.HetznerCloudName,
-		ServerType: &hcloud.ServerType{Name: internal.HetznerCloudType},
-		Image:      &hcloud.Image{Name: internal.HetznerCloudImage},
-		SSHKeys:    sshKeys,
-		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
-	}
-
-	sshKeyIDs := strings.Join(sshKeysRaw, "\n")
-	if sshKeyIDs == "" {
-		sshKeyIDs = "N/A (password auth)"
-	}
-
-	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.HetznerCloudName,
-		internal.HetznerCloudType,
-		internal.HetznerCloudImage,
-		sshKeyIDs,
-		internal.HetznerCloudLocation,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	res, _, err := client.Server.Create(context.Background(), serverOpts)
-	if err != nil {
-		return err
-	}
-
-	var rootPassword string
-	if len(sshKeys) > 0 {
-		rootPassword = "N/A (using SSH keys)"
-	} else {
-		rootPassword = res.RootPassword
-	}
-
-	ip := res.Server.PublicNet.IPv4.IP.String()
-
-	fmt.Println(fmt.Sprintf(`
-Your new Hetzner Cloud VPS has successfully been created! Here are the details:
-
-    name: %s
-    IP address: %s
-    root password: %s
-
-You can access this new VPS via SSH using the following command:
-
-    ssh root@%s
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record (manually
-or by using "abra record new") and add the server to your Abra configuration
-("abra server add") to have a working server that you can deploy Co-op Cloud
-apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A %s
-    *  1800 IN A %s
-	`,
-		internal.HetznerCloudName, ip, rootPassword,
-		ip, ip, ip,
-	))
-
-	return nil
-}
-
-func newCapsulVPS(c *cli.Context) error {
-	if err := internal.EnsureNewCapsulVPSFlags(c); err != nil {
-		return err
-	}
-
-	capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", internal.CapsulInstanceURL)
-
-	var sshKeys []string
-	for _, sshKey := range c.StringSlice("capsul-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-		sshKeys = append(sshKeys, sshKey)
-	}
-
-	tableColumns := []string{"instance", "name", "type", "image", "ssh-keys"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.CapsulInstanceURL,
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		strings.Join(sshKeys, "\n"),
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with capsul creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	capsulClient := libcapsul.New(capsulCreateURL, internal.CapsulAPIToken)
-	resp, err := capsulClient.Create(
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		sshKeys,
-	)
-	if err != nil {
-		return err
-	}
-
-	fmt.Println(fmt.Sprintf(`
-Your new Capsul has successfully been created! Here are the details:
-
-    Capsul name: %s
-    Capsul ID:   %v
-
-You will need to log into your Capsul instance web interface to retrieve the IP
-address. You can learn all about how to get SSH access to your new Capsul on:
-
-    %s/about-ssh
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record (manually
-or by using "abra record new") and add the server to your Abra configuration
-("abra server add") to have a working server that you can deploy Co-op Cloud
-apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A <your-capsul-ip>
-    *  1800 IN A <your-capsul-ip>
-	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
-
-	return nil
-}
-
-var serverNewCommand = cli.Command{
-	Name:    "new",
-	Aliases: []string{"n"},
-	Usage:   "Create a new server using a 3rd party provider",
-	Description: `
-Create a new server via a 3rd party provider.
-
-The following providers are supported:
-
-		Capsul https://git.cyberia.club/Cyberia/capsul-flask
-    Hetzner Cloud https://docs.hetzner.com/cloud
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-
-API tokens are read from the environment if specified, e.g.
-
-    export HCLOUD_TOKEN=...
-`,
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.ServerProviderFlag,
-
-		// Capsul
-		internal.CapsulInstanceURLFlag,
-		internal.CapsulTypeFlag,
-		internal.CapsulImageFlag,
-		internal.CapsulSSHKeysFlag,
-		internal.CapsulAPITokenFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudTypeFlag,
-		internal.HetznerCloudImageFlag,
-		internal.HetznerCloudSSHKeysFlag,
-		internal.HetznerCloudLocationFlag,
-		internal.HetznerCloudAPITokenFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		if err := internal.EnsureServerProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		switch internal.ServerProvider {
-		case "capsul":
-			if err := newCapsulVPS(c); err != nil {
-				logrus.Fatal(err)
-			}
-		case "hetzner-cloud":
-			if err := newHetznerCloudVPS(c); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		return nil
-	},
-}

cli/server/prune.go

@@ -20,12 +20,12 @@ var allFilterFlag = &cli.BoolFlag{
 	Destination: &allFilter,
 }
 
-var volunesFilter bool
+var volumesFilter bool
 
 var volumesFilterFlag = &cli.BoolFlag{
 	Name:        "volumes, v",
 	Usage:       "Prune volumes. This will remove app data, Be Careful!",
-	Destination: &volunesFilter,
+	Destination: &volumesFilter,
 }
 
 var serverPruneCommand = cli.Command{
@@ -35,7 +35,7 @@ var serverPruneCommand = cli.Command{
 	Description: `
 Prunes unused containers, networks, and dangling images.
 
-If passing "-v/--volumes" then volumes not connected with a deployed app will
+If passing "-v/--volumes" then volumes not connected to a deployed app will
 also be removed. This can result in unwanted data loss if not used carefully.
 	`,
 	ArgsUsage: "[<server>]",
@@ -43,12 +43,12 @@ also be removed. This can result in unwanted data loss if not used carefully.
 		allFilterFlag,
 		volumesFilterFlag,
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.ServerNameComplete,
 	Action: func(c *cli.Context) error {
-		var args filters.Args
-
 		serverName := internal.ValidateServer(c)
 
 		cl, err := client.New(serverName)
@@ -56,6 +56,8 @@ also be removed. This can result in unwanted data loss if not used carefully.
 			logrus.Fatal(err)
 		}
 
+		var args filters.Args
+
 		ctx := context.Background()
 		cr, err := cl.ContainersPrune(ctx, args)
 		if err != nil {
@@ -74,6 +76,7 @@ also be removed. This can result in unwanted data loss if not used carefully.
 
 		pruneFilters := filters.NewArgs()
 		if allFilter {
+			logrus.Debugf("removing all images, not only dangling ones")
 			pruneFilters.Add("dangling", "false")
 		}
 
@@ -85,7 +88,7 @@ also be removed. This can result in unwanted data loss if not used carefully.
 		imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
 		logrus.Infof("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed)
 
-		if volunesFilter {
+		if volumesFilter {
 			vr, err := cl.VolumesPrune(ctx, args)
 			if err != nil {
 				logrus.Fatal(err)

cli/server/remove.go

@@ -1,8 +1,6 @@
 package server
 
 import (
-	"context"
-	"fmt"
 	"os"
 	"path/filepath"
 
@@ -10,167 +8,31 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var rmServer bool
-var rmServerFlag = &cli.BoolFlag{
-	Name:        "server, s",
-	Usage:       "remove the actual server also",
-	Destination: &rmServer,
-}
-
-func rmHetznerCloudVPS(c *cli.Context) error {
-	if internal.HetznerCloudName == "" && !internal.NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &internal.HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if internal.HetznerCloudAPIToken == "" && !internal.NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &internal.HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			internal.HetznerCloudAPIToken = token
-		}
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	server, _, err := client.Server.Get(context.Background(), internal.HetznerCloudName)
-	if err != nil {
-		return err
-	}
-
-	if server == nil {
-		logrus.Fatalf("library provider reports that %s doesn't exist?", internal.HetznerCloudName)
-	}
-
-	fmt.Println(fmt.Sprintf(`
-You have requested that Abra delete the following server (%s). Please be
-absolutely sure that this is indeed the server that you would like to have
-removed. There will be no going back once you confirm, the server will be
-destroyed.
-`, server.Name))
-
-	tableColumns := []string{"name", "type", "image", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		server.Name,
-		server.ServerType.Name,
-		server.Image.Name,
-		server.Datacenter.Name,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS removal?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	_, err = client.Server.Delete(context.Background(), server)
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof("%s has been deleted from your hetzner cloud account", internal.HetznerCloudName)
-
-	return nil
-}
-
 var serverRemoveCommand = cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
-	ArgsUsage: "[<server>]",
+	ArgsUsage: "<server>",
 	Usage:     "Remove a managed server",
-	Description: `
-Remova a server from Abra management.
+	Description: `Remove a managed server.
 
-Depending on whether you used a 3rd party provider to create this server ("abra
-server new"), you can also destroy the virtual server as well. Pass
-"--server/-s" to tell Abra to try to delete this VPS.
-
-Otherwise, Abra will remove the internal bookkeeping (~/.abra/servers/...) and
-underlying client connection context. This server will then be lost in time,
-like tears in rain.
+Abra will remove the internal bookkeeping (~/.abra/servers/...) and underlying
+client connection context. This server will then be lost in time, like tears in
+rain.
 `,
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		rmServerFlag,
-		internal.ServerProviderFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudAPITokenFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.ServerNameComplete,
 	Action: func(c *cli.Context) error {
 		serverName := internal.ValidateServer(c)
 
-		warnMsg := `Did not pass -s/--server for actual server deletion, prompting!
-
-Abra doesn't currently know if it helped you create this server with one of the
-3rd party integrations (e.g. Capsul). You have a choice here to actually,
-really and finally destroy this server using those integrations. If you want to
-do this, choose Yes.
-
-If you just want to remove the server config files & context, choose No.
-`
-
-		if !rmServer {
-			logrus.Warn(fmt.Sprintf(warnMsg))
-
-			response := false
-			prompt := &survey.Confirm{
-				Message: "delete actual live server?",
-			}
-			if err := survey.AskOne(prompt, &response); err != nil {
-				logrus.Fatal(err)
-			}
-			if response {
-				logrus.Info("setting -s/--server and attempting to remove actual server")
-				rmServer = true
-			}
-		}
-
-		if rmServer {
-			if err := internal.EnsureServerProvider(); err != nil {
-				logrus.Fatal(err)
-			}
-
-			switch internal.ServerProvider {
-			case "capsul":
-				logrus.Warn("capsul provider does not support automatic removal yet, sorry!")
-			case "hetzner-cloud":
-				if err := rmHetznerCloudVPS(c); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
 		if err := client.DeleteContext(serverName); err != nil {
 			logrus.Fatal(err)
 		}

cli/server/server.go

@@ -9,16 +9,7 @@ var ServerCommand = cli.Command{
 	Name:    "server",
 	Aliases: []string{"s"},
 	Usage:   "Manage servers",
-	Description: `
-Create, manage and remove servers using 3rd party integrations. 
-
-Servers can be created from scratch using the "abra server new" command. If you
-already have a server, you can add it to your configuration using "abra server
-add". Abra can provision servers so that they are ready to deploy Co-op Cloud
-recipes, see available flags on "abra server add" for more.
-`,
 	Subcommands: []cli.Command{
-		serverNewCommand,
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,

cli/updater/updater.go

@@ -12,7 +12,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
@@ -49,6 +48,7 @@ var Notify = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		majorFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -97,6 +97,7 @@ var UpgradeApp = cli.Command{
 		internal.ChaosFlag,
 		majorFlag,
 		allFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -114,12 +115,10 @@ update chaos deployments use the "--chaos" flag. Use it with care.
 			logrus.Fatal(err)
 		}
 
-		conf := runtime.New()
-
 		if !updateAll {
 			stackName := c.Args().Get(0)
 			recipeName := c.Args().Get(1)
-			err = tryUpgrade(cl, stackName, recipeName, conf)
+			err = tryUpgrade(cl, stackName, recipeName)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -139,7 +138,7 @@ update chaos deployments use the "--chaos" flag. Use it with care.
 				logrus.Fatal(err)
 			}
 
-			err = tryUpgrade(cl, stackName, recipeName, conf)
+			err = tryUpgrade(cl, stackName, recipeName)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -149,7 +148,7 @@ update chaos deployments use the "--chaos" flag. Use it with care.
 	},
 }
 
-// getLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
+// getLabel reads docker labels from running services in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
 func getLabel(cl *dockerclient.Client, stackName string, label string) (string, error) {
 	filter := filters.NewArgs()
 	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
@@ -171,7 +170,7 @@ func getLabel(cl *dockerclient.Client, stackName string, label string) (string,
 	return "", nil
 }
 
-// getBoolLabel reads a boolean docker label.
+// getBoolLabel reads a boolean docker label from running services
 func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool, error) {
 	lableValue, err := getLabel(cl, stackName, label)
 	if err != nil {
@@ -273,7 +272,7 @@ func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName st
 // flag is set.
 func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
 	deployedVersion string) ([]string, error) {
-	catl, err := recipe.ReadRecipeCatalogue()
+	catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +283,8 @@ func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName
 	}
 
 	if len(versions) == 0 {
-		return nil, fmt.Errorf("no published releases for %s in the recipe catalogue?", recipeName)
+		logrus.Warnf("no published releases for %s in the recipe catalogue?", recipeName)
+		return nil, nil
 	}
 
 	var availableUpgrades []string
@@ -316,12 +316,12 @@ func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName
 
 // processRecipeRepoVersion clones, pulls, checks out the version and lints the
 // recipe repository.
-func processRecipeRepoVersion(recipeName, version string, conf *runtime.Config) error {
-	if err := recipe.EnsureExists(recipeName, conf); err != nil {
+func processRecipeRepoVersion(recipeName, version string) error {
+	if err := recipe.EnsureExists(recipeName); err != nil {
 		return err
 	}
 
-	if err := recipe.EnsureUpToDate(recipeName, conf); err != nil {
+	if err := recipe.EnsureUpToDate(recipeName); err != nil {
 		return err
 	}
 
@@ -329,7 +329,7 @@ func processRecipeRepoVersion(recipeName, version string, conf *runtime.Config)
 		return err
 	}
 
-	if r, err := recipe.Get(recipeName, conf); err != nil {
+	if r, err := recipe.Get(recipeName, internal.Offline); err != nil {
 		return err
 	} else if err := lint.LintForErrors(r); err != nil {
 		return err
@@ -364,7 +364,7 @@ func createDeployConfig(recipeName string, stackName string, env config.AppEnv)
 		ResolveImage: stack.ResolveImageAlways,
 	}
 
-	composeFiles, err := config.GetAppComposeFiles(recipeName, env)
+	composeFiles, err := config.GetComposeFiles(recipeName, env)
 	if err != nil {
 		return nil, deployOpts, err
 	}
@@ -386,9 +386,9 @@ func createDeployConfig(recipeName string, stackName string, env config.AppEnv)
 }
 
 // tryUpgrade performs the upgrade if all the requirements are fulfilled.
-func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *runtime.Config) error {
+func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string) error {
 	if recipeName == "" {
-		logrus.Debugf("Don't update %s due to missing recipe name", stackName)
+		logrus.Debugf("don't update %s due to missing recipe name", stackName)
 		return nil
 	}
 
@@ -398,7 +398,7 @@ func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *run
 	}
 
 	if chaos && !internal.Chaos {
-		logrus.Debugf("Don't update %s due to chaos deployment.", stackName)
+		logrus.Debugf("don't update %s due to chaos deployment", stackName)
 		return nil
 	}
 
@@ -408,7 +408,7 @@ func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *run
 	}
 
 	if !updatesEnabled {
-		logrus.Debugf("Don't update %s due to disabling auto updates or missing ENABLE_AUTOUPDATE env.", stackName)
+		logrus.Debugf("don't update %s due to disabled auto updates or missing ENABLE_AUTO_UPDATE env", stackName)
 		return nil
 	}
 
@@ -418,17 +418,18 @@ func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *run
 	}
 
 	if upgradeVersion == "" {
-		logrus.Debugf("Don't update %s due to no new version.", stackName)
+		logrus.Debugf("don't update %s due to no new version", stackName)
 		return nil
 	}
 
-	err = upgrade(cl, stackName, recipeName, upgradeVersion, conf)
+	err = upgrade(cl, stackName, recipeName, upgradeVersion)
 
 	return err
 }
 
 // upgrade performs all necessary steps to upgrade an app.
-func upgrade(cl *dockerclient.Client, stackName, recipeName, upgradeVersion string, conf *runtime.Config) error {
+func upgrade(cl *dockerclient.Client, stackName, recipeName,
+	upgradeVersion string) error {
 	env, err := getEnv(cl, stackName)
 	if err != nil {
 		return err
@@ -441,7 +442,7 @@ func upgrade(cl *dockerclient.Client, stackName, recipeName, upgradeVersion stri
 		Env:    env,
 	}
 
-	if err = processRecipeRepoVersion(recipeName, upgradeVersion, conf); err != nil {
+	if err = processRecipeRepoVersion(recipeName, upgradeVersion); err != nil {
 		return err
 	}
 
@@ -454,7 +455,7 @@ func upgrade(cl *dockerclient.Client, stackName, recipeName, upgradeVersion stri
 		return err
 	}
 
-	logrus.Infof("Upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
+	logrus.Infof("upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
 
 	err = stack.RunDeploy(cl, deployOpts, compose, stackName, true)
 

go.mod

@@ -1,29 +1,99 @@
 module coopcloud.tech/abra
 
-go 1.16
+go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
-	github.com/AlecAivazis/survey/v2 v2.3.6
-	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
+	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd
+	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.23+incompatible
-	github.com/docker/distribution v2.8.1+incompatible
-	github.com/docker/docker v20.10.23+incompatible
+	github.com/docker/cli v24.0.7+incompatible
+	github.com/docker/distribution v2.8.3+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.5.2
-	github.com/hetznercloud/hcloud-go v1.40.0
+	github.com/go-git/go-git/v5 v5.10.0
 	github.com/moby/sys/signal v0.7.0
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
+	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.13.0
-	github.com/sirupsen/logrus v1.9.0
-	gotest.tools/v3 v3.4.0
+	github.com/schollz/progressbar/v3 v3.14.1
+	github.com/sirupsen/logrus v1.9.3
+	gotest.tools/v3 v3.5.1
 )
 
 require (
-	coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/BurntSushi/toml v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.9.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/miekg/pkcs11 v1.0.3 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/runc v1.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/skeema/knownhosts v1.2.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	golang.org/x/tools v0.13.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/buger/goterm v1.0.4
 	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/containers/image v3.0.2+incompatible
@@ -34,17 +104,17 @@ require (
 	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.2
-	github.com/klauspost/pgzip v1.2.5
-	github.com/libdns/gandi v1.0.2
-	github.com/libdns/libdns v0.2.1
-	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.5
+	github.com/klauspost/pgzip v1.2.6
+	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/stretchr/testify v1.8.4
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/crypto v0.5.0 // indirect
-	golang.org/x/sys v0.5.0
+	golang.org/x/sys v0.14.0
 )

go.sum

@@ -46,15 +46,17 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e h1:o5OZInc5b9esiN4hlfjZY6u0r+qB2iSv/11jnMGuR38=
-coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e/go.mod h1:HEQ9pSJRsDKabMxPfYCCzpVpAreLoC4Gh4SkVyOaKvk=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52 h1:cyFFOl0tKe+dVHt8saejG8xoff33eQiHxFCVzRpPUjM=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/AlecAivazis/survey/v2 v2.3.6 h1:NvTuVHISgTHEHeBFqt6BHOe4Ny/NwGZr7w+F8S9ziyw=
-github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI=
-github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg=
-github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd h1:dctCkMhcsgIWMrkB1Br8S0RJF17eG+LKiqcXXVr3mdU=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
+github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
+github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -85,8 +87,9 @@ github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JP
 github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
@@ -104,8 +107,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
-github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
+github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg=
+github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -113,13 +116,12 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
-github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
-github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
+github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
@@ -158,15 +160,15 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembj
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
-github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
@@ -184,8 +186,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e85keuznYcH5rqI438v41pKcBl4ZxQ=
 github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
-github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
-github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
+github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -210,7 +212,6 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1
 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
-github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
@@ -241,7 +242,6 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL
 github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
 github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
-github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
@@ -313,11 +313,13 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI=
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -333,18 +335,20 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
+github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v20.10.23+incompatible h1:qwyha/T3rXk9lfuVcn533cKFc7n/6IzL5GXVAgMVPBg=
-github.com/docker/cli v20.10.23+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
+github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
+github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.23+incompatible h1:1ZQUUYAdh+oylOT85aA2ZcfRp22jmLhoaEcVEfK8dyA=
-github.com/docker/docker v20.10.23+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -369,6 +373,8 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -407,25 +413,22 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
-github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
-github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
-github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE=
-github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
-github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
-github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw=
-github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI=
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
+github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
+github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
+github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ=
+github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
@@ -492,8 +495,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -586,8 +590,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
-github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -605,8 +609,6 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn
 github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hetznercloud/hcloud-go v1.40.0 h1:rchS+LVd80GuAOQUM5pLs0Fn7tb4PflPCuxskLYc1JQ=
-github.com/hetznercloud/hcloud-go v1.40.0/go.mod h1:jcKomw7kDIjrJ9FI72S/RdRPK8X8arYzEnXjRM/uXT8=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -617,15 +619,13 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
-github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE=
 github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
 github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc=
@@ -635,7 +635,6 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -647,7 +646,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
@@ -661,8 +659,9 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
 github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
+github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -670,19 +669,15 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/libdns/gandi v1.0.2 h1:1Ts8UpI1x5PVKpOjKC7Dn4+EObndz9gm6vdZnloHSKQ=
-github.com/libdns/gandi v1.0.2/go.mod h1:hxpbQKcQFgQrTS5lV4tAgn6QoL6HcCnoBJaW5nOW4Sk=
-github.com/libdns/libdns v0.1.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
-github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
-github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
 github.com/magiconair/properties v1.5.3/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -710,8 +705,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -721,8 +716,9 @@ github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -745,18 +741,20 @@ github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGg
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
-github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM=
-github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM=
+github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=
+github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
+github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
+github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
 github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI=
 github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
-github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
-github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -768,7 +766,6 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -793,6 +790,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -833,8 +832,8 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI=
-github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -854,14 +853,15 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
 github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -869,8 +869,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -881,16 +881,18 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
-github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -899,13 +901,12 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/schollz/progressbar/v3 v3.13.0 h1:9TeeWRcjW2qd05I8Kf9knPkW4vLM/hYoa6z9ABvxje8=
-github.com/schollz/progressbar/v3 v3.13.0/go.mod h1:ZBYnSuLAX2LU8P8UiKN/KgF2DY58AJC8yfVYLPC8Ly4=
+github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI=
+github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
-github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -917,10 +918,10 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0=
-github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
+github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -966,8 +967,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -1035,7 +1037,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -1066,12 +1067,11 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
+golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1109,6 +1109,9 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1163,10 +1166,11 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
-golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1196,6 +1200,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1255,7 +1262,6 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1286,7 +1292,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1303,26 +1308,23 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
-golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1333,8 +1335,10 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
-golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1407,6 +1411,9 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1564,8 +1571,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
@@ -1606,14 +1614,13 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
-gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/autocomplete/autocomplete.go

@@ -25,9 +25,19 @@ func AppNameComplete(c *cli.Context) {
 	}
 }
 
+func ServiceNameComplete(appName string) {
+	serviceNames, err := config.GetAppServiceNames(appName)
+	if err != nil {
+		return
+	}
+	for _, s := range serviceNames {
+		fmt.Println(s)
+	}
+}
+
 // RecipeNameComplete completes recipe names.
 func RecipeNameComplete(c *cli.Context) {
-	catl, err := recipe.ReadRecipeCatalogue()
+	catl, err := recipe.ReadRecipeCatalogue(false)
 	if err != nil {
 		logrus.Warn(err)
 	}
@@ -68,7 +78,6 @@ func SubcommandComplete(c *cli.Context) {
 		"autocomplete",
 		"catalogue",
 		"recipe",
-		"record",
 		"server",
 		"upgrade",
 	}

pkg/catalogue/catalogue.go

@@ -12,49 +12,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                        true,
-	"abra-apps":                   true,
-	"abra-aur":                    true,
-	"abra-bash":                   true,
-	"abra-capsul":                 true,
-	"abra-gandi":                  true,
-	"abra-hetzner":                true,
-	"apps":                        true,
-	"aur-abra-git":                true,
-	"auto-recipes-catalogue-json": true,
-	"auto-mirror":                 true,
-	"backup-bot":                  true,
-	"backup-bot-two":              true,
-	"beta.coopcloud.tech":         true,
-	"comrade-renovate-bot":        true,
-	"coopcloud.tech":              true,
-	"coturn":                      true,
-	"docker-cp-deploy":            true,
-	"docker-dind-bats-kcov":       true,
-	"docs.coopcloud.tech":         true,
-	"drone-abra":                  true,
-	"example":                     true,
-	"gardening":                   true,
-	"go-abra":                     true,
-	"organising":                  true,
-	"pyabra":                      true,
-	"radicle-seed-node":           true,
-	"recipes-catalogue-json":      true,
-	"recipes-wishlist":            true,
-	"recipes.coopcloud.tech":      true,
-	"stack-ssh-deploy":            true,
-	"swarm-cronjob":               true,
-	"tagcmp":                      true,
-	"traefik-cert-dumper":         true,
-	"tyop":                        true,
-}
-
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		logrus.Warnf("local recipe catalogue is missing, retrieving now")
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
 		if err := gitPkg.Clone(catalogueDir, url); err != nil {
 			return err
@@ -66,9 +28,8 @@ func EnsureCatalogue() error {
 	return nil
 }
 
-// EnsureUpToDate ensures that the local catalogue has no unstaged changes as
-// is up to date. This is useful to run before doing catalogue generation.
-func EnsureUpToDate() error {
+// EnsureIsClean makes sure that the catalogue has no unstaged changes.
+func EnsureIsClean() error {
 	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
 	if err != nil {
 		return err
@@ -79,6 +40,11 @@ func EnsureUpToDate() error {
 		return fmt.Errorf(msg, config.CATALOGUE_DIR)
 	}
 
+	return nil
+}
+
+// EnsureUpToDate ensures that the local catalogue is up to date.
+func EnsureUpToDate() error {
 	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
 	if err != nil {
 		return err

pkg/client/volumes.go

@@ -3,13 +3,14 @@ package client
 import (
 	"context"
 
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
 )
 
-func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*types.Volume, error) {
-	volumeListOKBody, err := cl.VolumeList(ctx, fs)
+func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
+	volumeListOptions := volume.ListOptions{fs}
+	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
 		return volumeList, err
@@ -18,7 +19,7 @@ func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filter
 	return volumeList, nil
 }
 
-func GetVolumeNames(volumes []*types.Volume) []string {
+func GetVolumeNames(volumes []*volume.Volume) []string {
 	var volumeNames []string
 
 	for _, vol := range volumes {

pkg/compose/compose.go

@@ -122,7 +122,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 
 		discovered := false
 		for oldLabel, value := range service.Deploy.Labels {
-			if strings.HasPrefix(oldLabel, "coop-cloud") {
+			if strings.HasPrefix(oldLabel, "coop-cloud") && strings.Contains(oldLabel, "version") {
 				discovered = true
 
 				bytes, err := ioutil.ReadFile(composeFile)

pkg/config/app.go

@@ -25,6 +25,9 @@ import (
 // AppEnv is a map of the values in an apps env config
 type AppEnv = map[string]string
 
+// AppModifiers is a map of modifiers in an apps env config
+type AppModifiers = map[string]map[string]string
+
 // AppName is AppName
 type AppName = string
 
@@ -76,7 +79,7 @@ func (a App) StackName() string {
 func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) {
 	filters := filters.NewArgs()
 
-	composeFiles, err := GetAppComposeFiles(a.Recipe, a.Env)
+	composeFiles, err := GetComposeFiles(a.Recipe, a.Env)
 	if err != nil {
 		return filters, err
 	}
@@ -149,7 +152,7 @@ func (a ByName) Less(i, j int) bool {
 	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
 }
 
-func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
+func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	env, err := ReadEnv(appFile.Path)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
@@ -157,7 +160,7 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 
 	logrus.Debugf("read env %s from %s", env, appFile.Path)
 
-	app, err := newApp(env, name, appFile)
+	app, err := NewApp(env, name, appFile)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
 	}
@@ -165,8 +168,8 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	return app, nil
 }
 
-// newApp creates new App object
-func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+// NewApp creates new App object
+func NewApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	domain := env["DOMAIN"]
 
 	recipe, exists := env["RECIPE"]
@@ -232,7 +235,7 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
 		return App{}, fmt.Errorf("cannot find app with name %s", name)
 	}
 
-	app, err := readAppEnvFile(appFile, name)
+	app, err := ReadAppEnvFile(appFile, name)
 	if err != nil {
 		return App{}, err
 	}
@@ -277,7 +280,7 @@ func GetAppServiceNames(appName string) ([]string, error) {
 		return serviceNames, err
 	}
 
-	composeFiles, err := GetAppComposeFiles(app.Recipe, app.Env)
+	composeFiles, err := GetComposeFiles(app.Recipe, app.Env)
 	if err != nil {
 		return serviceNames, err
 	}
@@ -326,7 +329,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	}
 
 	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
-	if _, err := os.Stat(appEnvPath); os.IsExist(err) {
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
@@ -403,7 +406,25 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
 				result["status"] = "deployed"
 			}
 
-			labelKey := fmt.Sprintf("coop-cloud.%s.version", name)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name)
+			chaos, ok := service.Spec.Labels[labelKey]
+			if ok {
+				result["chaos"] = chaos
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name)
+			if chaosVersion, ok := service.Spec.Labels[labelKey]; ok {
+				result["chaosVersion"] = chaosVersion
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.autoupdate", name)
+			if autoUpdate, ok := service.Spec.Labels[labelKey]; ok {
+				result["autoUpdate"] = autoUpdate
+			} else {
+				result["autoUpdate"] = "false"
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.version", name)
 			if version, ok := service.Spec.Labels[labelKey]; ok {
 				result["version"] = version
 			} else {
@@ -419,26 +440,56 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
 	return statuses, nil
 }
 
-// GetAppComposeFiles gets the list of compose files for an app which should be
-// merged into a composetypes.Config while respecting the COMPOSE_FILE env var.
-func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
+// ensurePathExists ensures that a path exists.
+func ensurePathExists(path string) error {
+	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// GetComposeFiles gets the list of compose files for an app (or recipe if you
+// don't already have an app) which should be merged into a composetypes.Config
+// while respecting the COMPOSE_FILE env var.
+func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
 	var composeFiles []string
 
-	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
-		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
+	composeFileEnvVar, ok := appEnv["COMPOSE_FILE"]
+	if !ok {
 		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path)
 		composeFiles = append(composeFiles, path)
 		return composeFiles, nil
 	}
 
-	composeFileEnvVar := appEnv["COMPOSE_FILE"]
-	envVars := strings.Split(composeFileEnvVar, ":")
-	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
-	for _, file := range strings.Split(composeFileEnvVar, ":") {
+	if !strings.Contains(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("COMPOSE_FILE detected, loading %s", path)
+		composeFiles = append(composeFiles, path)
+		return composeFiles, nil
+	}
+
+	numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1
+	envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles)
+	if len(envVars) != numComposeFiles {
+		return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar)
+	}
+
+	for _, file := range envVars {
 		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
 		composeFiles = append(composeFiles, path)
 	}
 
+	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
 	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
 
 	return composeFiles, nil
@@ -499,6 +550,17 @@ func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) {
 	}
 }
 
+// SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container
+func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName)
+			service.Deploy.Labels[labelKey] = chaosVersion
+		}
+	}
+}
+
 // SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the
 // auto update process for this app. The default if this variable is not set is to disable
 // the auto update process.
@@ -515,3 +577,29 @@ func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEn
 		}
 	}
 }
+
+// GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files
+func GetLabel(compose *composetypes.Config, stackName string, label string) string {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+			logrus.Debugf("get label '%s'", labelKey)
+			if labelValue, ok := service.Deploy.Labels[labelKey]; ok {
+				return labelValue
+			}
+		}
+	}
+	logrus.Debugf("no %s label found for %s", label, stackName)
+	return ""
+}
+
+// GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value
+func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
+	var timeout = 50 // Default Timeout
+	var err error = nil
+	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
+		logrus.Debugf("timeout label: %s", timeoutLabel)
+		timeout, err = strconv.Atoi(timeoutLabel)
+	}
+	return timeout, err
+}

pkg/config/app_test.go

@@ -1,36 +1,108 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestNewApp(t *testing.T) {
-	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestReadAppEnvFile(t *testing.T) {
-	app, err := readAppEnvFile(expectedAppFile, appName)
+	app, err := config.ReadAppEnvFile(ExpectedAppFile, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestGetApp(t *testing.T) {
-	app, err := GetApp(expectedAppFiles, appName)
+	app, err := config.GetApp(ExpectedAppFiles, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
+	}
+}
+
+func TestGetComposeFiles(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		appEnv       map[string]string
+		composeFiles []string
+	}{
+		{
+			map[string]string{},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml:compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		composeFiles, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, composeFiles, test.composeFiles)
+	}
+}
+
+func TestGetComposeFilesError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct{ appEnv map[string]string }{
+		{map[string]string{"COMPOSE_FILE": "compose.yml::compose.foo.yml"}},
+		{map[string]string{"COMPOSE_FILE": "doesnt.exist.yml"}},
+	}
+
+	for _, test := range tests {
+		_, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err == nil {
+			t.Fatalf("should have failed: %v", test.appEnv)
+		}
 	}
 }

pkg/config/env.go

@@ -8,13 +8,24 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
+	"sort"
 	"strings"
 
-	"github.com/Autonomic-Cooperative/godotenv"
+	"git.coopcloud.tech/coop-cloud/godotenv"
 	"github.com/sirupsen/logrus"
 )
 
-var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
+// getBaseDir retrieves the Abra base directory.
+func getBaseDir() string {
+	home := os.ExpandEnv("$HOME/.abra")
+	if customAbraDir, exists := os.LookupEnv("ABRA_DIR"); exists && customAbraDir != "" {
+		home = customAbraDir
+	}
+	return home
+}
+
+var ABRA_DIR = getBaseDir()
 var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
 var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
 var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
@@ -25,6 +36,11 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+// envVarModifiers is a list of env var modifier strings. These are added to
+// env vars as comments and modify their processing by Abra, e.g. determining
+// how long secrets should be.
+var envVarModifiers = []string{"length"}
+
 // GetServers retrieves all servers.
 func GetServers() ([]string, error) {
 	var servers []string
@@ -41,16 +57,30 @@ func GetServers() ([]string, error) {
 
 // ReadEnv loads an app envivornment into a map.
 func ReadEnv(filePath string) (AppEnv, error) {
-	var envFile AppEnv
+	var envVars AppEnv
 
-	envFile, err := godotenv.Read(filePath)
+	envVars, _, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
-	logrus.Debugf("read %s from %s", envFile, filePath)
+	logrus.Debugf("read %s from %s", envVars, filePath)
 
-	return envFile, nil
+	return envVars, nil
+}
+
+// ReadEnv loads an app envivornment and their modifiers in two different maps.
+func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) {
+	var envVars AppEnv
+
+	envVars, mods, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, mods, err
+	}
+
+	logrus.Debugf("read %s from %s", envVars, filePath)
+
+	return envVars, mods, nil
 }
 
 // ReadServerNames retrieves all server names.
@@ -140,22 +170,107 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 		}
 		return envVars, err
 	}
+	defer file.Close()
+
+	exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`)
+	if err != nil {
+		return envVars, err
+	}
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, "export") {
-			splitVals := strings.Split(line, "export ")
+		txt := scanner.Text()
+		if exportRegex.MatchString(txt) {
+			splitVals := strings.Split(txt, "export ")
 			envVarDef := splitVals[len(splitVals)-1]
 			keyVal := strings.Split(envVarDef, "=")
 			if len(keyVal) != 2 {
-				return envVars, fmt.Errorf("couldn't parse %s", line)
+				return envVars, fmt.Errorf("couldn't parse %s", txt)
 			}
 			envVars[keyVal[0]] = keyVal[1]
 		}
 	}
 
-	logrus.Debugf("read %s from %s", envVars, abraSh)
+	if len(envVars) > 0 {
+		logrus.Debugf("read %s from %s", envVars, abraSh)
+	} else {
+		logrus.Debugf("read 0 env var exports from %s", abraSh)
+	}
 
 	return envVars, nil
 }
+
+type EnvVar struct {
+	Name    string
+	Present bool
+}
+
+func CheckEnv(app App) ([]EnvVar, error) {
+	var envVars []EnvVar
+
+	envSamplePath := path.Join(RECIPES_DIR, app.Recipe, ".env.sample")
+	if _, err := os.Stat(envSamplePath); err != nil {
+		if os.IsNotExist(err) {
+			return envVars, fmt.Errorf("%s does not exist?", envSamplePath)
+		}
+		return envVars, err
+	}
+
+	envSample, err := ReadEnv(envSamplePath)
+	if err != nil {
+		return envVars, err
+	}
+
+	var keys []string
+	for key := range envSample {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		if _, ok := app.Env[key]; ok {
+			envVars = append(envVars, EnvVar{Name: key, Present: true})
+		} else {
+			envVars = append(envVars, EnvVar{Name: key, Present: false})
+		}
+	}
+
+	return envVars, nil
+}
+
+// ReadAbraShCmdNames reads the names of commands.
+func ReadAbraShCmdNames(abraSh string) ([]string, error) {
+	var cmdNames []string
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cmdNames, nil
+		}
+		return cmdNames, err
+	}
+	defer file.Close()
+
+	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
+	if err != nil {
+		return cmdNames, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := cmdNameRegex.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			cmdNames = append(cmdNames, matches[1])
+		}
+	}
+
+	if len(cmdNames) > 0 {
+		logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh)
+	} else {
+		logrus.Debugf("read 0 command names from %s", abraSh)
+	}
+
+	return cmdNames, nil
+}

pkg/config/env_test.go

@@ -1,60 +1,69 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+var (
+	TestFolder    = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+	ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+)
 
 // make sure these are in alphabetical order
-var tFolders = []string{"folder1", "folder2"}
-var tFiles = []string{"bar.env", "foo.env"}
+var (
+	TFolders = []string{"folder1", "folder2"}
+	TFiles   = []string{"bar.env", "foo.env"}
+)
 
-var appName = "ecloud"
-var serverName = "evil.corp"
+var (
+	AppName    = "ecloud"
+	ServerName = "evil.corp"
+)
 
-var expectedAppEnv = AppEnv{
+var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
 	"RECIPE": "ecloud",
 }
 
-var expectedApp = App{
-	Name:   appName,
-	Recipe: expectedAppEnv["RECIPE"],
-	Domain: expectedAppEnv["DOMAIN"],
-	Env:    expectedAppEnv,
-	Path:   expectedAppFile.Path,
-	Server: expectedAppFile.Server,
+var ExpectedApp = config.App{
+	Name:   AppName,
+	Recipe: ExpectedAppEnv["RECIPE"],
+	Domain: ExpectedAppEnv["DOMAIN"],
+	Env:    ExpectedAppEnv,
+	Path:   ExpectedAppFile.Path,
+	Server: ExpectedAppFile.Server,
 }
 
-var expectedAppFile = AppFile{
-	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
-	Server: serverName,
+var ExpectedAppFile = config.AppFile{
+	Path:   path.Join(ValidAbraConf, "servers", ServerName, AppName+".env"),
+	Server: ServerName,
 }
 
-var expectedAppFiles = map[string]AppFile{
-	appName: expectedAppFile,
+var ExpectedAppFiles = map[string]config.AppFile{
+	AppName: ExpectedAppFile,
 }
 
-// var expectedServerNames = []string{"evil.corp"}
-
 func TestGetAllFoldersInDirectory(t *testing.T) {
-	folders, err := GetAllFoldersInDirectory(testFolder)
+	folders, err := config.GetAllFoldersInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(folders, tFolders) {
-		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	if !reflect.DeepEqual(folders, TFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(TFolders, ","), strings.Join(folders, ","))
 	}
 }
 
 func TestGetAllFilesInDirectory(t *testing.T) {
-	files, err := GetAllFilesInDirectory(testFolder)
+	files, err := config.GetAllFilesInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,23 +71,205 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 	for _, file := range files {
 		fileNames = append(fileNames, file.Name())
 	}
-	if !reflect.DeepEqual(fileNames, tFiles) {
-		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	if !reflect.DeepEqual(fileNames, TFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(TFiles, ","), strings.Join(fileNames, ","))
 	}
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := ReadEnv(expectedAppFile.Path)
+	env, err := config.ReadEnv(ExpectedAppFile.Path)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(env, expectedAppEnv) {
+	if !reflect.DeepEqual(env, ExpectedAppEnv) {
 		t.Fatalf(
 			"did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s",
-			expectedAppEnv["DOMAIN"],
-			expectedAppEnv["RECIPE"],
+			ExpectedAppEnv["DOMAIN"],
+			ExpectedAppEnv["RECIPE"],
 			env["DOMAIN"],
 			env["RECIPE"],
 		)
 	}
 }
+
+func TestReadAbraShEnvVars(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(abraShEnv) == 0 {
+		t.Error("at least one env var should be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_FOO"]; ok {
+		t.Error("INNER_FOO should not be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_BAZ"]; ok {
+		t.Error("INNER_BAZ should not be exported")
+	}
+
+	if _, ok := abraShEnv["OUTER_FOO"]; !ok {
+		t.Error("OUTER_FOO should be exported")
+	}
+}
+
+func TestReadAbraShCmdNames(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(cmdNames) == 0 {
+		t.Error("at least one command name should be found")
+	}
+
+	expectedCmdNames := []string{"test_cmd", "test_cmd_args"}
+	for _, cmdName := range expectedCmdNames {
+		if !slices.Contains(cmdNames, cmdName) {
+			t.Fatalf("%s should have been found in %s", cmdName, abraShPath)
+		}
+	}
+}
+
+func TestCheckEnv(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if !envVar.Present {
+			t.Fatalf("%s should be present", envVar.Name)
+		}
+	}
+}
+
+func TestCheckEnvError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	delete(envSample, "DOMAIN")
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if envVar.Name == "DOMAIN" && envVar.Present {
+			t.Fatalf("%s should not be present", envVar.Name)
+		}
+	}
+}
+
+func TestEnvVarCommentsRemoved(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envVar, exists := envSample["WITH_COMMENT"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "should be removed") {
+		t.Fatalf("comment from '%s' should be removed", envVar)
+	}
+
+	envVar, exists = envSample["SECRET_TEST_PASS_TWO_VERSION"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should have been removed")
+	}
+}
+
+func TestEnvVarModifiersIncluded(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "v1") {
+		t.Errorf("value should be 'v1', got: '%s'", envSample["SECRET_TEST_PASS_TWO_VERSION"])
+	}
+	if modifiers == nil || modifiers["SECRET_TEST_PASS_TWO_VERSION"] == nil {
+		t.Errorf("no modifiers included")
+	} else {
+		if modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"] != "10" {
+			t.Errorf("length modifier should be '10', got: '%s'", modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"])
+		}
+	}
+}

pkg/context/context.go

@@ -8,22 +8,19 @@ import (
 	"github.com/docker/cli/cli/context"
 	contextStore "github.com/docker/cli/cli/context/store"
 	cliflags "github.com/docker/cli/cli/flags"
-	"github.com/moby/term"
 )
 
 func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
-	_, _, stderr := term.StdStreams()
-	dockerConfig := dConfig.LoadDefaultConfigFile(stderr)
 	contextDir := dConfig.ContextStoreDir()
 	storeConfig := command.DefaultContextStoreConfig()
 	store := newContextStore(contextDir, storeConfig)
 
-	opts := &cliflags.CommonOptions{Context: "default"}
+	opts := &cliflags.ClientOptions{Context: "default"}
 
 	dockerContextStore := &command.ContextStoreWithDefault{
 		Store: store,
 		Resolver: func() (*command.DefaultContext, error) {
-			return command.ResolveDefaultContext(opts, dockerConfig, storeConfig, stderr)
+			return command.ResolveDefaultContext(opts, storeConfig)
 		},
 	}
 

pkg/dns/dns.go

@@ -3,39 +3,20 @@ package dns
 import (
 	"fmt"
 	"net"
-	"os"
-	"time"
-
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/sirupsen/logrus"
 )
 
-// NewToken constructs a new DNS provider token.
-func NewToken(provider, providerTokenEnvVar string) (string, error) {
-	if token, present := os.LookupEnv(providerTokenEnvVar); present {
-		return token, nil
-	}
-
-	logrus.Debugf("no %s in environment, asking via stdin", providerTokenEnvVar)
-
-	var token string
-	prompt := &survey.Input{
-		Message: fmt.Sprintf("%s API token?", provider),
-	}
-	if err := survey.AskOne(prompt, &token); err != nil {
-		return "", err
-	}
-
-	return token, nil
-}
-
 // EnsureIPv4 ensures that an ipv4 address is set for a domain name
 func EnsureIPv4(domainName string) (string, error) {
-	ipv4, err := net.ResolveIPAddr("ip", domainName)
+	ipv4, err := net.ResolveIPAddr("ip4", domainName)
 	if err != nil {
 		return "", err
 	}
 
+	// NOTE(d1): e.g. when there is only an ipv6 record available
+	if ipv4 == nil {
+		return "", fmt.Errorf("unable to resolve ipv4 address for %s", domainName)
+	}
+
 	return ipv4.String(), nil
 }
 
@@ -72,12 +53,3 @@ func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
 
 	return ipv4, nil
 }
-
-// GetTTL parses a ttl string into a duration
-func GetTTL(ttl string) (time.Duration, error) {
-	val, err := time.ParseDuration(ttl)
-	if err != nil {
-		return val, err
-	}
-	return val, nil
-}

pkg/dns/dns_test.go

@@ -0,0 +1,64 @@
+package dns
+
+import (
+	"fmt"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
+	tests := []struct {
+		domainName     string
+		serverName     string
+		shouldValidate bool
+	}{
+		// NOTE(d1): DNS records get checked, so use something that is maintained
+		// within the federation. if you're here because of a failing test, try
+		// `dig +short <domain>` to ensure stuff matches first! If flakyness
+		// becomes an issue we can look into mocking
+		{"docs.coopcloud.tech", "coopcloud.tech", true},
+		{"docs.coopcloud.tech", "swarm.autonomic.zone", true},
+
+		// NOTE(d1): special case handling for "--local"
+		{"", "default", true},
+		{"", "local", true},
+
+		{"", "", false},
+		{"123", "", false},
+	}
+
+	for _, test := range tests {
+		_, err := EnsureDomainsResolveSameIPv4(test.domainName, test.serverName)
+		if err != nil && test.shouldValidate {
+			t.Fatal(err)
+		}
+
+		if err == nil && !test.shouldValidate {
+			t.Fatal(fmt.Errorf("should have failed but did not: %v", test))
+		}
+	}
+}
+
+func TestEnsureIpv4(t *testing.T) {
+	// NOTE(d1): DNS records get checked, so use something that is maintained
+	// within the federation. if you're here because of a failing test, try `dig
+	// +short <domain>` to ensure stuff matches first! If flakyness becomes an
+	// issue we can look into mocking
+	domainName := "collabora.ostrom.collective.tools"
+	serverName := "ostrom.collective.tools"
+
+	for i := 0; i < 15; i++ {
+		domainIpv4, err := EnsureIPv4(domainName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		serverIpv4, err := EnsureIPv4(serverName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, domainIpv4, serverIpv4)
+	}
+}

pkg/dns/gandi/gandi.go

@@ -1,15 +0,0 @@
-package gandi
-
-import (
-	"coopcloud.tech/abra/pkg/dns"
-	"github.com/libdns/gandi"
-)
-
-// New constructs a new DNs provider.
-func New() (gandi.Provider, error) {
-	token, err := dns.NewToken("Gandi", "GANDI_TOKEN")
-	if err != nil {
-		return gandi.Provider{}, err
-	}
-	return gandi.Provider{APIToken: token}, nil
-}

pkg/git/commit.go

@@ -8,7 +8,7 @@ import (
 )
 
 // Commit runs a git commit
-func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
+func Commit(repoPath, commitMessage string, dryRun bool) error {
 	if commitMessage == "" {
 		return fmt.Errorf("no commit message specified?")
 	}
@@ -33,17 +33,8 @@ func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
 	}
 
 	if !dryRun {
-		err = commitWorktree.AddGlob(glob)
-		if err != nil {
-			return err
-		}
-		logrus.Debugf("staged %s for commit", glob)
-	} else {
-		logrus.Debugf("dry run: did not stage %s for commit", glob)
-	}
-
-	if !dryRun {
-		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
+		// NOTE(d1): `All: true` does not include untracked files
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{All: true})
 		if err != nil {
 			return err
 		}

pkg/git/diff.go

@@ -0,0 +1,42 @@
+package git
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+)
+
+// getGitDiffArgs builds the `git diff` invocation args. It removes the usage
+// of a pager and ensures that colours are specified even when Git might detect
+// otherwise.
+func getGitDiffArgs(repoPath string) []string {
+	return []string{
+		"-C",
+		repoPath,
+		"--no-pager",
+		"-c",
+		"color.diff=always",
+		"diff",
+	}
+}
+
+// DiffUnstaged shows a `git diff`. Due to limitations in the underlying go-git
+// library, this implementation requires the /usr/bin/git binary. It gracefully
+// skips if it cannot find the command on the system.
+func DiffUnstaged(path string) error {
+	if _, err := exec.LookPath("git"); err != nil {
+		logrus.Warnf("unable to locate git command, cannot output diff")
+		return nil
+	}
+
+	gitDiffArgs := getGitDiffArgs(path)
+	diff, err := exec.Command("git", gitDiffArgs...).Output()
+	if err != nil {
+		return nil
+	}
+
+	fmt.Print(string(diff))
+
+	return nil
+}

pkg/jsontable/jsontable.go

@@ -3,6 +3,7 @@ package jsontable
 import (
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/olekukonko/tablewriter"
 )
@@ -109,6 +110,9 @@ func (t *JSONTable) _JSONRenderInner() {
 		}
 		writeChar(t.out, '{')
 		for keyidx, key := range t.keys {
+			key := strings.ToLower(key)
+			key = strings.ReplaceAll(key, " ", "-")
+
 			value := "nil"
 			if keyidx < len(row) {
 				value = row[keyidx]
@@ -138,10 +142,8 @@ func (t *JSONTable) JSONRender() {
 
 		if t.hasCaption {
 			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
-
 		}
 		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
-
 	}
 
 	// write list
@@ -191,6 +193,12 @@ func (t *JSONTable) SetAutoMergeCellsByColumnIndex(cols []int) {
 	t.tbl.SetAutoMergeCellsByColumnIndex(cols)
 }
 
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAlignment(align int) {
+	// FIXME
+	t.tbl.SetAlignment(align)
+}
+
 func (t *JSONTable) SetAutoMergeCells(auto bool) {
 	// FIXME
 	t.tbl.SetAutoMergeCells(auto)

pkg/lint/recipe.go

@@ -11,6 +11,8 @@ import (
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/docker/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
 
@@ -151,6 +153,13 @@ var LintRules = map[string][]LintRule{
 			HowToResolve: "vendor config versions in an abra.sh",
 			Function:     LintAbraShVendors,
 		},
+		{
+			Ref:          "R014",
+			Level:        "error",
+			Description:  "only annotated tags used for recipe version",
+			HowToResolve: "replace lightweight tag with annotated tag",
+			Function:     LintValidTags,
+		},
 	},
 }
 
@@ -324,7 +333,7 @@ func LintImagePresent(recipe recipe.Recipe) (bool, error) {
 }
 
 func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
-	catl, err := recipePkg.ReadRecipeCatalogue()
+	catl, err := recipePkg.ReadRecipeCatalogue(false)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -391,3 +400,34 @@ func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
 
 	return true, nil
 }
+
+func LintValidTags(recipe recipe.Recipe) (bool, error) {
+	recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return false, fmt.Errorf("unable to open %s: %s", recipeDir, err)
+	}
+
+	iter, err := repo.Tags()
+	if err != nil {
+		logrus.Fatalf("unable to list local tags for %s", recipe.Name)
+	}
+
+	if err := iter.ForEach(func(ref *plumbing.Reference) error {
+		_, err := repo.TagObject(ref.Hash())
+		if err != nil {
+			switch err {
+			case plumbing.ErrObjectNotFound:
+				return fmt.Errorf("invalid lightweight tag detected")
+			default:
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}

pkg/recipe/recipe.go

@@ -2,15 +2,15 @@ package recipe
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
+	"sort"
 	"strconv"
 	"strings"
-	"time"
 
 	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/compose"
@@ -18,14 +18,13 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/limit"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/abra/pkg/web"
+	"coopcloud.tech/tagcmp"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
-	gitConfig "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
@@ -33,7 +32,7 @@ import (
 // RecipeCatalogueURL is the only current recipe catalogue available.
 const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
 
-// ReposMetadataURL is the recipe repository metadata
+// ReposMetadataURL is the recipe repository metadata.
 const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
 
 // tag represents a git tag.
@@ -65,6 +64,11 @@ type RecipeMeta struct {
 	Website       string         `json:"website"`
 }
 
+// TopicMeta represents a list of topics for a repository.
+type TopicMeta struct {
+	Topics []string `json:"topics"`
+}
+
 // LatestVersion returns the latest version of a recipe.
 func (r RecipeMeta) LatestVersion() string {
 	var version string
@@ -207,8 +211,8 @@ func (r Recipe) Tags() ([]string, error) {
 }
 
 // Get retrieves a recipe.
-func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
-	if err := EnsureExists(recipeName, conf); err != nil {
+func Get(recipeName string, offline bool) (Recipe, error) {
+	if err := EnsureExists(recipeName); err != nil {
 		return Recipe{}, err
 	}
 
@@ -234,11 +238,12 @@ func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
 		return Recipe{}, err
 	}
 
-	meta, err := GetRecipeMeta(recipeName, conf)
+	meta, err := GetRecipeMeta(recipeName, offline)
 	if err != nil {
-		if strings.Contains(err.Error(), "does not exist") {
+		switch err.(type) {
+		case RecipeMissingFromCatalogue:
 			meta = RecipeMeta{}
-		} else {
+		default:
 			return Recipe{}, err
 		}
 	}
@@ -250,12 +255,17 @@ func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
 	}, nil
 }
 
-// EnsureExists ensures that a recipe is locally cloned
-func EnsureExists(recipeName string, conf *runtime.Config) error {
-	if !conf.EnsureRecipeExists {
-		return nil
+func (r Recipe) SampleEnv() (map[string]string, error) {
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	sampleEnv, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}
+	return sampleEnv, nil
+}
 
+// EnsureExists ensures that a recipe is locally cloned
+func EnsureExists(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
@@ -277,15 +287,6 @@ func EnsureExists(recipeName string, conf *runtime.Config) error {
 func EnsureVersion(recipeName, version string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
-	isClean, err := gitPkg.IsClean(recipeDir)
-	if err != nil {
-		return err
-	}
-
-	if !isClean {
-		return fmt.Errorf("%s has locally unstaged changes", recipeName)
-	}
-
 	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
@@ -312,10 +313,13 @@ func EnsureVersion(recipeName, version string) error {
 		return err
 	}
 
-	logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
+	joinedTags := strings.Join(parsedTags, ", ")
+	if joinedTags != "" {
+		logrus.Debugf("read %s as tags for recipe %s", joinedTags, recipeName)
+	}
 
 	if tagRef.String() == "" {
-		return fmt.Errorf("no published release discovered for %s", recipeName)
+		return fmt.Errorf("the local copy of %s doesn't seem to have version %s available?", recipeName, version)
 	}
 
 	worktree, err := repo.Worktree()
@@ -337,29 +341,31 @@ func EnsureVersion(recipeName, version string) error {
 	return nil
 }
 
-// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
-func EnsureLatest(recipeName string, conf *runtime.Config) error {
-	if !conf.EnsureRecipeLatest {
-		return nil
-	}
-
+// EnsureIsClean makes sure that the recipe repository has no unstaged changes.
+func EnsureIsClean(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	isClean, err := gitPkg.IsClean(recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to check git clean status in %s: %s", recipeDir, err)
 	}
 
 	if !isClean {
-		return fmt.Errorf("%s has locally unstaged changes", recipeName)
+		msg := "%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, recipeName, recipeDir)
 	}
 
+	return nil
+}
+
+// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
+func EnsureLatest(recipeName string) error {
+	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+
 	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
 
-	logrus.Debugf("attempting to open git repository in %s", recipeDir)
-
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return err
@@ -370,21 +376,11 @@ func EnsureLatest(recipeName string, conf *runtime.Config) error {
 		return err
 	}
 
-	meta, err := GetRecipeMeta(recipeName, conf)
+	branch, err := gitPkg.GetDefaultBranch(repo, recipeDir)
 	if err != nil {
 		return err
 	}
 
-	var branch plumbing.ReferenceName
-	if meta.DefaultBranch != "" {
-		branch = plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", meta.DefaultBranch))
-	} else {
-		branch, err = gitPkg.GetDefaultBranch(repo, recipeDir)
-		if err != nil {
-			return err
-		}
-	}
-
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
 		Force:  true,
@@ -441,7 +437,7 @@ func GetVersionLabelLocal(recipe Recipe) (string, error) {
 
 	for _, service := range recipe.Config.Services {
 		for label, value := range service.Deploy.Labels {
-			if strings.HasPrefix(label, "coop-cloud") {
+			if strings.HasPrefix(label, "coop-cloud") && strings.Contains(label, "version") {
 				return value, nil
 			}
 		}
@@ -587,23 +583,9 @@ func GetStringInBetween(recipeName, str, start, end string) (result string, err
 }
 
 // EnsureUpToDate ensures that the local repo is synced to the remote
-func EnsureUpToDate(recipeName string, conf *runtime.Config) error {
-	if !conf.EnsureRecipeLatest {
-		return nil
-	}
-
+func EnsureUpToDate(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
-	isClean, err := gitPkg.IsClean(recipeDir)
-	if err != nil {
-		return fmt.Errorf("unable to check git clean status in %s: %s", recipeDir, err)
-	}
-
-	if !isClean {
-		msg := "%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding"
-		return fmt.Errorf(msg, recipeName, recipeDir)
-	}
-
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return fmt.Errorf("unable to open %s: %s", recipeDir, err)
@@ -629,12 +611,7 @@ func EnsureUpToDate(recipeName string, conf *runtime.Config) error {
 		return fmt.Errorf("unable to check out default branch in %s: %s", recipeDir, err)
 	}
 
-	fetchOpts := &git.FetchOptions{
-		Tags: git.AllTags,
-		RefSpecs: []gitConfig.RefSpec{
-			gitConfig.RefSpec(fmt.Sprintf("%s:%s", branch, branch)),
-		},
-	}
+	fetchOpts := &git.FetchOptions{Tags: git.AllTags}
 	if err := repo.Fetch(fetchOpts); err != nil {
 		if !strings.Contains(err.Error(), "already up-to-date") {
 			return fmt.Errorf("unable to fetch tags in %s: %s", recipeDir, err)
@@ -658,78 +635,20 @@ func EnsureUpToDate(recipeName string, conf *runtime.Config) error {
 	return nil
 }
 
-type CatalogueOfflineError struct {
-	msg string
-}
-
-func (e *CatalogueOfflineError) Error() string {
-	return fmt.Sprintf("catalogue offline: %s", e.msg)
-}
-
-// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
-// is up to date.
-func recipeCatalogueFSIsLatest() (bool, error) {
-	httpClient := web.NewHTTPRetryClient()
-	res, err := httpClient.Head(RecipeCatalogueURL)
-	if err != nil {
-		return false, &CatalogueOfflineError{err.Error()}
-	}
-
-	lastModified := res.Header["Last-Modified"][0]
-	parsed, err := time.Parse(time.RFC1123, lastModified)
-	if err != nil {
-		return false, err
-	}
-
-	info, err := os.Stat(config.RECIPES_JSON)
-	if err != nil {
-		if os.IsNotExist(err) {
-			logrus.Debugf("no recipe catalogue found in file system cache")
-			return false, nil
-		}
-		return false, err
-	}
-
-	localModifiedTime := info.ModTime().Unix()
-	remoteModifiedTime := parsed.Unix()
-
-	if localModifiedTime < remoteModifiedTime {
-		logrus.Debug("file system cached recipe catalogue is out-of-date")
-		return false, nil
-	}
-
-	logrus.Debug("file system cached recipe catalogue is up-to-date")
-
-	return true, nil
-}
-
 // ReadRecipeCatalogue reads the recipe catalogue.
-func ReadRecipeCatalogue() (RecipeCatalogue, error) {
+func ReadRecipeCatalogue(offline bool) (RecipeCatalogue, error) {
 	recipes := make(RecipeCatalogue)
 
 	if err := catalogue.EnsureCatalogue(); err != nil {
 		return nil, err
 	}
 
-	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
-	if err != nil {
-		var offlineErr *CatalogueOfflineError
-		if errors.As(err, &offlineErr) {
-			logrus.Error(err)
-			logrus.Error("unable to retrieve catalogue from internet, using local copy.")
-			recipeFSIsLatest = true
-		} else {
+	if !offline {
+		if err := catalogue.EnsureUpToDate(); err != nil {
 			return nil, err
 		}
 	}
 
-	if !recipeFSIsLatest {
-		if err := readRecipeCatalogueWeb(&recipes); err != nil {
-			return nil, err
-		}
-		return recipes, nil
-	}
-
 	if err := readRecipeCatalogueFS(&recipes); err != nil {
 		return nil, err
 	}
@@ -753,31 +672,11 @@ func readRecipeCatalogueFS(target interface{}) error {
 	return nil
 }
 
-// readRecipeCatalogueWeb reads the catalogue from the web.
-func readRecipeCatalogueWeb(target interface{}) error {
-	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
-		return err
-	}
-
-	recipesJSON, err := json.MarshalIndent(target, "", "    ")
-	if err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
-		return err
-	}
-
-	logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL)
-
-	return nil
-}
-
 // VersionsOfService lists the version of a service.
-func VersionsOfService(recipe, serviceName string) ([]string, error) {
+func VersionsOfService(recipe, serviceName string, offline bool) ([]string, error) {
 	var versions []string
 
-	catalogue, err := ReadRecipeCatalogue()
+	catalogue, err := ReadRecipeCatalogue(offline)
 	if err != nil {
 		return nil, err
 	}
@@ -802,20 +701,26 @@ func VersionsOfService(recipe, serviceName string) ([]string, error) {
 	return versions, nil
 }
 
+// RecipeMissingFromCatalogue signifies a recipe is not present in the catalogue.
+type RecipeMissingFromCatalogue struct{ err string }
+
+// Error outputs the error message.
+func (r RecipeMissingFromCatalogue) Error() string {
+	return r.err
+}
+
 // GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
-func GetRecipeMeta(recipeName string, conf *runtime.Config) (RecipeMeta, error) {
-	catl, err := ReadRecipeCatalogue()
+func GetRecipeMeta(recipeName string, offline bool) (RecipeMeta, error) {
+	catl, err := ReadRecipeCatalogue(offline)
 	if err != nil {
 		return RecipeMeta{}, err
 	}
 
 	recipeMeta, ok := catl[recipeName]
 	if !ok {
-		return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
-	}
-
-	if err := EnsureExists(recipeName, conf); err != nil {
-		return RecipeMeta{}, err
+		return RecipeMeta{}, RecipeMissingFromCatalogue{
+			err: fmt.Sprintf("recipe %s does not exist?", recipeName),
+		}
 	}
 
 	logrus.Debugf("recipe metadata retrieved for %s", recipeName)
@@ -923,7 +828,16 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 		}
 
 		for idx, repo := range reposList {
-			reposMeta[repo.Name] = reposList[idx]
+			var topicMeta TopicMeta
+
+			topicsURL := getReposTopicUrl(repo.Name)
+			if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
+				return reposMeta, err
+			}
+
+			if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
+				reposMeta[repo.Name] = reposList[idx]
+			}
 		}
 
 		pageIdx++
@@ -936,7 +850,7 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 }
 
 // GetRecipeVersions retrieves all recipe versions.
-func GetRecipeVersions(recipeName string, conf *runtime.Config) (RecipeVersions, error) {
+func GetRecipeVersions(recipeName string, offline bool) (RecipeVersions, error) {
 	versions := RecipeVersions{}
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
@@ -974,7 +888,7 @@ func GetRecipeVersions(recipeName string, conf *runtime.Config) (RecipeVersions,
 
 		logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
 
-		recipe, err := Get(recipeName, conf)
+		recipe, err := Get(recipeName, offline)
 		if err != nil {
 			return err
 		}
@@ -1017,12 +931,52 @@ func GetRecipeVersions(recipeName string, conf *runtime.Config) (RecipeVersions,
 	if err != nil {
 		return versions, err
 	}
+	sortRecipeVersions(versions)
 
 	logrus.Debugf("collected %s for %s", versions, recipeName)
 
 	return versions, nil
 }
 
+// sortRecipeVersions sorts the recipe semver versions
+func sortRecipeVersions(versions RecipeVersions) {
+	sort.Slice(versions, func(i, j int) bool {
+		version1, err := tagcmp.Parse(getVersionString(versions[i]))
+		if err != nil {
+			panic(err)
+		}
+		version2, err := tagcmp.Parse(getVersionString(versions[j]))
+		if err != nil {
+			panic(err)
+		}
+		return version1.IsLessThan(version2)
+	})
+}
+
+// getVersionString returns the version string from RecipeVersions
+func getVersionString(versionMap map[string]map[string]ServiceMeta) string {
+	// Assuming there's only one key in versionMap
+	for k := range versionMap {
+		return k
+	}
+	return ""
+}
+
+// sortVersionStrings sorts a list of semver version strings
+func sortVersionStrings(versions []string) {
+	sort.Slice(versions, func(i, j int) bool {
+		version1, err := tagcmp.Parse(versions[i])
+		if err != nil {
+			panic(err)
+		}
+		version2, err := tagcmp.Parse(versions[j])
+		if err != nil {
+			panic(err)
+		}
+		return version1.IsLessThan(version2)
+	})
+}
+
 // GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
 func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
 	var versions []string
@@ -1035,11 +989,13 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri
 		}
 	}
 
+	sortVersionStrings(versions)
+
 	return versions, nil
 }
 
 // UpdateRepositories clones and updates all recipe repositories locally.
-func UpdateRepositories(repos RepoCatalogue, recipeName string, conf *runtime.Config) error {
+func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 	var barLength int
 	if recipeName != "" {
 		barLength = 1
@@ -1061,22 +1017,12 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string, conf *runtime.Co
 				retrieveBar.Add(1)
 				return
 			}
-			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
 
 			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
 			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
 				logrus.Fatal(err)
 			}
 
-			if err := EnsureUpToDate(rm.Name, conf); err != nil {
-				logrus.Fatal(err)
-			}
-
 			ch <- rm.Name
 			retrieveBar.Add(1)
 		}(repoMeta)
@@ -1088,3 +1034,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string, conf *runtime.Co
 
 	return nil
 }
+
+// getReposTopicUrl retrieves the repository specific topic listing.
+func getReposTopicUrl(repoName string) string {
+	return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
+}

pkg/recipe/recipe_test.go

@@ -0,0 +1,31 @@
+package recipe
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetVersionLabelLocalDoesNotUseTimeoutLabel(t *testing.T) {
+	offline := true
+	recipe, err := Get("traefik", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 1; i < 1000; i++ {
+		label, err := GetVersionLabelLocal(recipe)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// NOTE(d1): this is potentially quite a brittle unit test as it needs to
+		// hardcode the default timeout label to ensure that the label parser never
+		// returns it. hopefully this won't fail too often! if you're here because
+		// of a failure, just update the `defaultTimeoutLabel` value & permalink
+		// below
+		// https://git.coopcloud.tech/coop-cloud/traefik/src/commit/ac3a47fe8ca3ef92db84f64cfedfbb348000faee/.env.sample#L2
+		defaultTimeoutLabel := "300"
+		assert.NotEqual(t, label, defaultTimeoutLabel)
+	}
+}

pkg/runtime/runtime.go

@@ -1,61 +0,0 @@
-package runtime
-
-import "github.com/sirupsen/logrus"
-
-// Config is an internal configuration modifier. It can be instantiated on a
-// command call and can be changed on the fly to help make decisions further
-// down in the internals, e.g. whether or not to clone the recipe locally or
-// not.
-type Config struct {
-	EnsureRecipeExists bool // ensure that the recipe is cloned locally
-	EnsureRecipeLatest bool // ensure the local recipe has latest changes
-}
-
-// Option modified a Config. The convetion for passing Options to functions is
-// so far, only used in internal/validate.go. A Config is then constructed and
-// passed down further into the code. This may change in the future but this is
-// at least the abstraction so far.
-type Option func(c *Config)
-
-// New instantiates a new Config.
-func New(opts ...Option) *Config {
-	conf := &Config{
-		EnsureRecipeExists: true,
-	}
-
-	for _, optFunc := range opts {
-		optFunc(conf)
-	}
-
-	return conf
-}
-
-// WithEnsureRecipeExists determines whether or not we should be cloning the
-// local recipe or not. This can be useful for being more adaptable to offline
-// scenarios.
-func WithEnsureRecipeExists(ensureRecipeExists bool) Option {
-	return func(c *Config) {
-		if ensureRecipeExists {
-			logrus.Debugf("runtime config: EnsureRecipeExists = %v, ensuring recipes are cloned", ensureRecipeExists)
-		} else {
-			logrus.Debugf("runtime config: EnsureRecipeExists = %v, not cloning recipes", ensureRecipeExists)
-		}
-
-		c.EnsureRecipeExists = ensureRecipeExists
-	}
-}
-
-// WithEnsureRecipeLatest determines whether we should update the local recipes
-// remotely via Git. This can be useful when e.g. ensuring we have the latest
-// changes before making new ones.
-func WithEnsureRecipeLatest(ensureRecipeLatest bool) Option {
-	return func(c *Config) {
-		if ensureRecipeLatest {
-			logrus.Debugf("runtime config: EnsureRecipeLatest = %v, ensuring recipes have latest changes", ensureRecipeLatest)
-		} else {
-			logrus.Debugf("runtime config: EnsureRecipeLatest = %v, leaving recipes alone", ensureRecipeLatest)
-		}
-
-		c.EnsureRecipeLatest = ensureRecipeLatest
-	}
-}

pkg/secret/secret.go

@@ -4,22 +4,26 @@
 package secret
 
 import (
+	"context"
 	"fmt"
-	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/decentral1se/passgen"
+	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
-// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
+// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
 // secret definition.
-type secretValue struct {
+type SecretValue struct {
 	Version string
 	Length  int
 }
@@ -31,7 +35,6 @@ func GeneratePasswords(count, length uint) ([]string, error) {
 		length,
 		passgen.AlphabetDefault,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -50,7 +53,6 @@ func GeneratePassphrases(count uint) ([]string, error) {
 		passgen.PassphraseCasingDefault,
 		passgen.WordListDefault,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -60,88 +62,93 @@ func GeneratePassphrases(count uint) ([]string, error) {
 	return passphrases, nil
 }
 
-// ReadSecretEnvVars reads secret env vars from an app env var config.
-func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
-	secretEnvVars := make(map[string]string)
+// ReadSecretsConfig reads secret names/versions from the recipe config. The
+// function generalises appEnv/composeFiles because some times you have an app
+// and some times you don't (as the caller). We need to be able to handle the
+// "app new" case where we pass in the .env.sample and the "secret generate"
+// case where the app is created.
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
+	appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
+	if err != nil {
+		return nil, err
+	}
 
-	for envVar := range appEnv {
-		regex := regexp.MustCompile(`^SECRET.*VERSION.*`)
-		if string(regex.Find([]byte(envVar))) != "" {
-			secretEnvVars[envVar] = appEnv[envVar]
+	opts := stack.Deploy{Composefiles: composeFiles}
+	config, err := loader.LoadComposefile(opts, appEnv)
+	if err != nil {
+		return nil, err
+	}
+	configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
+	if err != nil {
+		return nil, err
+	}
+
+	var enabledSecrets []string
+	for _, service := range config.Services {
+		for _, secret := range service.Secrets {
+			enabledSecrets = append(enabledSecrets, secret.Source)
 		}
 	}
 
-	logrus.Debugf("read %s as secrets from %s", secretEnvVars, appEnv)
-
-	return secretEnvVars
-}
-
-func ParseSecretEnvVarName(secretEnvVar string) string {
-	withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_")
-	withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION")
-	name := strings.ToLower(withoutSuffix)
-	logrus.Debugf("parsed %s as name from %s", name, secretEnvVar)
-	return name
-}
-
-func ParseGeneratedSecretName(secret string, appEnv config.App) string {
-	name := fmt.Sprintf("%s_", appEnv.StackName())
-	withoutAppName := strings.TrimPrefix(secret, name)
-	idx := strings.LastIndex(withoutAppName, "_")
-	parsed := withoutAppName[:idx]
-	logrus.Debugf("parsed %s as name from %s", parsed, secret)
-	return parsed
-}
-
-func ParseSecretEnvVarValue(secret string) (secretValue, error) {
-	values := strings.Split(secret, "#")
-	if len(values) == 0 {
-		return secretValue{}, fmt.Errorf("unable to parse %s", secret)
+	if len(enabledSecrets) == 0 {
+		logrus.Debugf("not generating app secrets, none enabled in recipe config")
+		return nil, nil
 	}
 
-	if len(values) == 1 {
-		return secretValue{Version: values[0], Length: 0}, nil
+	secretValues := map[string]SecretValue{}
+	for secretId, secretConfig := range config.Secrets {
+		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
+			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
+		}
+
+		if !(slices.Contains(enabledSecrets, secretId)) {
+			logrus.Warnf("%s not enabled in recipe config, skipping", secretId)
+			continue
+		}
+
+		lastIdx := strings.LastIndex(secretConfig.Name, "_")
+		secretVersion := secretConfig.Name[lastIdx+1:]
+		value := SecretValue{Version: secretVersion}
+
+		// Check if the length modifier is set for this secret.
+		for k, v := range appModifiers {
+			// configWithoutEnv contains the raw name as defined in the compose.yaml
+			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
+				continue
+			}
+			lengthRaw, ok := v["length"]
+			if ok {
+				length, err := strconv.Atoi(lengthRaw)
+				if err != nil {
+					return nil, err
+				}
+				value.Length = length
+			}
+			break
+		}
+		secretValues[secretId] = value
 	}
 
-	split := strings.Split(values[1], "=")
-	parsed := split[len(split)-1]
-	stripped := strings.ReplaceAll(parsed, " ", "")
-	length, err := strconv.Atoi(stripped)
-	if err != nil {
-		return secretValue{}, err
-	}
-	version := strings.ReplaceAll(values[0], " ", "")
-
-	logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
-
-	return secretValue{Version: version, Length: length}, nil
+	return secretValues, nil
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
-	secrets := make(map[string]string)
-
+func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
+	secretsGenerated := map[string]string{}
 	var mutex sync.Mutex
 	var wg sync.WaitGroup
-	ch := make(chan error, len(secretEnvVars))
-	for secretEnvVar := range secretEnvVars {
+	ch := make(chan error, len(secrets))
+	for n, v := range secrets {
 		wg.Add(1)
 
-		go func(s string) {
+		go func(secretName string, secret SecretValue) {
 			defer wg.Done()
 
-			secretName := ParseSecretEnvVarName(s)
-			secretValue, err := ParseSecretEnvVarValue(secretEnvVars[s])
-			if err != nil {
-				ch <- err
-				return
-			}
-
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version)
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
 			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
 
-			if secretValue.Length > 0 {
-				passwords, err := GeneratePasswords(1, uint(secretValue.Length))
+			if secret.Length > 0 {
+				passwords, err := GeneratePasswords(1, uint(secret.Length))
 				if err != nil {
 					ch <- err
 					return
@@ -159,7 +166,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, a
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secrets[secretName] = passwords[0]
+				secretsGenerated[secretName] = passwords[0]
 			} else {
 				passphrases, err := GeneratePassphrases(1)
 				if err != nil {
@@ -179,22 +186,80 @@ func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, a
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secrets[secretName] = passphrases[0]
+				secretsGenerated[secretName] = passphrases[0]
 			}
 			ch <- nil
-		}(secretEnvVar)
+		}(n, v)
 	}
 
 	wg.Wait()
 
-	for range secretEnvVars {
+	for range secrets {
 		err := <-ch
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	logrus.Debugf("generated and stored %s on %s", secrets, server)
+	logrus.Debugf("generated and stored %v on %s", secrets, server)
 
-	return secrets, nil
+	return secretsGenerated, nil
+}
+
+type secretStatus struct {
+	LocalName       string
+	RemoteName      string
+	Version         string
+	CreatedOnRemote bool
+}
+
+type secretStatuses []secretStatus
+
+// PollSecretsStatus checks status of secrets by comparing the local recipe
+// config and deploymend server state.
+func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, error) {
+	var secStats secretStatuses
+
+	composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+	if err != nil {
+		return secStats, err
+	}
+
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+	if err != nil {
+		return secStats, err
+	}
+
+	remoteSecretNames := make(map[string]bool)
+	for _, cont := range secretList {
+		remoteSecretNames[cont.Spec.Annotations.Name] = true
+	}
+
+	for secretName, val := range secretsConfig {
+		createdRemote := false
+
+		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+		if _, ok := remoteSecretNames[secretRemoteName]; ok {
+			createdRemote = true
+		}
+
+		secStats = append(secStats, secretStatus{
+			LocalName:       secretName,
+			RemoteName:      secretRemoteName,
+			Version:         val.Version,
+			CreatedOnRemote: createdRemote,
+		})
+	}
+
+	return secStats, nil
 }

pkg/secret/secret_test.go

@@ -0,0 +1,42 @@
+package secret
+
+import (
+	"path"
+	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadSecretsConfig(t *testing.T) {
+	offline := true
+	recipe, err := recipe.Get("matrix-synapse", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sampleEnv, err := recipe.SampleEnv()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	config, err := loader.LoadComposefile(opts, sampleEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for secretId := range config.Secrets {
+		assert.Contains(t, secretsFromConfig, secretId)
+	}
+}

pkg/upstream/convert/service.go

@@ -420,7 +420,7 @@ func convertServiceSecrets(
 			return nil, err
 		}
 
-		// NOTE(d1): strip # length=... modifiers
+		// NOTE(d1): strip all comments
 		if strings.Contains(obj.Name, "#") {
 			vals := strings.Split(obj.Name, "#")
 			obj.Name = strings.TrimSpace(vals[0])

pkg/upstream/stack/loader.go

@@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) {
 	opts.SkipValidation = false
 }
 
+// SkipInterpolation skip interpolating environment variables.
+func SkipInterpolation(opts *loader.Options) {
+	opts.SkipInterpolation = true
+}
+
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
-func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
+func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
 	if err != nil {
 		return nil, err
 	}
 
+	if options == nil {
+		options = []func(*loader.Options){DontSkipValidation}
+	}
+
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails, DontSkipValidation)
+	config, err := loader.Load(configDetails, options...)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
 			return nil, fmt.Errorf("compose file contains unsupported options: %s",

pkg/upstream/stack/stack.go

@@ -5,7 +5,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"strings"
+	"os"
+	"os/signal"
 	"time"
 
 	"coopcloud.tech/abra/pkg/upstream/convert"
@@ -31,6 +32,9 @@ const (
 	ResolveImageNever    = "never"
 )
 
+// Timeout to wait until docker services converge, default is 50s (random choice)
+var WaitTimeout int = 50
+
 type StackStatus struct {
 	Services []swarm.Service
 	Err      error
@@ -411,7 +415,7 @@ func deployServices(
 		return nil
 	}
 
-	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
+	logrus.Infof("Waiting for %s to deploy... please hold 🤚", appName)
 	ch := make(chan error, len(serviceIDs))
 	for serviceID, serviceName := range serviceIDs {
 		logrus.Debugf("waiting on %s to converge", serviceName)
@@ -428,7 +432,7 @@ func deployServices(
 		logrus.Debugf("assuming %s converged successfully", serviceID)
 	}
 
-	logrus.Info("services converged 👌")
+	logrus.Infof("Successfully deployed %s", appName)
 
 	return nil
 }
@@ -451,17 +455,27 @@ func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appN
 	errChan := make(chan error, 1)
 	pipeReader, pipeWriter := io.Pipe()
 
+	sigintChannel := make(chan os.Signal, 1)
+	signal.Notify(sigintChannel, os.Interrupt)
+	defer signal.Stop(sigintChannel)
+
 	go func() {
 		errChan <- progress.ServiceProgress(ctx, cl, serviceID, pipeWriter)
 	}()
 
 	go io.Copy(ioutil.Discard, pipeReader)
 
-	timeout := 50 * time.Second
+	timeout := time.Duration(WaitTimeout) * time.Second
 
 	select {
 	case err := <-errChan:
 		return err
+	case <-sigintChannel:
+		return fmt.Errorf(fmt.Sprintf(`
+Not waiting for %s to deploy. The deployment is ongoing...
+
+If you want to stop the deployment, try:
+  abra app undeploy %s`, appName, appName))
 	case <-time.After(timeout):
 		return fmt.Errorf(fmt.Sprintf(`
 %s has not converged (%s second timeout reached).
@@ -478,7 +492,7 @@ And inspect the logs with:
 
     abra app logs %s
 
-If a service is failing to even start, try smoke out the error with:
+If a service is failing to even start, try to smoke out the error with:
 
     abra app errors --watch %s
 

renovate.json

@@ -8,6 +8,7 @@
     "gomodTidy"
   ],
   "ignoreDeps": [
-    "github.com/urfave/cli"
+    "github.com/urfave/cli",
+    "goreleaser/goreleaser"
   ]
 }

scripts/docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ ! -f .envrc ]; then
+    . .envrc.sample
+else
+    . .envrc
+fi
+
+git config --global --add safe.directory /abra  # work around funky file permissions
+
+make build

scripts/installer/installer

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.7.0-beta"
+ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.7.0-beta"
+RC_VERSION="0.8.1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do

scripts/release/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+for f in $(find ./tests/integration -name "*.bats"); do
+  bats -Tp "$f"
+  res=$?
+  if [[ "$res" -ne 0 ]]; then
+    exit 1
+  fi
+done

tests/README.md

@@ -0,0 +1,10 @@
+# Test suite
+
+* Unit testing is done in the packages themselves, run `find . -name
+  "*_test.go"` to find those. Use `make test` to run the entire unit test
+  suite. Some unit tests require mocked files which are located in
+  `./tests/resources.`
+
+* Integration tests are in `./tests/integration`. Please see [these
+  docs](https://docs.coopcloud.tech/abra/hack/#integration-tests) for
+  instructions and tips on how to run them.

tests/integration/.envrc.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

tests/integration/.gitignore

@@ -1 +0,0 @@
-logs

tests/integration/README.md

@@ -1,28 +0,0 @@
-# integration tests
-
-> You need to be a member of Autonomic Co-op to run these tests, sorry!
-
-`testfunctions.sh` contains the functions necessary to save and manipulate
-logs. Run `test_all.sh logdir` to run tests specified in that file and save the
-logs to `logdir`.
-
-When creating new tests, make sure the test command is a one-liner (you can use
-`;` to separate commands). Include `testfunctions.sh` and then write your tests
-like this:
-
-```
-run_test '$ABRA other stuff here'
-```
-
-By default, the testing script will ask after every command if the execution
-succeeded. If you reply `n`, it will log the test in the `logdir`. If you want
-all tests to run without questions, run `export logall=yes` before executing
-the test script.
-
-To run tests, you'll need to prepare your environment:
-
-```
-cp .envrc.sample .envrc # fill out values...
-direnv allow
-./test_all.sh logs
-```

tests/integration/app.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-source ./testfunctions.sh
-source ./common.sh
-
-run_test '$ABRA app ls'
-
-run_test '$ABRA app ls --status'
-
-run_test '$ABRA app ls --type wordpress'
-
-run_test '$ABRA app ls --type wordpress --server swarm.autonomic.zone'
-
-run_test '$ABRA app ls --type wordpress --server swarm.autonomic.zone --status'

tests/integration/app_backup.bats

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+setup_file() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file() {
+  _rm_app
+  _rm_server
+  _reset_recipe
+}
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "retrieve recipe if missing" {
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app backup "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'no containers matching'
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --chaos --no-input
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_success
+  assert_output --partial 'running backup for the app service'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe
+}
+
+@test "ensure recipe not up to date if --offline" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  refute [ -z "$latestCommit" ];
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug --offline
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe
+}
+
+@test "detect backup labels" {
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  assert_output --partial 'detected backup paths'
+  assert_output --partial 'detected pre-hook command'
+  assert_output --partial 'detected post-hook command'
+}
+
+@test "error if backups not enabled" {
+  run sed -i '/backupbot.backup=true/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_failure
+  assert_output --partial 'no backup config for app'
+
+  _checkout_recipe
+}
+
+@test "error if backup paths not configured" {
+  run sed -i '/backupbot.backup.path=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_failure
+  assert_output --partial 'backup paths are empty for app?'
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "backup single service" {
+  _deploy_app
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app
+  assert_success
+  assert_output --partial 'running backup for the app service'
+
+  sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
+  assert_output --partial "_$sanitisedDomainName_app"
+
+  assert_exists "$ABRA_DIR/backups"
+  assert bash -c "ls $ABRA_DIR/backups | grep -q $1_$sanitisedDomainName_app"
+
+  _undeploy_app
+}

tests/integration/app_check.bats

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+  _reset_recipe
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app check
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app check DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+  refute_output --partial '❌'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --chaos
+  assert_success
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe
+}
+
+@test "ensure recipe not up to date if --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~1
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 1'
+
+  # NOTE(d1): we can't quite tell if this will fail or not in the future, so,
+  # since it isn't an important part of what we're testing here, we don't check
+  # it
+  run $ABRA app check "$TEST_APP_DOMAIN" --offline
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 1'
+
+  _reset_recipe
+}
+
+@test "error if missing .env.sample" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"
+  assert_success
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --chaos
+  assert_failure
+  assert_output --partial '.env.sample does not exist?'
+
+  _checkout_recipe
+}
+
+@test "error if missing env var" {
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+  refute_output --partial '❌'
+
+  run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
+  assert_success
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --chaos
+  assert_success
+  assert_output --partial '❌'
+
+  _checkout_recipe
+}

tests/integration/app_cmd.bats

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+  _reset_recipe
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+# bats test_tags=slow
+@test "autocomplete" {
+  run $ABRA app cmd --generate-bash-completion
+  assert_success
+  assert_output "$TEST_APP_DOMAIN"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" --generate-bash-completion
+  assert_success
+  assert_output "app"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" app --generate-bash-completion
+  assert_success
+  assert_output "test_cmd
+test_cmd_arg
+test_cmd_args
+test_cmd_export"
+}
+
+@test "validate app argument" {
+  run $ABRA app cmd
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app cmd DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_success
+  assert_output --partial 'baz'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
+  assert_success
+  assert_output --partial 'baz'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_success
+  assert_output --partial 'baz'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "ensure recipe not up to date if --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --offline
+  assert_success
+  assert_output --partial 'baz'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "error if missing arguments without passing --local" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing arguments'
+}
+
+@test "error if missing arguments when passing --local" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" --local
+  assert_failure
+  assert_output --partial 'missing arguments'
+}
+
+@test "cannot use --local and --user at same time" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --user root
+  assert_failure
+  assert_output --partial 'cannot use --local & --user together'
+}
+
+@test "error if missing abra.sh" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh"
+  assert_success
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
+  assert_failure
+  assert_output --partial "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh does not exist"
+
+  _checkout_recipe
+}
+
+@test "error if missing command" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist --local
+  assert_failure
+  assert_output --partial "doesn't have a doesnt_exist function"
+}
+
+@test "run --local command" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_success
+  assert_output --partial 'baz'
+}
+
+@test "run command with single arg" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_arg --local -- bing
+  assert_success
+  assert_output --partial 'bing'
+}
+
+@test "run command with several args" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_args --local -- bong bang
+  assert_success
+  assert_output --partial 'bong bang'
+}
+
+# bats test_tags=slow
+@test "run command on service" {
+  _deploy_app
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" app test_cmd
+  assert_success
+  assert_output --partial 'baz'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "error if missing service" {
+  _deploy_app
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist test_cmd
+  assert_failure
+  assert_output --partial 'no service doesnt_exist'
+
+  _undeploy_app
+}

tests/integration/app_config.bats

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+}
+
+teardown_file(){
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app config
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app config DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}

tests/integration/app_cp.bats

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app cp
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app cp DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if missing src/dest arguments" {
+  run $ABRA app cp "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing <src> argument'
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt
+  assert_failure
+  assert_output --partial 'missing <dest> argument'
+}
+
+@test "either src/dest has correct syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app
+  assert_failure
+  assert_output --partial 'arguments must take $SERVICE:$PATH form'
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app .
+  assert_failure
+  assert_output --partial 'arguments must take $SERVICE:$PATH form'
+}
+
+@test "detect 'coming FROM' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug
+  assert_failure
+  assert_output --partial 'coming FROM the container'
+}
+
+@test "detect 'going TO' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug
+  assert_failure
+  assert_output --partial 'going TO the container'
+}
+
+@test "error if local file missing" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere
+  assert_failure
+  assert_output --partial 'myfile.txt does not exist locally?'
+}
+
+# bats test_tags=slow
+@test "error if service doesn't exist" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "copy to container" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
+  assert_success
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "copy from container" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
+  assert_success
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR"
+  assert_success
+  assert_exists "$BATS_TMPDIR/myfile.txt"
+  assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
+}

tests/integration/app_deploy.bats

@@ -0,0 +1,364 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+  _reset_recipe
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app deploy
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app deploy DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+  refute_output --partial 'chaos'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --chaos --no-input --no-converge-checks
+  assert_success
+  assert_output --partial 'chaos'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+# bats test_tags=slow
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "ensure recipe not up to date if --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  # NOTE(d1): need to use --chaos to force same commit
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos --offline
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  _undeploy_app
+  _reset_recipe
+}
+
+# bats test_tags=slow
+@test "deploy latest commit if no published versions and no --chaos" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  _remove_tags
+
+  # NOTE(d1): need to pass --offline to stop tags being pulled again
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --offline
+  assert_success
+  assert_output --partial "$latestCommit"
+  assert_output --partial 'using latest commit'
+  refute_output --partial 'chaos'
+
+  _undeploy_app
+  _reset_tags
+}
+
+# bats test_tags=slow
+@test "ensure same commit if --chaos" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  refute_output --partial "$latestCommit"
+  assert_output --partial "$threeCommitsBack"
+  assert_output --partial 'chaos'
+
+  _undeploy_app
+  _reset_recipe
+}
+
+# bats test_tags=slow
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+
+  _undeploy_app
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "no deploy if lint error" {
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_failure
+  assert_output --partial 'failed lint checks'
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "error if already deployed and no --force/--chaos" {
+  _deploy_app
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'already deployed'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "re-deploy deployed app if --force/--chaos" {
+  _deploy_app
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --force
+  assert_success
+  assert_output --partial 'already deployed but continuing'
+  assert_output --partial '--force'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial 'already deployed but continuing'
+  assert_output --partial '--chaos'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "deploy latest version from catalogue if no --chaos" {
+  latestVersion=$(jq -r '.gitea.versions[-1] | keys[0]' < "$ABRA_DIR/catalogue/recipes.json")
+  refute [ -z "$latestVersion" ];
+
+  run $ABRA app new gitea \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "gitea.$TEST_SERVER" \
+    --secrets
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/gitea.$TEST_SERVER.env"
+
+  run $ABRA app deploy "gitea.$TEST_SERVER" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestVersion"
+
+  run $ABRA app undeploy "gitea.$TEST_SERVER" --no-input
+  assert_success
+
+  run $ABRA app secret remove "gitea.$TEST_SERVER" --all --no-input
+  assert_success
+
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume remove "gitea.$TEST_SERVER" --no-input
+  assert_success
+
+  run $ABRA app remove "gitea.$TEST_SERVER" --no-input
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/gitea.$TEST_SERVER.env"
+}
+
+# bats test_tags=slow
+@test "skip domain check if missing DOMAIN=" {
+  run cp "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" "$BATS_TMPDIR/$TEST_APP_DOMAIN.env"
+  assert_success
+  assert_exists "$BATS_TMPDIR/$TEST_APP_DOMAIN.env"
+
+  run grep -q "DOMAIN=" "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i '/DOMAIN=.*/d' "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run grep -q "DOMAIN=" "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_failure
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial 'no DOMAIN=... configured for app'
+
+  run $ABRA app undeploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run mv "$BATS_TMPDIR/$TEST_APP_DOMAIN.env" "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+}
+
+@test "ensure domain is checked" {
+  appDomain="custom-html.DOESNTEXIST"
+
+  run $ABRA app new custom-html \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$appDomain"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$appDomain.env"
+
+  run $ABRA app deploy "$appDomain" --no-input
+  assert_failure
+  assert_output --partial 'no such host'
+}
+
+# bats test_tags=slow
+@test "skip domain check when requested" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --no-domain-checks
+  assert_success
+  assert_output --partial 'skipping domain checks as requested'
+
+  _undeploy_app
+}
+
+@test "error if specific version does not exist" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" DOESNTEXIST --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial "doesn't seem to have version DOESNTEXIST available"
+}
+
+# bats test_tags=slow
+@test "deploy specific version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input
+  assert_success
+  assert_output --partial "0.2.0+1.21.0"
+
+  _undeploy_app
+}
+
+@test "bail out if specific version and chaos" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
+# bats test_tags=slow
+@test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
+  _reset_recipe
+
+  run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  # NOTE(d1): --chaos used to bypass versions and access compose.extra_env.yml
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial "compose.yml"
+  assert_output --partial "compose.extra_env.yml"
+
+  _undeploy_app
+  _reset_app
+}
+
+@test "error if no secrets generated" {
+  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'unable to deploy, secrets not generated'
+
+  _reset_app
+}
+
+@test "recipe config comments not present in values" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app env
+  assert_success
+  refute_output --partial 'should be removed'
+}

tests/integration/app_errors.bats

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app errors
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app errors DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app errors "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "report errors" {
+  _deploy_app
+
+  run $ABRA app errors "$TEST_APP_DOMAIN"
+  assert_success
+
+  _undeploy_app
+}

tests/integration/app_list.bats

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "list without status" {
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "$TEST_APP_DOMAIN"
+}
+
+# bats test_tags=slow
+@test "list with status" {
+  run $ABRA app ls --status
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "$TEST_APP_DOMAIN"
+  assert_output --partial "unknown"
+
+  _deploy_app
+
+  run $ABRA app ls --status
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "$TEST_APP_DOMAIN"
+  assert_output --partial "deployed"
+
+  _undeploy_app
+}
+
+@test "filter by server" {
+  run mkdir -p "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/foo.com"
+
+  run cp \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" \
+    "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert_exists "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "foo.com"
+
+  run $ABRA app ls --server foo.com
+  assert_success
+  refute_output --partial "server: $TEST_SERVER |"
+  assert_output --partial "server: foo.com |"
+
+  run rm -rf "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/foo.com"
+}
+
+@test "filter by recipe" {
+  run mkdir -p "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/foo.com"
+
+  run cp \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" \
+    "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert_exists "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run sed -i "s/TYPE=$TEST_RECIPE/TYPE=foo-recipe/g" "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert grep -q "TYPE=foo-recipe" "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_RECIPE"
+  assert_output --partial "foo-recipe"
+
+  run $ABRA app ls --recipe foo-recipe
+  assert_success
+  refute_output --partial "$TEST_RECIPE"
+  assert_output --partial "foo-recipe"
+}
+
+@test "server stats are correct" {
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "server: $TEST_SERVER"
+  assert_output --partial "total apps: 1"
+
+  run mkdir -p "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/foo.com"
+
+  run cp \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" \
+    "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert_exists "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "foo.com"
+  assert_output --partial "total servers: 2"
+  assert_output --partial "total apps: 2"
+
+  run rm -rf "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/foo.com"
+}
+
+@test "output is machine readable" {
+  run $ABRA app ls --machine
+
+  expectedOutput='{"'
+  expectedOutput+="$TEST_SERVER"
+  expectedOutput+='":{"apps":'
+
+  assert_output --partial "$expectedOutput"
+}

tests/integration/app_logs.bats

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app logs
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app logs DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app logs "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app logs "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}