fix: app version includes tags, sorts & tests

See coop-cloud/organising#442

.dockerignore

@@ -0,0 +1,4 @@
+Dockerfile
+.dockerignore
+*.swp
+*.swo

.drone.yml

@@ -3,35 +3,23 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.18
+    image: golang:1.21
     commands:
       - make check
 
   - name: make build
-    image: golang:1.18
+    image: golang:1.21
     commands:
       - make build
-
-  - name: make test
-    image: golang:1.18
-    commands:
-      - make test
-
-  - name: notify on failure
-    image: plugins/matrix
-    settings:
-      homeserver: https://matrix.autonomic.zone
-      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
-      userid: "@autono-bot:autonomic.zone"
-      accesstoken:
-        from_secret: autono_bot_access_token
     depends_on:
       - make check
-      - make build
+
+  - name: make test
+    image: golang:1.21
+    commands:
       - make test
-    when:
-      status:
-        - failure
+    depends_on:
+      - make check
 
   - name: fetch
     image: docker:git
@@ -45,7 +33,7 @@ steps:
       event: tag
 
   - name: release
-    image: golang:1.18
+    image: goreleaser/goreleaser:v1.18.2
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token
@@ -53,12 +41,29 @@ steps:
       - name: deps
         path: /go
     commands:
-      - curl -sL https://git.io/goreleaser | bash
+      - goreleaser release
     depends_on:
       - fetch
     when:
       event: tag
 
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: 3wordchant
+      password:
+        from_secret: git_coopcloud_tech_token_3wc
+      repo: git.coopcloud.tech/coop-cloud/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      event:
+        exclude:
+          - pull_request
+    depends_on:
+      - make check
+
 volumes:
   - name: deps
     temp: {}

.e2e.env.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

.envrc.sample

@@ -1,6 +1,7 @@
 go env -w GOPRIVATE=coopcloud.tech
 
 # export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
-# export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
-# export CAPSUL_TOKEN=...
-# export GITEA_TOKEN=...
+
+# export ABRA_DIR="$HOME/.abra_test"
+# export ABRA_TEST_DOMAIN=test.example.com
+# export ABRA_SKIP_TEARDOWN=1 # for faster feedback when developing tests

.gitignore

@@ -2,7 +2,8 @@
 .e2e.env
 .envrc
 .vscode/
+/kadabra
 abra
 dist/
-tests/integration/.abra/catalogue
+tests/integration/.bats
 vendor/

.goreleaser.yml

@@ -1,16 +1,19 @@
 ---
-project_name: abra
 gitea_urls:
   api: https://git.coopcloud.tech/api/v1
   download: https://git.coopcloud.tech/
   skip_tls_verify: false
+
 before:
   hooks:
     - go mod tidy
+
 builds:
-  - env:
-      - CGO_ENABLED=0
+  - id: abra
+    binary: abra
     dir: cmd/abra
+    env:
+      - CGO_ENABLED=0
     goos:
       - linux
       - darwin
@@ -26,21 +29,48 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
+
+  - id: kadabra
+    binary: kadabra
+    dir: cmd/kadabra
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+
 archives:
   - replacements:
       386: i386
       amd64: x86_64
     format: binary
+
 checksum:
   name_template: "checksums.txt"
+
 snapshot:
   name_template: "{{ incpatch .Version }}-next"
+
 changelog:
   sort: desc
   filters:
     exclude:
+      - "^Merge"
+      - "^Revert"
       - "^WIP:"
+      - "^chore(deps):"
       - "^style:"
       - "^test:"
       - "^tests:"
-      - "^Revert"

AUTHORS.md

@@ -1,10 +1,17 @@
 # authors
 
-> If you're looking at this and you hack on Abra and you're not listed here,
-> please do add yourself! This is a community project, let's show
+> If you're looking at this and you hack on `abra` and you're not listed here,
+> please do add yourself! This is a community project, let's show some 💞
 
 - 3wordchant
+- cassowary
+- codegod100
 - decentral1se
+- frando
 - kawaiipunk
 - knoflook
+- moritz
+- rix
 - roxxers
+- vera
+- yksflip

Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.21-alpine AS build
+
+ENV GOPRIVATE coopcloud.tech
+
+RUN apk add --no-cache make git gcc musl-dev
+
+COPY . /app
+
+WORKDIR /app
+
+RUN CGO_ENABLED=0 make build
+
+FROM scratch
+
+COPY --from=build /app/abra /abra
+
+ENTRYPOINT ["/abra"]

Makefile

@@ -1,42 +1,49 @@
 ABRA         := ./cmd/abra
+KADABRA      := ./cmd/kadabra
 COMMIT       := $(shell git rev-list -1 HEAD)
 GOPATH       := $(shell go env GOPATH)
 LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+
 export GOPRIVATE=coopcloud.tech
 
 all: format check build test
 
-run:
+run-abra:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
 
-install:
+run-kadabra:
+	@go run -ldflags=$(LDFLAGS) $(KADABRA)
+
+install-abra:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
-build-dev:
-	@go build -ldflags=$(LDFLAGS) $(ABRA)
+install-kadaabra:
+	@go install -ldflags=$(LDFLAGS) $(KADABRA)
+
+build-abra:
+	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
+
+build-kadabra:
+	@go build -v -ldflags=$(LDFLAGS) $(KADABRA)
 
 build:
-	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
 clean:
 	@rm '$(GOPATH)/bin/abra'
+	@rm '$(GOPATH)/bin/kadabra'
 
 format:
 	@gofmt -s -w .
 
 check:
-	@test -z $$(gofmt -l .) || (echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+	@test -z $$(gofmt -l .) || \
+	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
 
 test:
 	@go test ./... -cover -v
 
 loc:
 	@find . -name "*.go" | xargs wc -l
-
-loc-author:
-	@git ls-files -z | \
-		xargs -0rn 1 -P "$$(nproc)" -I{} sh -c 'git blame -w -M -C -C --line-porcelain -- {} | grep -I --line-buffered "^author "' | \
-   	sort -f | \
-   	uniq -ic | \
-   	sort -n

README.md

@@ -2,11 +2,12 @@
 
 [![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
 [![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
+[![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
 
 The Co-op Cloud utility belt 🎩🐇
 
 <a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-`abra` is our flagship client & command-line tool which has been developed specifically in the context of the Co-op Cloud project for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community 💖
 
 Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

cli/app/app.go

@@ -9,28 +9,29 @@ var AppCommand = cli.Command{
 	Aliases:     []string{"a"},
 	Usage:       "Manage apps",
 	ArgsUsage:   "<domain>",
-	Description: "This command provides functionality for managing the life cycle of your apps",
+	Description: "Functionality for managing the life cycle of your apps",
 	Subcommands: []cli.Command{
-		appNewCommand,
-		appConfigCommand,
-		appRestartCommand,
-		appDeployCommand,
-		appUpgradeCommand,
-		appUndeployCommand,
-		appRemoveCommand,
-		appCheckCommand,
-		appListCommand,
-		appPsCommand,
-		appLogsCommand,
-		appCpCommand,
-		appRunCommand,
-		appRollbackCommand,
-		appSecretCommand,
-		appVolumeCommand,
-		appVersionCommand,
-		appErrorsCommand,
-		appCmdCommand,
 		appBackupCommand,
+		appCheckCommand,
+		appCmdCommand,
+		appConfigCommand,
+		appCpCommand,
+		appDeployCommand,
+		appErrorsCommand,
+		appListCommand,
+		appLogsCommand,
+		appNewCommand,
+		appPsCommand,
+		appRemoveCommand,
+		appRestartCommand,
 		appRestoreCommand,
+		appRollbackCommand,
+		appRunCommand,
+		appSecretCommand,
+		appServicesCommand,
+		appUndeployCommand,
+		appUpgradeCommand,
+		appVersionCommand,
+		appVolumeCommand,
 	},
 }

cli/app/backup.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 
@@ -16,13 +15,13 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/system"
 	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -41,11 +40,13 @@ var appBackupCommand = cli.Command{
 	ArgsUsage: "<domain> [<service>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Description: `
-This command runs an app backup.
+Run an app backup.
 
 A backup command and pre/post hook commands are defined in the recipe
 configuration. Abra reads this configuration and run the comands in the context
@@ -73,11 +74,27 @@ This single file can be used to restore your app. See "abra app restore" for mor
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipe.Get(app.Recipe)
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		backupConfigs := make(map[string]backupConfig)
 		for _, service := range recipe.Config.Services {
 			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
@@ -107,6 +124,11 @@ This single file can be used to restore your app. See "abra app restore" for mor
 			}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName != "" {
 			backupConfig, ok := backupConfigs[serviceName]
@@ -116,14 +138,18 @@ This single file can be used to restore your app. See "abra app restore" for mor
 
 			logrus.Infof("running backup for the %s service", serviceName)
 
-			if err := runBackup(app, serviceName, backupConfig); err != nil {
+			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
 				logrus.Fatal(err)
 			}
 		} else {
+			if len(backupConfigs) == 0 {
+				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
+			}
+
 			for serviceName, backupConfig := range backupConfigs {
 				logrus.Infof("running backup for the %s service", serviceName)
 
-				if err := runBackup(app, serviceName, backupConfig); err != nil {
+				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
 					logrus.Fatal(err)
 				}
 			}
@@ -133,15 +159,16 @@ This single file can be used to restore your app. See "abra app restore" for mor
 	},
 }
 
-// runBackup does the actual backup logic.
-func runBackup(app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
+// TimeStamp generates a file name friendly timestamp.
+func TimeStamp() string {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	return strings.Replace(ts, ":", "-", -1)
 }
 
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
+// runBackup does the actual backup logic.
+func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
+	if len(bkConfig.backupPaths) == 0 {
+		return fmt.Errorf("backup paths are empty for %s?", serviceName)
 	}
 
 	// FIXME: avoid instantiating a new CLI
@@ -182,9 +209,8 @@ func runBackup(app config.App, serviceName string, bkConfig backupConfig) error
 
 	var tempBackupPaths []string
 	for _, remoteBackupPath := range bkConfig.backupPaths {
-		timestamp := strconv.Itoa(time.Now().Nanosecond())
 		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp))
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
 		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
 
 		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
@@ -251,7 +277,7 @@ func runBackup(app config.App, serviceName string, bkConfig backupConfig) error
 }
 
 func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp")
+	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
 	if err != nil {
 		return err
 	}
@@ -290,8 +316,7 @@ func mergeArchives(tarPaths []string, serviceName string) error {
 	var out io.Writer
 	var cout *pgzip.Writer
 
-	timestamp := strconv.Itoa(time.Now().Nanosecond())
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, timestamp))
+	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
 
 	fout, err := os.Create(localBackupPath)
 	if err != nil {

cli/app/check.go

@@ -8,6 +8,8 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -15,15 +17,38 @@ import (
 var appCheckCommand = cli.Command{
 	Name:      "check",
 	Aliases:   []string{"chk"},
-	Usage:     "Check if app is configured correctly",
+	Usage:     "Check if an app is configured correctly",
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.ChaosFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
 		if _, err := os.Stat(envSamplePath); err != nil {
 			if os.IsNotExist(err) {
@@ -53,5 +78,4 @@ var appCheckCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cmd.go

@@ -1,10 +1,8 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -14,38 +12,18 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/pkg/archive"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var localCmd bool
-var localCmdFlag = &cli.BoolFlag{
-	Name:        "local, l",
-	Usage:       "Run command locally",
-	Destination: &localCmd,
-}
-
-var remoteUser string
-var remoteUserFlag = &cli.StringFlag{
-	Name:        "user, u",
-	Value:       "",
-	Usage:       "User to run command within a service context",
-	Destination: &remoteUser,
-}
-
 var appCmdCommand = cli.Command{
 	Name:    "command",
 	Aliases: []string{"cmd"},
 	Usage:   "Run app commands",
 	Description: `
-This command runs app specific commands.
+Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
@@ -56,21 +34,46 @@ Example:
 
   abra app cmd example.com app create_user -- me@example.com
 `,
-	ArgsUsage: "<domain> [<service>] <command>",
+	ArgsUsage: "<domain> [<service>] <command> [-- <args>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
-		localCmdFlag,
-		remoteUserFlag,
+		internal.LocalCmdFlag,
+		internal.RemoteUserFlag,
+		internal.TtyFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	BashComplete: autocomplete.AppNameComplete,
 	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if localCmd && remoteUser != "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & <user> together"))
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.LocalCmd && internal.RemoteUser != "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
+		}
+
+		hasCmdArgs, parsedCmdArgs := parseCmdArgs(c.Args(), internal.LocalCmd)
+
 		abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
 		if _, err := os.Stat(abraSh); err != nil {
 			if os.IsNotExist(err) {
@@ -79,47 +82,51 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		var parsedCmdArgs string
-		var cmdArgsIdx int
-		var hasCmdArgs bool
-		for idx, arg := range c.Args() {
-			if arg == "--" {
-				cmdArgsIdx = idx
-				hasCmdArgs = true
+		if internal.LocalCmd {
+			if !(len(c.Args()) >= 2) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
 			}
 
-			if hasCmdArgs && idx > cmdArgsIdx {
-				parsedCmdArgs += fmt.Sprintf("%s ", c.Args().Get(idx))
-			}
-		}
-
-		if localCmd {
 			cmdName := c.Args().Get(1)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
 				logrus.Fatal(err)
 			}
 
 			logrus.Debugf("--local detected, running %s on local work station", cmdName)
 
+			var exportEnv string
+			for k, v := range app.Env {
+				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
+			}
+
 			var sourceAndExec string
 			if hasCmdArgs {
 				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; . %s; %s %s", app.StackName(), abraSh, cmdName, parsedCmdArgs)
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName, parsedCmdArgs)
 			} else {
 				logrus.Debug("did not detect any command arguments")
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; . %s; %s", app.StackName(), abraSh, cmdName)
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName)
 			}
 
-			cmd := exec.Command("/bin/sh", "-c", sourceAndExec)
+			shell := "/bin/bash"
+			if _, err := os.Stat(shell); errors.Is(err, os.ErrNotExist) {
+				logrus.Debugf("%s does not exist locally, use /bin/sh as fallback", shell)
+				shell = "/bin/sh"
+			}
+			cmd := exec.Command(shell, "-c", sourceAndExec)
 
 			if err := internal.RunCmd(cmd); err != nil {
 				logrus.Fatal(err)
 			}
 		} else {
+			if !(len(c.Args()) >= 3) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
 			targetServiceName := c.Args().Get(1)
 
 			cmdName := c.Args().Get(2)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
 				logrus.Fatal(err)
 			}
 
@@ -147,7 +154,12 @@ Example:
 				logrus.Debug("did not detect any command arguments")
 			}
 
-			if err := runCmdRemote(app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := internal.RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -156,78 +168,21 @@ Example:
 	},
 }
 
-func ensureCommand(abraSh, recipeName, execCmd string) error {
-	bytes, err := ioutil.ReadFile(abraSh)
-	if err != nil {
-		return err
+func parseCmdArgs(args []string, isLocal bool) (bool, string) {
+	var (
+		parsedCmdArgs string
+		hasCmdArgs    bool
+	)
+
+	if isLocal {
+		if len(args) > 2 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[2:], " "))
 		}
-
-	if !strings.Contains(string(bytes), execCmd) {
-		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
-	}
-
-	return nil
-}
-
-func runCmdRemote(app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
-
-	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-	content, err := archive.TarWithOptions(abraSh, toTarOpts)
-	if err != nil {
-		return err
-	}
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
-		return err
-	}
-
-	var cmd []string
-	if cmdArgs != "" {
-		cmd = []string{"/bin/sh", "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.StackName(), cmdName, cmdArgs)}
 	} else {
-		cmd = []string{"/bin/sh", "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.StackName(), cmdName)}
+		if len(args) > 3 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[3:], " "))
+		}
 	}
 
-	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
-
-	execCreateOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          cmd,
-		Detach:       false,
-		Tty:          true,
-	}
-
-	if remoteUser != "" {
-		logrus.Debugf("running command with user %s", remoteUser)
-		execCreateOpts.User = remoteUser
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
-		return err
-	}
-
-	return nil
+	return hasCmdArgs, parsedCmdArgs
 }

cli/app/cmd_test.go

@@ -0,0 +1,31 @@
+package app
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseCmdArgs(t *testing.T) {
+	tests := []struct {
+		input          []string
+		shouldParse    bool
+		expectedOutput string
+	}{
+		// `--` is not parsed when passed in from the command-line e.g. -- foo bar baz
+		// so we need to eumlate that as missing when testing if bash args are passed in
+		// see https://git.coopcloud.tech/coop-cloud/organising/issues/336 for more
+		{[]string{"foo.com", "app", "test"}, false, ""},
+		{[]string{"foo.com", "app", "test", "foo"}, true, "foo "},
+		{[]string{"foo.com", "app", "test", "foo", "bar", "baz"}, true, "foo bar baz "},
+	}
+
+	for _, test := range tests {
+		ok, parsed := parseCmdArgs(test.input, false)
+		if ok != test.shouldParse {
+			t.Fatalf("[%s] should not parse", strings.Join(test.input, " "))
+		}
+		if parsed != test.expectedOutput {
+			t.Fatalf("%s does not match %s", parsed, test.expectedOutput)
+		}
+	}
+}

cli/app/config.go

@@ -22,6 +22,7 @@ var appConfigCommand = cli.Command{
 		internal.DebugFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		appName := c.Args().First()
 
@@ -60,5 +61,4 @@ var appConfigCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cp.go

@@ -14,6 +14,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -28,9 +29,9 @@ var appCpCommand = cli.Command{
 		internal.NoInputFlag,
 	},
 	Before: internal.SubCommandBefore,
-	Usage:  "Copy files to/from a running app service",
+	Usage:  "Copy files to/from a deployed app service",
 	Description: `
-This command supports copying files to and from any app service file system.
+Copy files to and from any app service file system.
 
 If you want to copy a myfile.txt to the root of the app service:
 
@@ -40,6 +41,7 @@ And if you want to copy that file back to your current working directory locally
 
     abra app cp <domain> app:/myfile.txt .
 `,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -83,33 +85,33 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Debugf("assuming transfer is going TO the container")
 		}
 
-		if !isToContainer {
-			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist locally?", dstPath)
+		if isToContainer {
+			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", srcPath)
 			}
 		}
-		err := configureAndCp(c, app, srcPath, dstPath, service, isToContainer)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		return nil
-
-	},
-	BashComplete: autocomplete.AppNameComplete,
-}
 
-func configureAndCp(
-	c *cli.Context,
-	app config.App,
-	srcPath string,
-	dstPath string,
-	service string,
-	isToContainer bool) error {
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+func configureAndCp(
+	c *cli.Context,
+	cl *dockerClient.Client,
+	app config.App,
+	srcPath string,
+	dstPath string,
+	service string,
+	isToContainer bool) error {
 	filters := filters.NewArgs()
 	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
 
@@ -121,10 +123,6 @@ func configureAndCp(
 	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 	if isToContainer {
-		if _, err := os.Stat(srcPath); err != nil {
-			logrus.Fatalf("%s does not exist?", srcPath)
-		}
-
 		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
 		content, err := archive.TarWithOptions(srcPath, toTarOpts)
 		if err != nil {

cli/app/deploy.go

@@ -1,8 +1,21 @@
 package app
 
 import (
+	"context"
+	"fmt"
+
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
@@ -10,7 +23,7 @@ var appDeployCommand = cli.Command{
 	Name:      "deploy",
 	Aliases:   []string{"d"},
 	Usage:     "Deploy an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -18,20 +31,197 @@ var appDeployCommand = cli.Command{
 		internal.ChaosFlag,
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
-This command deploys an app. It does not support incrementing the version of a
-deployed app, for this you need to look at the "abra app upgrade <domain>"
-command.
+Deploy an app. It does not support incrementing the version of a deployed app,
+for this you need to look at the "abra app upgrade <domain>" command.
 
 You may pass "--force" to re-deploy the same version again. This can be useful
 if the container runtime has gotten into a weird state.
 
-Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	Action:       internal.DeployAction,
 	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		r, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(r); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			if internal.Force || internal.Chaos {
+				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
+			} else {
+				logrus.Fatalf("%s is already deployed", app.Name)
+			}
+		}
+
+		version := deployedVersion
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			version = specificVersion
+			logrus.Debugf("choosing %s as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if !internal.Chaos && specificVersion == "" {
+			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if len(versions) == 0 && !internal.Chaos {
+				logrus.Warn("no published versions in catalogue, trying local recipe repository")
+				recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+				if err != nil {
+					logrus.Warn(err)
+				}
+				for _, recipeVersion := range recipeVersions {
+					for version := range recipeVersion {
+						versions = append(versions, version)
+					}
+				}
+			}
+
+			if len(versions) > 0 && !internal.Chaos {
+				version = versions[len(versions)-1]
+				logrus.Debugf("choosing %s as version to deploy", version)
+				if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				head, err := git.GetRecipeHead(app.Recipe)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				version = formatter.SmallSHA(head.String())
+				logrus.Warn("no versions detected, using latest commit")
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warnf("chaos mode engaged")
+			var err error
+			version, err = recipe.ChaosVersion(app.Recipe)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, version)
+		config.SetUpdateLabel(compose, stackName, app.Env)
+
+		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.NoDomainChecks {
+			domainName, ok := app.Env["DOMAIN"]
+			if ok {
+				if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
+			}
+		} else {
+			logrus.Warn("skipping domain checks as requested")
+		}
+
+		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_DEPLOY_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+			}
+		}
+		return nil
+	},
 }

cli/app/errors.go

@@ -25,7 +25,7 @@ var appErrorsCommand = cli.Command{
 	Usage:     "List errors for a deployed app",
 	ArgsUsage: "<domain>",
 	Description: `
-This command lists errors for a deployed app.
+List errors for a deployed app.
 
 This is a best-effort implementation and an attempt to gather a number of tips
 & tricks for finding errors together into one convenient command. When an app
@@ -50,6 +50,7 @@ the logs.
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.WatchFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
@@ -83,13 +84,11 @@ the logs.
 			}
 			time.Sleep(2 * time.Second)
 		}
-
-		return nil
 	},
 }
 
 func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
-	recipe, err := recipe.Get(app.Recipe)
+	recipe, err := recipe.Get(app.Recipe, internal.Offline)
 	if err != nil {
 		return err
 	}

cli/app/list.go

@@ -1,15 +1,16 @@
 package app
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/ssh"
 	"coopcloud.tech/tagcmp"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -22,12 +23,12 @@ var statusFlag = &cli.BoolFlag{
 	Destination: &status,
 }
 
-var appRecipe string
+var recipeFilter string
 var recipeFlag = &cli.StringFlag{
 	Name:        "recipe, r",
 	Value:       "",
 	Usage:       "Show apps of a specific recipe",
-	Destination: &appRecipe,
+	Destination: &recipeFilter,
 }
 
 var listAppServer string
@@ -39,22 +40,25 @@ var listAppServerFlag = &cli.StringFlag{
 }
 
 type appStatus struct {
-	server  string
-	recipe  string
-	appName string
-	domain  string
-	status  string
-	version string
-	upgrade string
+	Server       string `json:"server"`
+	Recipe       string `json:"recipe"`
+	AppName      string `json:"appName"`
+	Domain       string `json:"domain"`
+	Status       string `json:"status"`
+	Chaos        string `json:"chaos"`
+	ChaosVersion string `json:"chaosVersion"`
+	AutoUpdate   string `json:"autoUpdate"`
+	Version      string `json:"version"`
+	Upgrade      string `json:"upgrade"`
 }
 
 type serverStatus struct {
-	apps             []appStatus
-	appCount         int
-	versionCount     int
-	unversionedCount int
-	latestCount      int
-	upgradeCount     int
+	Apps             []appStatus `json:"apps"`
+	AppCount         int         `json:"appCount"`
+	VersionCount     int         `json:"versionCount"`
+	UnversionedCount int         `json:"unversionedCount"`
+	LatestCount      int         `json:"latestCount"`
+	UpgradeCount     int         `json:"upgradeCount"`
 }
 
 var appListCommand = cli.Command{
@@ -62,8 +66,8 @@ var appListCommand = cli.Command{
 	Aliases: []string{"ls"},
 	Usage:   "List all managed apps",
 	Description: `
-This command looks at your local file system listing of apps and servers (e.g.
-in ~/.abra/) to generate a report of all your apps.
+Read the local file system listing of apps and servers (e.g. ~/.abra/) to
+generate a report of all your apps.
 
 By passing the "--status/-S" flag, you can query all your servers for the
 actual live deployment status. Depending on how many servers you manage, this
@@ -71,9 +75,11 @@ can take some time.
 `,
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.MachineReadableFlag,
 		statusFlag,
 		listAppServerFlag,
 		recipeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
@@ -82,7 +88,7 @@ can take some time.
 			logrus.Fatal(err)
 		}
 
-		apps, err := config.GetApps(appFiles)
+		apps, err := config.GetApps(appFiles, recipeFilter)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -95,20 +101,16 @@ can take some time.
 			alreadySeen := make(map[string]bool)
 			for _, app := range apps {
 				if _, ok := alreadySeen[app.Server]; !ok {
-					if err := ssh.EnsureHostKey(app.Server); err != nil {
-						logrus.Fatal(fmt.Sprintf(internal.SSHFailMsg, app.Server))
-					}
 					alreadySeen[app.Server] = true
 				}
 			}
 
-			statuses, err = config.GetAppStatuses(appFiles)
+			statuses, err = config.GetAppStatuses(apps, internal.MachineReadable)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-			var err error
-			catl, err = recipe.ReadRecipeCatalogue()
+			catl, err = recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -122,41 +124,56 @@ can take some time.
 			var ok bool
 			if stats, ok = allStats[app.Server]; !ok {
 				stats = serverStatus{}
-				if appRecipe == "" {
+				if recipeFilter == "" {
 					// count server, no filtering
 					totalServersCount++
 				}
 			}
 
-			if app.Recipe == appRecipe || appRecipe == "" {
-				if appRecipe != "" {
+			if app.Recipe == recipeFilter || recipeFilter == "" {
+				if recipeFilter != "" {
 					// only count server if matches filter
 					totalServersCount++
 				}
 
 				appStats := appStatus{}
-				stats.appCount++
+				stats.AppCount++
 				totalAppsCount++
 
 				if status {
 					status := "unknown"
 					version := "unknown"
+					chaos := "unknown"
+					chaosVersion := "unknown"
+					autoUpdate := "unknown"
 					if statusMeta, ok := statuses[app.StackName()]; ok {
 						if currentVersion, exists := statusMeta["version"]; exists {
 							if currentVersion != "" {
 								version = currentVersion
 							}
 						}
+						if chaosDeploy, exists := statusMeta["chaos"]; exists {
+							chaos = chaosDeploy
+						}
+						if chaosDeployVersion, exists := statusMeta["chaosVersion"]; exists {
+							chaosVersion = chaosDeployVersion
+						}
+						if autoUpdateState, exists := statusMeta["autoUpdate"]; exists {
+							autoUpdate = autoUpdateState
+						}
 						if statusMeta["status"] != "" {
 							status = statusMeta["status"]
 						}
-						stats.versionCount++
+						stats.VersionCount++
 					} else {
-						stats.unversionedCount++
+						stats.UnversionedCount++
 					}
 
-					appStats.status = status
-					appStats.version = version
+					appStats.Status = status
+					appStats.Chaos = chaos
+					appStats.ChaosVersion = chaosVersion
+					appStats.Version = version
+					appStats.AutoUpdate = autoUpdate
 
 					var newUpdates []string
 					if version != "unknown" {
@@ -184,29 +201,38 @@ can take some time.
 
 					if len(newUpdates) == 0 {
 						if version == "unknown" {
-							appStats.upgrade = "unknown"
+							appStats.Upgrade = "unknown"
 						} else {
-							appStats.upgrade = "latest"
-							stats.latestCount++
+							appStats.Upgrade = "latest"
+							stats.LatestCount++
 						}
 					} else {
 						newUpdates = internal.ReverseStringList(newUpdates)
-						appStats.upgrade = strings.Join(newUpdates, "\n")
-						stats.upgradeCount++
+						appStats.Upgrade = strings.Join(newUpdates, "\n")
+						stats.UpgradeCount++
 					}
 				}
 
-				appStats.server = app.Server
-				appStats.recipe = app.Recipe
-				appStats.appName = app.Name
-				appStats.domain = app.Domain
+				appStats.Server = app.Server
+				appStats.Recipe = app.Recipe
+				appStats.AppName = app.Name
+				appStats.Domain = app.Domain
 
-				stats.apps = append(stats.apps, appStats)
+				stats.Apps = append(stats.Apps, appStats)
 			}
-
 			allStats[app.Server] = stats
 		}
 
+		if internal.MachineReadable {
+			jsonstring, err := json.Marshal(allStats)
+			if err != nil {
+				logrus.Fatal(err)
+			} else {
+				fmt.Println(string(jsonstring))
+			}
+			return nil
+		}
+
 		alreadySeen := make(map[string]bool)
 		for _, app := range apps {
 			if _, ok := alreadySeen[app.Server]; ok {
@@ -217,15 +243,25 @@ can take some time.
 
 			tableCol := []string{"recipe", "domain"}
 			if status {
-				tableCol = append(tableCol, []string{"status", "version", "upgrade"}...)
+				tableCol = append(tableCol, []string{"status", "chaos", "version", "upgrade", "autoupdate"}...)
 			}
 
 			table := formatter.CreateTable(tableCol)
 
-			for _, appStat := range serverStat.apps {
-				tableRow := []string{appStat.recipe, appStat.domain}
+			for _, appStat := range serverStat.Apps {
+				tableRow := []string{appStat.Recipe, appStat.Domain}
 				if status {
-					tableRow = append(tableRow, []string{appStat.status, appStat.version, appStat.upgrade}...)
+					chaosStatus := appStat.Chaos
+					if chaosStatus != "unknown" {
+						chaosEnabled, err := strconv.ParseBool(chaosStatus)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+						if chaosEnabled && appStat.ChaosVersion != "unknown" {
+							chaosStatus = appStat.ChaosVersion
+						}
+					}
+					tableRow = append(tableRow, []string{appStat.Status, chaosStatus, appStat.Version, appStat.Upgrade, appStat.AutoUpdate}...)
 				}
 				table.Append(tableRow)
 			}
@@ -237,14 +273,14 @@ can take some time.
 					fmt.Println(fmt.Sprintf(
 						"server: %s | total apps: %v | versioned: %v | unversioned: %v | latest: %v | upgrade: %v",
 						app.Server,
-						serverStat.appCount,
-						serverStat.versionCount,
-						serverStat.unversionedCount,
-						serverStat.latestCount,
-						serverStat.upgradeCount,
+						serverStat.AppCount,
+						serverStat.VersionCount,
+						serverStat.UnversionedCount,
+						serverStat.LatestCount,
+						serverStat.UpgradeCount,
 					))
 				} else {
-					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.appCount))
+					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.AppCount))
 				}
 			}
 

cli/app/logs.go

@@ -12,6 +12,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
@@ -20,12 +21,14 @@ import (
 )
 
 var logOpts = types.ContainerLogsOptions{
-	Details:    false,
-	Follow:     true,
 	ShowStderr: true,
 	ShowStdout: true,
-	Tail:       "20",
+	Since:      "",
+	Until:      "",
 	Timestamps: true,
+	Follow:     true,
+	Tail:       "20",
+	Details:    false,
 }
 
 // stackLogs lists logs for all stack services
@@ -74,18 +77,31 @@ var appLogsCommand = cli.Command{
 	Usage:     "Tail app logs",
 	Flags: []cli.Flag{
 		internal.StdErrOnlyFlag,
+		internal.SinceLogsFlag,
 		internal.DebugFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
+		stackName := app.StackName()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		logOpts.Since = internal.SinceLogs
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			logrus.Debugf("tailing logs for all %s services", app.Recipe)
@@ -103,7 +119,7 @@ var appLogsCommand = cli.Command{
 
 func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
 	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
 
 	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
 	if err != nil {

cli/app/new.go

@@ -1,14 +1,28 @@
 package app
 
 import (
+	"fmt"
+	"path"
+
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/jsontable"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
 var appNewDescription = `
-This command takes a recipe and uses it to create a new app. This new app
-configuration is stored in your ~/.abra directory under the appropriate server.
+Take a recipe and uses it to create a new app. This new app configuration is
+stored in your ~/.abra directory under the appropriate server.
 
 This command does not deploy your app for you. You will need to run "abra app
 deploy <domain>" to do so.
@@ -38,9 +52,204 @@ var appNewCommand = cli.Command{
 		internal.DomainFlag,
 		internal.PassFlag,
 		internal.SecretsFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "[<recipe>]",
-	Action:       internal.NewAction,
 	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		if !internal.Offline {
+			if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := ensureServerFlag(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := ensureDomainFlag(recipe, internal.NewAppServer); err != nil {
+			logrus.Fatal(err)
+		}
+
+		sanitisedAppName := config.SanitiseAppName(internal.Domain)
+		logrus.Debugf("%s sanitised as %s for new app", internal.Domain, sanitisedAppName)
+
+		if err := config.TemplateAppEnvSample(
+			recipe.Name,
+			internal.Domain,
+			internal.NewAppServer,
+			internal.Domain,
+		); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := promptForSecrets(internal.Domain); err != nil {
+			logrus.Fatal(err)
+		}
+
+		var secrets AppSecrets
+		var secretTable *jsontable.JSONTable
+		if internal.Secrets {
+			cl, err := client.New(internal.NewAppServer)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secrets, err := createSecrets(cl, sanitisedAppName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secretCols := []string{"Name", "Value"}
+			secretTable = formatter.CreateTable(secretCols)
+			for secret := range secrets {
+				secretTable.Append([]string{secret, secrets[secret]})
+			}
+
+		}
+
+		if internal.NewAppServer == "default" {
+			internal.NewAppServer = "local"
+		}
+
+		tableCol := []string{"server", "recipe", "domain"}
+		table := formatter.CreateTable(tableCol)
+		table.Append([]string{internal.NewAppServer, recipe.Name, internal.Domain})
+
+		fmt.Println("")
+		fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
+		fmt.Println("")
+		table.Render()
+		fmt.Println("")
+		fmt.Println("You can configure this app by running the following:")
+		fmt.Println(fmt.Sprintf("\n    abra app config %s", internal.Domain))
+		fmt.Println("")
+		fmt.Println("You can deploy this app by running the following:")
+		fmt.Println(fmt.Sprintf("\n    abra app deploy %s", internal.Domain))
+		fmt.Println("")
+
+		if len(secrets) > 0 {
+			fmt.Println("Here are your generated secrets:")
+			fmt.Println("")
+			secretTable.Render()
+			fmt.Println("")
+			logrus.Warn("generated secrets are not shown again, please take note of them *now*")
+		}
+
+		return nil
+	},
+}
+
+// AppSecrets represents all app secrest
+type AppSecrets map[string]string
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets, error) {
+	appEnvPath := path.Join(
+		config.ABRA_DIR,
+		"servers",
+		internal.NewAppServer,
+		fmt.Sprintf("%s.env", internal.Domain),
+	)
+
+	appEnv, err := config.ReadEnv(appEnvPath)
+	if err != nil {
+		return nil, err
+	}
+
+	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
+	secrets, err := secret.GenerateSecrets(cl, secretEnvVars, sanitisedAppName, internal.NewAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if internal.Pass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(
+				secretValue,
+				secretName,
+				internal.Domain,
+				internal.NewAppServer,
+			); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return secrets, nil
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag(recipe recipe.Recipe, server string) error {
+	if internal.Domain == "" && !internal.NoInput {
+		prompt := &survey.Input{
+			Message: "Specify app domain",
+			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
+		}
+		if err := survey.AskOne(prompt, &internal.Domain); err != nil {
+			return err
+		}
+	}
+
+	if internal.Domain == "" {
+		return fmt.Errorf("no domain provided")
+	}
+
+	return nil
+}
+
+// promptForSecrets asks if we should generate secrets for a new app.
+func promptForSecrets(appName string) error {
+	app, err := app.Get(appName)
+	if err != nil {
+		return err
+	}
+
+	secretEnvVars := secret.ReadSecretEnvVars(app.Env)
+	if len(secretEnvVars) == 0 {
+		logrus.Debugf("%s has no secrets to generate, skipping...", app.Recipe)
+		return nil
+	}
+
+	if !internal.Secrets && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: "Generate app secrets?",
+		}
+		if err := survey.AskOne(prompt, &internal.Secrets); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	servers, err := config.GetServers()
+	if err != nil {
+		return err
+	}
+
+	if internal.NewAppServer == "" && !internal.NoInput {
+		prompt := &survey.Select{
+			Message: "Select app server:",
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &internal.NewAppServer); err != nil {
+			return err
+		}
+	}
+
+	if internal.NewAppServer == "" {
+		return fmt.Errorf("no server provided")
+	}
+
+	return nil
 }

cli/app/ps.go

@@ -25,7 +25,7 @@ var appPsCommand = cli.Command{
 	Aliases:     []string{"p"},
 	Usage:       "Check app status",
 	ArgsUsage:   "<domain>",
-	Description: "This command shows a more detailed status output of a specific deployed app.",
+	Description: "Show a more detailed status output of a specific deployed app",
 	Flags: []cli.Flag{
 		internal.WatchFlag,
 		internal.DebugFlag,

cli/app/remove.go

@@ -11,39 +11,49 @@ import (
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-// Volumes stores the variable from VolumesFlag
-var Volumes bool
-
-// VolumesFlag is used to specify if volumes should be deleted when deleting an app
-var VolumesFlag = &cli.BoolFlag{
-	Name:        "volumes, V",
-	Destination: &Volumes,
-}
-
 var appRemoveCommand = cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
 	ArgsUsage: "<domain>",
-	Usage:     "Remove an already undeployed app",
+	Usage:     "Remove all app data, locally and remotely",
+	Description: `
+This command removes everything related to an app which is already undeployed.
+
+By default, it will prompt for confirmation before proceeding. All secrets,
+volumes and the local app env file will be deleted.
+
+Only run this command when you are sure you want to completely remove the app
+and all associated app data. This is a destructive action, Be Careful!
+
+If you would like to delete specific volumes or secrets, please use removal
+sub-commands under "app volume" and "app secret" instead.
+
+Please note, if you delete the local app env file without removing volumes and
+secrets first, Abra will *not* be able to help you remove them afterwards.
+
+To delete everything without prompt, use the "--force/-f" or the "--no-input/n"
+flag.
+`,
 	Flags: []cli.Flag{
-		VolumesFlag,
 		internal.ForceFlag,
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
+	BashComplete: autocomplete.AppNameComplete,
 	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
 		if !internal.Force && !internal.NoInput {
 			response := false
-			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("about to remove %s, are you sure?", app.Name),
-			}
+			msg := "ALERTA ALERTA: this will completely remove %s data and configurations locally and remotely, are you sure?"
+			prompt := &survey.Confirm{Message: fmt.Sprintf(msg, app.Name)}
 			if err := survey.AskOne(prompt, &response); err != nil {
 				logrus.Fatal(err)
 			}
@@ -84,26 +94,7 @@ var appRemoveCommand = cli.Command{
 		}
 
 		if len(secrets) > 0 {
-			var secretNamesToRemove []string
-
-			if !internal.Force && !internal.NoInput {
-				secretsPrompt := &survey.MultiSelect{
-					Message: "which secrets do you want to remove?",
-					Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-					VimMode: true,
-					Options: secretNames,
-					Default: secretNames,
-				}
-				if err := survey.AskOne(secretsPrompt, &secretNamesToRemove); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if internal.Force || internal.NoInput {
-				secretNamesToRemove = secretNames
-			}
-
-			for _, name := range secretNamesToRemove {
+			for _, name := range secretNames {
 				err := cl.SecretRemove(context.Background(), secrets[name])
 				if err != nil {
 					logrus.Fatal(err)
@@ -119,7 +110,8 @@ var appRemoveCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		volumeListOKBody, err := cl.VolumeList(context.Background(), fs)
+		volumeListOptions := volume.ListOptions{fs}
+		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
 		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
@@ -131,22 +123,7 @@ var appRemoveCommand = cli.Command{
 		}
 
 		if len(vols) > 0 {
-			if Volumes {
-				var removeVols []string
-				if !internal.Force && !internal.NoInput {
-					volumesPrompt := &survey.MultiSelect{
-						Message: "which volumes do you want to remove?",
-						Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-						VimMode: true,
-						Options: vols,
-						Default: vols,
-					}
-					if err := survey.AskOne(volumesPrompt, &removeVols); err != nil {
-						logrus.Fatal(err)
-					}
-				}
-
-				for _, vol := range removeVols {
+			for _, vol := range vols {
 				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
 				if err != nil {
 					logrus.Fatal(err)
@@ -154,21 +131,15 @@ var appRemoveCommand = cli.Command{
 				logrus.Info(fmt.Sprintf("volume %s removed", vol))
 			}
 		} else {
-				logrus.Info("no volumes were removed")
-			}
-		} else {
-			if Volumes {
 			logrus.Info("no volumes to remove")
 		}
-		}
 
-		err = os.Remove(app.Path)
-		if err != nil {
+		if err = os.Remove(app.Path); err != nil {
 			logrus.Fatal(err)
 		}
+
 		logrus.Info(fmt.Sprintf("file: %s removed", app.Path))
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/restart.go

@@ -21,6 +21,7 @@ var appRestartCommand = cli.Command{
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Description:  `This command restarts a service within a deployed app.`,
@@ -39,6 +40,15 @@ var appRestartCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
 		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
 
 		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)

cli/app/restore.go

@@ -12,10 +12,12 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -33,11 +35,13 @@ var appRestoreCommand = cli.Command{
 	ArgsUsage: "<domain> <service> <file>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Description: `
-This command runs an app restore.
+Run an app restore.
 
 Pre/post hook commands are defined in the recipe configuration. Abra reads this
 configuration and run the comands in the context of the service before
@@ -46,7 +50,8 @@ restoring the backup.
 Unlike "abra app backup", restore must be run on a per-service basis. You can
 not restore all services in one go. Backup files produced by Abra are
 compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic.
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
 
 Example:
 
@@ -55,6 +60,27 @@ Example:
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
@@ -71,11 +97,6 @@ Example:
 			}
 		}
 
-		recipe, err := recipe.Get(app.Recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		restoreConfigs := make(map[string]restoreConfig)
 		for _, service := range recipe.Config.Services {
 			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
@@ -104,7 +125,13 @@ Example:
 		if !ok {
 			rsConfig = restoreConfig{}
 		}
-		if err := runRestore(app, backupPath, serviceName, rsConfig); err != nil {
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -113,12 +140,7 @@ Example:
 }
 
 // runRestore does the actual restore logic.
-func runRestore(app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
-
+func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
 	// FIXME: avoid instantiating a new CLI
 	dcli, err := command.NewDockerCli()
 	if err != nil {
@@ -165,8 +187,8 @@ func runRestore(app config.App, backupPath, serviceName string, rsConfig restore
 		return err
 	}
 
-	// we use absolute paths so tar knows what to do. it will restore files
-	// according to the paths set in the compresed archive
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
 	restorePath := "/"
 
 	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}

cli/app/rollback.go

@@ -22,7 +22,7 @@ var appRollbackCommand = cli.Command{
 	Name:      "rollback",
 	Aliases:   []string{"rl"},
 	Usage:     "Roll an app back to a previous version",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -30,6 +30,7 @@ var appRollbackCommand = cli.Command{
 		internal.ChaosFlag,
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -50,13 +51,27 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
 		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
 				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-		r, err := recipe.Get(app.Recipe)
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		r, err := recipe.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -81,7 +96,7 @@ recipes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -92,16 +107,41 @@ recipes.
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
 		}
 
 		var availableDowngrades []string
 		if deployedVersion == "unknown" {
 			availableDowngrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		if deployedVersion != "unknown" && !internal.Chaos {
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsGreaterThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not a downgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableDowngrades = append(availableDowngrades, specificVersion)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
 			for _, version := range versions {
 				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 				if err != nil {
@@ -111,28 +151,26 @@ recipes.
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				if parsedVersion != parsedDeployedVersion && parsedVersion.IsLessThan(parsedDeployedVersion) {
+				if parsedVersion.IsLessThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
 					availableDowngrades = append(availableDowngrades, version)
 				}
 			}
 
-			if len(availableDowngrades) == 0 {
+			if len(availableDowngrades) == 0 && !internal.Force {
 				logrus.Info("no available downgrades, you're on oldest ✌️")
 				return nil
 			}
 		}
 
-		availableDowngrades = internal.ReverseStringList(availableDowngrades)
-
 		var chosenDowngrade string
-		if !internal.Chaos {
-			if internal.Force || internal.NoInput {
-				chosenDowngrade = availableDowngrades[0]
-				logrus.Debugf("choosing %s as version to downgrade to (--force)", chosenDowngrade)
+		if len(availableDowngrades) > 0 && !internal.Chaos {
+			if internal.Force || internal.NoInput || specificVersion != "" {
+				chosenDowngrade = availableDowngrades[len(availableDowngrades)-1]
+				logrus.Debugf("choosing %s as version to downgrade to (--force/--no-input)", chosenDowngrade)
 			} else {
 				prompt := &survey.Select{
 					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
-					Options: availableDowngrades,
+					Options: internal.ReverseStringList(availableDowngrades),
 				}
 				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
 					return err
@@ -178,14 +216,18 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, chosenDowngrade)
+		config.SetUpdateLabel(compose, stackName, app.Env)
 
-		if !internal.Force {
+		// NOTE(d1): no release notes implemeneted for rolling back
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
 			logrus.Fatal(err)
 		}
-		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose, stackName, internal.DontWaitConverge); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/secret.go

@@ -79,7 +79,12 @@ var appSecretGenerateCommand = cli.Command{
 			}
 		}
 
-		secretVals, err := secret.GenerateSecrets(secretsToCreate, app.StackName(), app.Server)
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -139,12 +144,17 @@ Example:
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		name := c.Args().Get(1)
 		version := c.Args().Get(2)
 		data := c.Args().Get(3)
 
 		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
-		if err := client.StoreSecret(secretName, data, app.Server); err != nil {
+		if err := client.StoreSecret(cl, secretName, data, app.Server); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -281,6 +291,7 @@ var appSecretLsCommand = cli.Command{
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "List all secrets",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		secrets := secret.ReadSecretEnvVars(app.Env)
@@ -331,7 +342,6 @@ var appSecretLsCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appSecretCommand = cli.Command{

cli/app/services.go

@@ -0,0 +1,80 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appServicesCommand = cli.Command{
+	Name:      "services",
+	Aliases:   []string{"sr"},
+	Usage:     "Display all services of an app",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		filters, err := app.Filters(true, true)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableCol := []string{"service name", "image"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, container := range containers {
+			var containerNames []string
+			for _, containerName := range container.Names {
+				trimmed := strings.TrimPrefix(containerName, "/")
+				containerNames = append(containerNames, trimmed)
+			}
+
+			serviceShortName := service.ContainerToServiceName(container.Names, app.StackName())
+			serviceLongName := fmt.Sprintf("%s_%s", app.StackName(), serviceShortName)
+
+			tableRow := []string{
+				serviceLongName,
+				formatter.RemoveSha(container.Image),
+			}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+
+		return nil
+	},
+}

cli/app/undeploy.go

@@ -2,15 +2,82 @@ package app
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+var prune bool
+
+var pruneFlag = &cli.BoolFlag{
+	Name:        "prune, p",
+	Destination: &prune,
+	Usage:       "Prunes unused containers, networks, and dangling images for an app",
+}
+
+// pruneApp runs the equivalent of a "docker system prune" but only filtering
+// against resources connected with the app deployment. It is not a system wide
+// prune. Volumes are not pruned to avoid unwated data loss.
+func pruneApp(c *cli.Context, cl *dockerClient.Client, app config.App) error {
+	stackName := app.StackName()
+	ctx := context.Background()
+
+	for {
+		logrus.Debugf("polling for %s stack, waiting to be undeployed...", stackName)
+
+		services, err := stack.GetStackServices(ctx, cl, stackName)
+		if err != nil {
+			return err
+		}
+
+		if len(services) == 0 {
+			logrus.Debugf("%s undeployed, moving on with pruning logic", stackName)
+			time.Sleep(time.Second) // give runtime more time to tear down related state
+			break
+		}
+
+		time.Sleep(time.Second)
+	}
+
+	pruneFilters := filters.NewArgs()
+	stackSearch := fmt.Sprintf("%s*", stackName)
+	pruneFilters.Add("label", stackSearch)
+	cr, err := cl.ContainersPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	cntSpaceReclaimed := formatter.ByteCountSI(cr.SpaceReclaimed)
+	logrus.Infof("containers pruned: %d; space reclaimed: %s", len(cr.ContainersDeleted), cntSpaceReclaimed)
+
+	nr, err := cl.NetworksPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("networks pruned: %d", len(nr.NetworksDeleted))
+
+	ir, err := cl.ImagesPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
+	logrus.Infof("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed)
+
+	return nil
+}
+
 var appUndeployCommand = cli.Command{
 	Name:      "undeploy",
 	Aliases:   []string{"un"},
@@ -18,13 +85,18 @@ var appUndeployCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		pruneFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "Undeploy an app",
+	BashComplete: autocomplete.AppNameComplete,
 	Description: `
-This does not destroy any of the application data. However, you should remain
-vigilant, as your swarm installation will consider any previously attached
-volumes as eligiblef or pruning once undeployed.
+This does not destroy any of the application data.
+
+However, you should remain vigilant, as your swarm installation will consider
+any previously attached volumes as eligible for pruning once undeployed.
+
+Passing "-p/--prune" does not remove those volumes.
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
@@ -55,7 +127,12 @@ volumes as eligiblef or pruning once undeployed.
 			logrus.Fatal(err)
 		}
 
+		if prune {
+			if err := pruneApp(c, cl, app); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/upgrade.go

@@ -10,6 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -21,7 +22,7 @@ var appUpgradeCommand = cli.Command{
 	Name:      "upgrade",
 	Aliases:   []string{"up"},
 	Usage:     "Upgrade an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -29,11 +30,12 @@ var appUpgradeCommand = cli.Command{
 		internal.ChaosFlag,
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
-This command supports upgrading an app. You can use it to choose and roll out a
-new upgrade to an existing app.
+Upgrade an app. You can use it to choose and roll out a new upgrade to an
+existing app.
 
 This command specifically supports incrementing the version of running apps, as
 opposed to "abra app deploy <domain>" which will not change the version of a
@@ -49,32 +51,43 @@ Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
 		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
 				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-		r, err := recipe.Get(app.Recipe)
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := lint.LintForErrors(r); err != nil {
+		if err := lint.LintForErrors(recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
-
 		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
@@ -84,37 +97,63 @@ recipes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		versions, err := recipePkg.GetRecipeCatalogueVersions(app.Recipe, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipePkg.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
 		}
 
 		var availableUpgrades []string
 		if deployedVersion == "unknown" {
 			availableUpgrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		if deployedVersion != "unknown" && !internal.Chaos {
-			for _, version := range versions {
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsLessThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not an upgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableUpgrades = append(availableUpgrades, specificVersion)
+		}
+
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
+			for _, version := range versions {
 				parsedVersion, err := tagcmp.Parse(version)
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				if parsedVersion.IsGreaterThan(parsedDeployedVersion) {
+				if parsedVersion.IsGreaterThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
 					availableUpgrades = append(availableUpgrades, version)
 				}
 			}
@@ -125,17 +164,15 @@ recipes.
 			}
 		}
 
-		availableUpgrades = internal.ReverseStringList(availableUpgrades)
-
 		var chosenUpgrade string
 		if len(availableUpgrades) > 0 && !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force || internal.NoInput || specificVersion != "" {
 				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
 				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
 			} else {
 				prompt := &survey.Select{
 					Message: fmt.Sprintf("Please select an upgrade (current version: %s):", deployedVersion),
-					Options: availableUpgrades,
+					Options: internal.ReverseStringList(availableUpgrades),
 				}
 				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
 					return err
@@ -143,16 +180,38 @@ recipes.
 			}
 		}
 
+		if internal.Force && chosenUpgrade == "" {
+			logrus.Warnf("%s is already upgraded to latest but continuing (--force/--chaos)", app.Name)
+			chosenUpgrade = deployedVersion
+		}
+
 		// if release notes written after git tag published, read them before we
 		// check out the tag and then they'll appear to be missing. this covers
 		// when we obviously will forget to write release notes before publishing
-		releaseNotes, err := internal.GetReleaseNotes(app.Recipe, chosenUpgrade)
+		var releaseNotes string
+		for _, version := range versions {
+			parsedVersion, err := tagcmp.Parse(version)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !(parsedVersion.Equals(parsedDeployedVersion)) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
+				note, err := internal.GetReleaseNotes(app.Recipe, version)
 				if err != nil {
 					return err
 				}
+				if note != "" {
+					releaseNotes += fmt.Sprintf("%s\n", note)
+				}
+			}
+		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
+			if err := recipePkg.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -160,7 +219,7 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenUpgrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenUpgrade, err = recipePkg.ChaosVersion(app.Recipe)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -189,16 +248,34 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
+		config.SetUpdateLabel(compose, stackName, app.Env)
 
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
 			logrus.Fatal(err)
 		}
+		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, stackName, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_UPGRADE_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+			}
+		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/version.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"sort"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -10,10 +11,21 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
+	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
 // getImagePath returns the image name
 func getImagePath(image string) (string, error) {
 	img, err := reference.ParseNormalizedNamed(image)
@@ -37,14 +49,11 @@ var appVersionCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
-	Usage:  "Show app versions",
-	Description: `
-This command shows all information about versioning related to a deployed app.
-This includes the individual image names, tags and digests. But also the Co-op
-Cloud recipe version.
-`,
+	Usage:        "Show version info of a deployed app",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
@@ -61,15 +70,15 @@ Cloud recipe version.
 			logrus.Fatal(err)
 		}
 
-		if deployedVersion == "unknown" {
-			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
-		}
-
 		if !isDeployed {
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe)
+		if deployedVersion == "unknown" {
+			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
+		}
+
+		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -85,17 +94,24 @@ Cloud recipe version.
 			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"version", "service", "image", "digest"}
+		tableCol := []string{"version", "service", "image", "tag"}
 		table := formatter.CreateTable(tableCol)
-		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
+		var versions [][]string
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Digest})
+			versions = append(versions, []string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Tag})
 		}
 
+		sort.Slice(versions, sortServiceByName(versions))
+
+		for _, version := range versions {
+			table.Append(version)
+		}
+
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+		table.SetAlignment(tablewriter.ALIGN_LEFT)
 		table.Render()
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/volume.go

@@ -7,6 +7,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -26,12 +27,17 @@ var appVolumeListCommand = cli.Command{
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		filters, err := app.Filters(false, true)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		volumeList, err := client.GetVolumes(context.Background(), app.Server, filters)
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -77,15 +83,30 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		internal.ForceFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+		}
+
 		filters, err := app.Filters(false, true)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		volumeList, err := client.GetVolumes(context.Background(), app.Server, filters)
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -109,16 +130,19 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 			volumesToRemove = volumeNames
 		}
 
-		err = client.RemoveVolumes(context.Background(), app.Server, volumesToRemove, internal.Force)
+		if len(volumesToRemove) > 0 {
+			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
 			logrus.Info("volumes removed successfully")
+		} else {
+			logrus.Info("no volumes removed")
+		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appVolumeCommand = cli.Command{

cli/catalogue/catalogue.go

@@ -8,56 +8,16 @@ import (
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/limit"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                   true,
-	"abra-apps":              true,
-	"abra-aur":               true,
-	"abra-bash":              true,
-	"abra-capsul":            true,
-	"abra-gandi":             true,
-	"abra-hetzner":           true,
-	"apps":                   true,
-	"aur-abra-git":           true,
-	"auto-apps-json":         true,
-	"auto-mirror":            true,
-	"backup-bot":             true,
-	"backup-bot-two":         true,
-	"beta.coopcloud.tech":    true,
-	"comrade-renovate-bot":   true,
-	"coopcloud.tech":         true,
-	"coturn":                 true,
-	"docker-cp-deploy":       true,
-	"docker-dind-bats-kcov":  true,
-	"docs.coopcloud.tech":    true,
-	"drone-abra":             true,
-	"example":                true,
-	"gardening":              true,
-	"go-abra":                true,
-	"organising":             true,
-	"outline-with-patch":     true,
-	"pyabra":                 true,
-	"radicle-seed-node":      true,
-	"recipes-catalogue-json": true,
-	"recipes-wishlist":       true,
-	"recipes.coopcloud.tech": true,
-	"stack-ssh-deploy":       true,
-	"swarm-cronjob":          true,
-	"tagcmp":                 true,
-	"traefik-cert-dumper":    true,
-	"tyop":                   true,
-}
-
 var catalogueGenerateCommand = cli.Command{
 	Name:    "generate",
 	Aliases: []string{"g"},
@@ -68,16 +28,19 @@ var catalogueGenerateCommand = cli.Command{
 		internal.PublishFlag,
 		internal.DryFlag,
 		internal.SkipUpdatesFlag,
+		internal.ChaosFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
-This command generates a new copy of the recipe catalogue which can be found on:
+Generate a new copy of the recipe catalogue which can be found on:
 
-    https://recipes.coopcloud.tech
+    https://recipes.coopcloud.tech (website that humans read)
+    https://recipes.coopcloud.tech/recipes.json (JSON that Abra reads)
 
 It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
-listing, parses README.md and git tags of those repositories to produce recipe
-metadata and produces a recipes JSON file.
+listing, parses README.md and git tags to produce recipe metadata which is
+loaded into the catalogue JSON file.
 
 It is possible to generate new metadata for a single recipe by passing
 <recipe>. The existing local catalogue will be updated, not overwritten.
@@ -86,15 +49,23 @@ It is quite easy to get rate limited by Docker Hub when running this command.
 If you have a Hub account you can have Abra log you in to avoid this. Pass
 "--user" and "--pass".
 
-Push your new release git.coopcloud.tech with "-p/--publish". This requires
+Push your new release to git.coopcloud.tech with "-p/--publish". This requires
 that you have permission to git push to these repositories and have your SSH
 keys configured on your account.
 `,
 	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
+
 		if recipeName != "" {
-			internal.ValidateRecipe(c, true)
+			internal.ValidateRecipe(c)
+		}
+
+		if !internal.Chaos {
+			if err := catalogue.EnsureIsClean(); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 		repos, err := recipe.ReadReposMetadata()
@@ -114,7 +85,7 @@ keys configured on your account.
 
 		if !internal.SkipUpdates {
 			logrus.Warn(logMsg)
-			if err := updateRepositories(repos, recipeName); err != nil {
+			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -127,12 +98,12 @@ keys configured on your account.
 				continue
 			}
 
-			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
+			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
 				catlBar.Add(1)
 				continue
 			}
 
-			versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
+			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
 			}
@@ -168,7 +139,7 @@ keys configured on your account.
 				logrus.Fatal(err)
 			}
 		} else {
-			catlFS, err := recipe.ReadRecipeCatalogue()
+			catlFS, err := recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -242,7 +213,6 @@ keys configured on your account.
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }
 
 // CatalogueCommand defines the `abra catalogue` command and sub-commands.
@@ -256,62 +226,3 @@ var CatalogueCommand = cli.Command{
 		catalogueGenerateCommand,
 	},
 }
-
-func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error {
-	var barLength int
-	if recipeName != "" {
-		barLength = 1
-	} else {
-		barLength = len(repos)
-	}
-
-	cloneLimiter := limit.New(10)
-
-	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
-	ch := make(chan string, barLength)
-	for _, repoMeta := range repos {
-		go func(rm recipe.RepoMeta) {
-			cloneLimiter.Begin()
-			defer cloneLimiter.End()
-
-			if recipeName != "" && recipeName != rm.Name {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
-			if _, exists := CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
-
-			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
-			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
-				logrus.Fatal(err)
-			}
-
-			isClean, err := gitPkg.IsClean(recipeDir)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !isClean {
-				logrus.Fatalf("%s has locally unstaged changes", rm.Name)
-			}
-
-			if err := recipe.EnsureUpToDate(rm.Name); err != nil {
-				logrus.Fatal(err)
-			}
-
-			ch <- rm.Name
-			retrieveBar.Add(1)
-		}(repoMeta)
-	}
-
-	for range repos {
-		<-ch // wait for everything
-	}
-
-	return nil
-}

cli/cli.go

@@ -12,10 +12,10 @@ import (
 	"coopcloud.tech/abra/cli/catalogue"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/cli/recipe"
-	"coopcloud.tech/abra/cli/record"
 	"coopcloud.tech/abra/cli/server"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -27,19 +27,13 @@ var AutoCompleteCommand = cli.Command{
 	Aliases: []string{"ac"},
 	Usage:   "Configure shell autocompletion (recommended)",
 	Description: `
-This command helps set up autocompletion in your shell by downloading the
-relevant autocompletion files and laying out what additional information must
-be loaded.
+Set up auto-completion in your shell by downloading the relevant files and
+laying out what additional information must be loaded. Supported shells are as
+follows: bash, fish, fizsh & zsh.
 
 Example:
 
     abra autocomplete bash
-
-Supported shells are as follows:
-
-		fizsh
-		zsh
-    bash
 `,
 	ArgsUsage: "<shell>",
 	Flags: []cli.Flag{
@@ -56,6 +50,7 @@ Supported shells are as follows:
 			"bash":  true,
 			"zsh":   true,
 			"fizsh": true,
+			"fish":  true,
 		}
 
 		if _, ok := supportedShells[shellType]; !ok {
@@ -86,19 +81,27 @@ Supported shells are as follows:
 		switch shellType {
 		case "bash":
 			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install autocompletion
+# Run the following commands to install auto-completion
 sudo mkdir /etc/bash_completion.d/
 sudo cp %s /etc/bash_completion.d/abra
 echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
 `, autocompletionFile))
 		case "zsh":
 			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install autocompletion
+# Run the following commands to install auto-completion
 sudo mkdir /etc/zsh/completion.d/
 sudo cp %s /etc/zsh/completion.d/abra
 echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
+`, autocompletionFile))
+		case "fish":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install auto-completion
+sudo mkdir -p /etc/fish/completions
+sudo cp %s /etc/fish/completions/abra
+echo "source /etc/fish/completions/abra" >> ~/.config/fish/config.fish
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
 `, autocompletionFile))
 		}
 
@@ -112,13 +115,11 @@ var UpgradeCommand = cli.Command{
 	Aliases: []string{"u"},
 	Usage:   "Upgrade Abra itself",
 	Description: `
-This command allows you to upgrade Abra in-place with the latest stable or
-release candidate.
+Upgrade Abra in-place with the latest stable or release candidate.
 
-If you would like to install the latest release candidate, please pass the
-"-r/--rc" option. Please bear in mind that the latest release candidate may
-have some catastrophic bugs contained in it. In any case, thank you very much
-for the testing efforts!
+Pass "-r/--rc" to install the latest release candidate. Please bear in mind
+that it may contain catastrophic bugs. Thank you very much for the testing
+efforts!
 `,
 	Flags: []cli.Flag{internal.RCFlag},
 	Action: func(c *cli.Context) error {
@@ -157,7 +158,6 @@ func newAbraApp(version, commit string) *cli.App {
 			server.ServerCommand,
 			recipe.RecipeCommand,
 			catalogue.CatalogueCommand,
-			record.RecordCommand,
 			UpgradeCommand,
 			AutoCompleteCommand,
 		},
@@ -169,10 +169,10 @@ func newAbraApp(version, commit string) *cli.App {
 	app.Before = func(c *cli.Context) error {
 		paths := []string{
 			config.ABRA_DIR,
-			path.Join(config.SERVERS_DIR),
-			path.Join(config.RECIPES_DIR),
-			path.Join(config.VENDOR_DIR),
-			path.Join(config.BACKUP_DIR),
+			config.SERVERS_DIR,
+			config.RECIPES_DIR,
+			config.VENDOR_DIR,
+			config.BACKUP_DIR,
 		}
 
 		for _, path := range paths {
@@ -184,10 +184,19 @@ func newAbraApp(version, commit string) *cli.App {
 			}
 		}
 
+		if _, err := os.Stat(config.CATALOGUE_DIR); os.IsNotExist(err) {
+			url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+			logrus.Warnf("local recipe catalogue is missing, retrieving now")
+			if err := git.Clone(config.CATALOGUE_DIR, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		logrus.Debugf("abra version %s, commit %s", version, commit)
 
 		return nil
 	}
+
 	return app
 }
 

cli/internal/cli.go

@@ -58,15 +58,14 @@ var ChaosFlag = &cli.BoolFlag{
 	Destination: &Chaos,
 }
 
-// DNSProvider specifies a DNS provider.
-var DNSProvider string
+// Disable tty to run commands from script
+var Tty bool
 
-// DNSProviderFlag selects a DNS provider.
-var DNSProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Value:       "",
-	Usage:       "DNS provider",
-	Destination: &DNSProvider,
+// TtyFlag turns on/off tty mode.
+var TtyFlag = &cli.BoolFlag{
+	Name:        "tty, T",
+	Usage:       "Disables TTY mode to run this command from a script.",
+	Destination: &Tty,
 }
 
 var NoInput bool
@@ -76,163 +75,6 @@ var NoInputFlag = &cli.BoolFlag{
 	Destination: &NoInput,
 }
 
-var DNSType string
-
-var DNSTypeFlag = &cli.StringFlag{
-	Name:        "record-type, rt",
-	Value:       "",
-	Usage:       "Domain name record type (e.g. A)",
-	Destination: &DNSType,
-}
-
-var DNSName string
-
-var DNSNameFlag = &cli.StringFlag{
-	Name:        "record-name, rn",
-	Value:       "",
-	Usage:       "Domain name record name (e.g. mysubdomain)",
-	Destination: &DNSName,
-}
-
-var DNSValue string
-
-var DNSValueFlag = &cli.StringFlag{
-	Name:        "record-value, rv",
-	Value:       "",
-	Usage:       "Domain name record value (e.g. 192.168.1.1)",
-	Destination: &DNSValue,
-}
-
-var DNSTTL string
-var DNSTTLFlag = &cli.StringFlag{
-	Name:        "record-ttl, rl",
-	Value:       "600s",
-	Usage:       "Domain name TTL value (seconds)",
-	Destination: &DNSTTL,
-}
-
-var DNSPriority int
-
-var DNSPriorityFlag = &cli.IntFlag{
-	Name:        "record-priority, rp",
-	Value:       10,
-	Usage:       "Domain name priority value",
-	Destination: &DNSPriority,
-}
-
-var ServerProvider string
-
-var ServerProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Usage:       "3rd party server provider",
-	Destination: &ServerProvider,
-}
-
-var CapsulInstanceURL string
-
-var CapsulInstanceURLFlag = &cli.StringFlag{
-	Name:        "capsul-url, cu",
-	Value:       "yolo.servers.coop",
-	Usage:       "capsul instance URL",
-	Destination: &CapsulInstanceURL,
-}
-
-var CapsulName string
-
-var CapsulNameFlag = &cli.StringFlag{
-	Name:        "capsul-name, cn",
-	Value:       "",
-	Usage:       "capsul name",
-	Destination: &CapsulName,
-}
-
-var CapsulType string
-
-var CapsulTypeFlag = &cli.StringFlag{
-	Name:        "capsul-type, ct",
-	Value:       "f1-xs",
-	Usage:       "capsul type",
-	Destination: &CapsulType,
-}
-
-var CapsulImage string
-
-var CapsulImageFlag = &cli.StringFlag{
-	Name:        "capsul-image, ci",
-	Value:       "debian10",
-	Usage:       "capsul image",
-	Destination: &CapsulImage,
-}
-
-var CapsulSSHKeys cli.StringSlice
-var CapsulSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "capsul-ssh-keys, cs",
-	Usage: "capsul SSH key",
-	Value: &CapsulSSHKeys,
-}
-
-var CapsulAPIToken string
-
-var CapsulAPITokenFlag = &cli.StringFlag{
-	Name:        "capsul-token, ca",
-	Usage:       "capsul API token",
-	EnvVar:      "CAPSUL_TOKEN",
-	Destination: &CapsulAPIToken,
-}
-
-var HetznerCloudName string
-
-var HetznerCloudNameFlag = &cli.StringFlag{
-	Name:        "hetzner-name, hn",
-	Value:       "",
-	Usage:       "hetzner cloud name",
-	Destination: &HetznerCloudName,
-}
-
-var HetznerCloudType string
-
-var HetznerCloudTypeFlag = &cli.StringFlag{
-	Name:        "hetzner-type, ht",
-	Usage:       "hetzner cloud type",
-	Destination: &HetznerCloudType,
-	Value:       "cx11",
-}
-
-var HetznerCloudImage string
-
-var HetznerCloudImageFlag = &cli.StringFlag{
-	Name:        "hetzner-image, hi",
-	Usage:       "hetzner cloud image",
-	Value:       "debian-10",
-	Destination: &HetznerCloudImage,
-}
-
-var HetznerCloudSSHKeys cli.StringSlice
-
-var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "hetzner-ssh-keys, hs",
-	Usage: "hetzner cloud SSH keys (e.g. me@foo.com)",
-	Value: &HetznerCloudSSHKeys,
-}
-
-var HetznerCloudLocation string
-
-var HetznerCloudLocationFlag = &cli.StringFlag{
-	Name:        "hetzner-location, hl",
-	Usage:       "hetzner cloud server location",
-	Value:       "hel1",
-	Destination: &HetznerCloudLocation,
-}
-
-var HetznerCloudAPIToken string
-
-var HetznerCloudAPITokenFlag = &cli.StringFlag{
-	Name:        "hetzner-token, ha",
-	Usage:       "hetzner cloud API token",
-	EnvVar:      "HCLOUD_TOKEN",
-	Destination: &HetznerCloudAPIToken,
-}
-
 // Debug stores the variable from DebugFlag.
 var Debug bool
 
@@ -243,6 +85,26 @@ var DebugFlag = &cli.BoolFlag{
 	Usage:       "Show DEBUG messages",
 }
 
+// Offline stores the variable from OfflineFlag.
+var Offline bool
+
+// DebugFlag turns on/off offline mode.
+var OfflineFlag = &cli.BoolFlag{
+	Name:        "offline, o",
+	Destination: &Offline,
+	Usage:       "Prefer offline & filesystem access when possible",
+}
+
+// MachineReadable stores the variable from MachineReadableFlag
+var MachineReadable bool
+
+// MachineReadableFlag turns on/off machine readable output where supported
+var MachineReadableFlag = &cli.BoolFlag{
+	Name:        "machine, m",
+	Destination: &MachineReadable,
+	Usage:       "Output in a machine-readable format (where supported)",
+}
+
 // RC signifies the latest release candidate
 var RC bool
 
@@ -250,7 +112,7 @@ var RC bool
 var RCFlag = &cli.BoolFlag{
 	Name:        "rc, r",
 	Destination: &RC,
-	Usage:       "Insatll the latest release candidate",
+	Usage:       "Install the latest release candidate",
 }
 
 var Major bool
@@ -318,6 +180,14 @@ var StdErrOnlyFlag = &cli.BoolFlag{
 	Destination: &StdErrOnly,
 }
 
+var SinceLogs string
+var SinceLogsFlag = &cli.StringFlag{
+	Name:        "since, S",
+	Value:       "",
+	Usage:       "tail logs since YYYY-MM-DDTHH:MM:SSZ",
+	Destination: &SinceLogs,
+}
+
 var DontWaitConverge bool
 var DontWaitConvergeFlag = &cli.BoolFlag{
 	Name:        "no-converge-checks, c",
@@ -353,72 +223,20 @@ var AllTagsFlag = &cli.BoolFlag{
 	Destination: &AllTags,
 }
 
-// SSHFailMsg is a hopefully helpful SSH failure message
-var SSHFailMsg = `
-Woops, Abra is unable to connect to connect to %s.
+var LocalCmd bool
+var LocalCmdFlag = &cli.BoolFlag{
+	Name:        "local, l",
+	Usage:       "Run command locally",
+	Destination: &LocalCmd,
+}
 
-Here are a few tips for debugging your local SSH config. Abra uses plain 'ol
-SSH to make connections to servers, so if your SSH config is working, Abra is
-working.
-
-In the first place, Abra will always try to read your Docker context connection
-string for SSH connection details. You can view your server context configs
-with the following command. Are they correct?
-
-    abra server ls
-
-Is your ssh-agent running? You can start it by running the following command:
-
-    eval "$(ssh-agent)"
-
-If your SSH private key loaded? You can check by running the following command:
-
-    ssh-add -L
-
-If, you can add it with:
-
-    ssh-add ~/.ssh/<private-key-part>
-
-If you are using a non-default public/private key, you can configure this in
-your ~/.ssh/config file which Abra will read in order to figure out connection
-details:
-
-Host foo.coopcloud.tech
-  Hostname foo.coopcloud.tech
-  User bar
-  Port 12345
-  IdentityFile ~/.ssh/bar@foo.coopcloud.tech
-
-If you're only using password authentication, you can use the following config:
-
-Host foo.coopcloud.tech
-  Hostname foo.coopcloud.tech
-  User bar
-  Port 12345
-  PreferredAuthentications=password
-  PubkeyAuthentication=no
-
-Good luck!
-
-`
-
-var ServerAddFailMsg = `
-Failed to add server %s.
-
-This could be caused by two things.
-
-Abra isn't picking up your SSH configuration or you need to specify it on the
-command-line (e.g you use a non-standard port or username to connect). Run
-"server add" with "-d/--debug" to learn more about what Abra is doing under the
-hood.
-
-Docker is not installed on your server. You can pass "-p/--provision" to
-install Docker and initialise Docker Swarm mode. See help output for "server
-add"
-
-See "abra server add -h" for more.
-
-`
+var RemoteUser string
+var RemoteUserFlag = &cli.StringFlag{
+	Name:        "user, u",
+	Value:       "",
+	Usage:       "User to run command within a service context",
+	Destination: &RemoteUser,
+}
 
 // SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
 func SubCommandBefore(c *cli.Context) error {

cli/internal/command.go

@@ -2,10 +2,109 @@ package internal
 
 import (
 	"bufio"
+	"context"
 	"fmt"
+	"io/ioutil"
 	"os/exec"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
 )
 
+// RunCmdRemote executes an abra.sh command in the target service
+func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
+
+	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(abraSh, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
+		return err
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	shell := "/bin/bash"
+	findShell := []string{"test", "-e", shell}
+	execCreateOpts := types.ExecConfig{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          findShell,
+		Detach:       false,
+		Tty:          false,
+	}
+
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
+		shell = "/bin/sh"
+	}
+
+	var cmd []string
+	if cmdArgs != "" {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.Name, app.StackName(), cmdName, cmdArgs)}
+	} else {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.Name, app.StackName(), cmdName)}
+	}
+
+	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
+
+	if RemoteUser != "" {
+		logrus.Debugf("running command with user %s", RemoteUser)
+		execCreateOpts.User = RemoteUser
+	}
+
+	execCreateOpts.Cmd = cmd
+	execCreateOpts.Tty = true
+	if Tty {
+		execCreateOpts.Tty = false
+	}
+
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func EnsureCommand(abraSh, recipeName, execCmd string) error {
+	bytes, err := ioutil.ReadFile(abraSh)
+	if err != nil {
+		return err
+	}
+
+	if !strings.Contains(string(bytes), execCmd) {
+		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
+	}
+
+	return nil
+}
+
 // RunCmd runs a shell command and streams stdout/stderr in real-time.
 func RunCmd(cmd *exec.Cmd) error {
 	r, err := cmd.StdoutPipe()

cli/internal/deploy.go

@@ -1,157 +1,136 @@
 package internal
 
 import (
-	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/lint"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
-// DeployAction is the main command-line action for this package
-func DeployAction(c *cli.Context) error {
-	app := ValidateApp(c)
+// NewVersionOverview shows an upgrade or downgrade overview
+func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
+	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
+	table := formatter.CreateTable(tableCol)
 
-	if !Chaos {
-		if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
 	}
 
-	r, err := recipe.Get(app.Recipe)
-	if err != nil {
-		logrus.Fatal(err)
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
 	}
 
-	if err := lint.LintForErrors(r); err != nil {
-		logrus.Fatal(err)
-	}
+	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
+	table.Render()
 
-	cl, err := client.New(app.Server)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("checking whether %s is already deployed", app.StackName())
-
-	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if isDeployed {
-		if Force || Chaos {
-			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
+	if releaseNotes != "" && newVersion != "" {
+		fmt.Println()
+		fmt.Print(releaseNotes)
 	} else {
-			logrus.Fatalf("%s is already deployed", app.Name)
-		}
+		logrus.Warnf("no release notes available for %s", newVersion)
 	}
 
-	version := deployedVersion
-	if version == "unknown" && !Chaos {
-		catl, err := recipe.ReadRecipeCatalogue()
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with deployment?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}
+
+// GetReleaseNotes prints release notes for a recipe version
+func GetReleaseNotes(recipeName, version string) (string, error) {
+	if version == "" {
+		return "", nil
+	}
+
+	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
+
+	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
+		releaseNotes, err := ioutil.ReadFile(fpath)
 		if err != nil {
-			logrus.Fatal(err)
+			return "", err
 		}
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		withTitle := fmt.Sprintf("%s release notes:\n%s", version, string(releaseNotes))
+		return withTitle, nil
+	}
+
+	return "", nil
+}
+
+// PostCmds parses a string of commands and executes them inside of the respective services
+// the commands string must have the following format:
+// "<service> <command> <arguments>|<service> <command> <arguments>|... "
+func PostCmds(cl *dockerClient.Client, app config.App, commands string) error {
+	abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
+	if _, err := os.Stat(abraSh); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf(fmt.Sprintf("%s does not exist for %s?", abraSh, app.Name))
+		}
+		return err
+	}
+
+	for _, command := range strings.Split(commands, "|") {
+		commandParts := strings.Split(command, " ")
+		if len(commandParts) < 2 {
+			return fmt.Errorf(fmt.Sprintf("not enough arguments: %s", command))
+		}
+		targetServiceName := commandParts[0]
+		cmdName := commandParts[1]
+		parsedCmdArgs := ""
+		if len(commandParts) > 2 {
+			parsedCmdArgs = fmt.Sprintf("%s ", strings.Join(commandParts[2:], " "))
+		}
+		logrus.Infof("running post-command '%s %s' in container %s", cmdName, parsedCmdArgs, targetServiceName)
+
+		if err := EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			return err
+		}
+
+		serviceNames, err := config.GetAppServiceNames(app.Name)
 		if err != nil {
-			logrus.Fatal(err)
-		}
-		if len(versions) > 0 {
-			version = versions[len(versions)-1]
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			head, err := git.GetRecipeHead(app.Recipe)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			version = formatter.SmallSHA(head.String())
-			logrus.Warn("no versions detected, using latest commit")
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
+			return err
 		}
+
+		matchingServiceName := false
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
 			}
 		}
 
-	if version == "unknown" && !Chaos {
-		logrus.Debugf("choosing %s as version to deploy", version)
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-			logrus.Fatal(err)
-		}
+		if !matchingServiceName {
+			return fmt.Errorf(fmt.Sprintf("no service %s for %s?", targetServiceName, app.Name))
 		}
 
-	if version != "unknown" && !Chaos {
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-			logrus.Fatal(err)
-		}
-	}
+		logrus.Debugf("running command %s %s within the context of %s_%s", cmdName, parsedCmdArgs, app.StackName(), targetServiceName)
 
-	if Chaos {
-		logrus.Warnf("chaos mode engaged")
-		var err error
-		version, err = recipe.ChaosVersion(app.Recipe)
-		if err != nil {
-			logrus.Fatal(err)
+		Tty = true
+		if err := RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			return err
 		}
 	}
-
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	for k, v := range abraShEnv {
-		app.Env[k] = v
-	}
-
-	composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	deployOpts := stack.Deploy{
-		Composefiles: composeFiles,
-		Namespace:    app.StackName(),
-		Prune:        false,
-		ResolveImage: stack.ResolveImageAlways,
-	}
-	compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if !NoDomainChecks {
-		domainName := app.Env["DOMAIN"]
-		if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
-			logrus.Fatal(err)
-		}
-	} else {
-		logrus.Warn("skipping domain checks as requested")
-	}
-
-	if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, DontWaitConverge); err != nil {
-		logrus.Fatal(err)
-	}
-
 	return nil
 }
 
@@ -192,75 +171,3 @@ func DeployOverview(app config.App, version, message string) error {
 
 	return nil
 }
-
-// NewVersionOverview shows an upgrade or downgrade overview
-func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
-	table := formatter.CreateTable(tableCol)
-
-	deployConfig := "compose.yml"
-	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
-		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
-	}
-
-	server := app.Server
-	if app.Server == "default" {
-		server = "local"
-	}
-
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
-	table.Render()
-
-	if releaseNotes == "" {
-		var err error
-		releaseNotes, err = GetReleaseNotes(app.Recipe, newVersion)
-		if err != nil {
-			return err
-		}
-	}
-
-	if releaseNotes != "" && newVersion != "" {
-		fmt.Println()
-		fmt.Println(fmt.Sprintf("%s release notes:\n\n%s", newVersion, releaseNotes))
-	} else {
-		logrus.Warnf("no release notes available for %s", newVersion)
-	}
-
-	if NoInput {
-		return nil
-	}
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with deployment?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	return nil
-}
-
-// GetReleaseNotes prints release notes for a recipe version
-func GetReleaseNotes(recipeName, version string) (string, error) {
-	if version == "" {
-		return "", nil
-	}
-
-	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
-
-	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
-		releaseNotes, err := ioutil.ReadFile(fpath)
-		if err != nil {
-			return "", err
-		}
-		return string(releaseNotes), nil
-	}
-
-	return "", nil
-}

cli/internal/new.go

@@ -1,196 +0,0 @@
-package internal
-
-import (
-	"fmt"
-	"path"
-
-	"coopcloud.tech/abra/pkg/app"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/secret"
-	"coopcloud.tech/abra/pkg/ssh"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/olekukonko/tablewriter"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// AppSecrets represents all app secrest
-type AppSecrets map[string]string
-
-// RecipeName is used for configuring recipe name programmatically
-var RecipeName string
-
-// createSecrets creates all secrets for a new app.
-func createSecrets(sanitisedAppName string) (AppSecrets, error) {
-	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", Domain))
-	appEnv, err := config.ReadEnv(appEnvPath)
-	if err != nil {
-		return nil, err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
-	secrets, err := secret.GenerateSecrets(secretEnvVars, sanitisedAppName, NewAppServer)
-	if err != nil {
-		return nil, err
-	}
-
-	if Pass {
-		for secretName := range secrets {
-			secretValue := secrets[secretName]
-			if err := secret.PassInsertSecret(secretValue, secretName, Domain, NewAppServer); err != nil {
-				return nil, err
-			}
-		}
-	}
-	return secrets, nil
-}
-
-// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
-func ensureDomainFlag(recipe recipe.Recipe, server string) error {
-	if Domain == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify app domain",
-			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
-		}
-		if err := survey.AskOne(prompt, &Domain); err != nil {
-			return err
-		}
-	}
-
-	if Domain == "" {
-		return fmt.Errorf("no domain provided")
-	}
-
-	return nil
-}
-
-// promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(appName string) error {
-	app, err := app.Get(appName)
-	if err != nil {
-		return err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(app.Env)
-	if len(secretEnvVars) == 0 {
-		logrus.Debugf("%s has no secrets to generate, skipping...", app.Recipe)
-		return nil
-	}
-
-	if !Secrets && !NoInput {
-		prompt := &survey.Confirm{
-			Message: "Generate app secrets?",
-		}
-		if err := survey.AskOne(prompt, &Secrets); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
-func ensureServerFlag() error {
-	servers, err := config.GetServers()
-	if err != nil {
-		return err
-	}
-
-	if NewAppServer == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select app server:",
-			Options: servers,
-		}
-		if err := survey.AskOne(prompt, &NewAppServer); err != nil {
-			return err
-		}
-	}
-
-	if NewAppServer == "" {
-		return fmt.Errorf("no server provided")
-	}
-
-	return nil
-}
-
-// NewAction is the new app creation logic
-func NewAction(c *cli.Context) error {
-	recipe := ValidateRecipeWithPrompt(c, false)
-
-	if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ensureServerFlag(); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ensureDomainFlag(recipe, NewAppServer); err != nil {
-		logrus.Fatal(err)
-	}
-
-	sanitisedAppName := config.SanitiseAppName(Domain)
-	logrus.Debugf("%s sanitised as %s for new app", Domain, sanitisedAppName)
-
-	if err := config.TemplateAppEnvSample(recipe.Name, Domain, NewAppServer, Domain); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := promptForSecrets(Domain); err != nil {
-		logrus.Fatal(err)
-	}
-
-	var secrets AppSecrets
-	var secretTable *tablewriter.Table
-	if Secrets {
-		if err := ssh.EnsureHostKey(NewAppServer); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var err error
-		secrets, err = createSecrets(sanitisedAppName)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretCols := []string{"Name", "Value"}
-		secretTable = formatter.CreateTable(secretCols)
-		for secret := range secrets {
-			secretTable.Append([]string{secret, secrets[secret]})
-		}
-
-	}
-
-	if NewAppServer == "default" {
-		NewAppServer = "local"
-	}
-
-	tableCol := []string{"server", "recipe", "domain"}
-	table := formatter.CreateTable(tableCol)
-	table.Append([]string{NewAppServer, recipe.Name, Domain})
-
-	fmt.Println("")
-	fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
-	fmt.Println("")
-	table.Render()
-	fmt.Println("")
-	fmt.Println("You can configure this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app config %s", Domain))
-	fmt.Println("")
-	fmt.Println("You can deploy this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", Domain))
-	fmt.Println("")
-
-	if len(secrets) > 0 {
-		fmt.Println("Here are your generated secrets:")
-		fmt.Println("")
-		secretTable.Render()
-		fmt.Println("")
-		logrus.Warn("generated secrets are not shown again, please take note of them *now*")
-	}
-
-	return nil
-}

cli/internal/validate.go

@@ -2,62 +2,24 @@ package internal
 
 import (
 	"errors"
-	"fmt"
-	"os"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-// AppName is used for configuring app name programmatically
-var AppName string
-
 // ValidateRecipe ensures the recipe arg is valid.
-func ValidateRecipe(c *cli.Context, ensureLatest bool) recipe.Recipe {
-	recipeName := c.Args().First()
-
-	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
-	}
-
-	chosenRecipe, err := recipe.Get(recipeName)
-	if err != nil {
-		if c.Command.Name == "generate" {
-			if strings.Contains(err.Error(), "missing a compose") {
-				logrus.Fatal(err)
-			}
-			logrus.Warn(err)
-		} else {
-			logrus.Fatal(err)
-		}
-	}
-
-	if ensureLatest {
-		if err := recipe.EnsureLatest(recipeName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	logrus.Debugf("validated %s as recipe argument", recipeName)
-
-	return chosenRecipe
-}
-
-// ValidateRecipeWithPrompt ensures a recipe argument is present before
-// validating, asking for input if required.
-func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
 
 	if recipeName == "" && !NoInput {
 		var recipes []string
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipe.ReadRecipeCatalogue(Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -91,23 +53,22 @@ func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
 		}
 	}
 
-	if RecipeName != "" {
-		recipeName = RecipeName
-		logrus.Debugf("programmatically setting recipe name to %s", recipeName)
-	}
-
 	if recipeName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName)
+	chosenRecipe, err := recipe.Get(recipeName, Offline)
 	if err != nil {
+		if c.Command.Name == "generate" {
+			if strings.Contains(err.Error(), "missing a compose") {
 				logrus.Fatal(err)
 			}
-
-	if ensureLatest {
-		if err := recipe.EnsureLatest(recipeName); err != nil {
-			logrus.Fatal(err)
+			logrus.Warn(err)
+		} else {
+			if strings.Contains(err.Error(), "template_driver is not allowed") {
+				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
+			}
+			logrus.Fatalf("unable to validate recipe: %s", err)
 		}
 	}
 
@@ -120,11 +81,6 @@ func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
 func ValidateApp(c *cli.Context) config.App {
 	appName := c.Args().First()
 
-	if AppName != "" {
-		appName = AppName
-		logrus.Debugf("programmatically setting app name to %s", appName)
-	}
-
 	if appName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
 	}
@@ -134,14 +90,6 @@ func ValidateApp(c *cli.Context) config.App {
 		logrus.Fatal(err)
 	}
 
-	if err := recipe.EnsureExists(app.Recipe); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ssh.EnsureHostKey(app.Server); err != nil {
-		logrus.Fatal(err)
-	}
-
 	logrus.Debugf("validated %s as app argument", appName)
 
 	return app
@@ -210,309 +158,15 @@ func ValidateServer(c *cli.Context) string {
 		}
 	}
 
-	if !matched {
-		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
-	}
-
 	if serverName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
 	}
 
+	if !matched {
+		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
+	}
+
 	logrus.Debugf("validated %s as server argument", serverName)
 
 	return serverName
 }
-
-// EnsureDNSProvider ensures a DNS provider is chosen.
-func EnsureDNSProvider() error {
-	if DNSProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select DNS provider",
-			Options: []string{"gandi"},
-		}
-
-		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
-			return err
-		}
-	}
-
-	if DNSProvider == "" {
-		return fmt.Errorf("missing DNS provider?")
-	}
-
-	return nil
-}
-
-// EnsureDNSTypeFlag ensures a DNS type flag is present.
-func EnsureDNSTypeFlag(c *cli.Context) error {
-	if DNSType == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record type",
-			Default: "A",
-		}
-		if err := survey.AskOne(prompt, &DNSType); err != nil {
-			return err
-		}
-	}
-
-	if DNSType == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSNameFlag ensures a DNS name flag is present.
-func EnsureDNSNameFlag(c *cli.Context) error {
-	if DNSName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record name",
-			Default: "mysubdomain",
-		}
-		if err := survey.AskOne(prompt, &DNSName); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSValueFlag ensures a DNS value flag is present.
-func EnsureDNSValueFlag(c *cli.Context) error {
-	if DNSValue == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record value",
-			Default: "192.168.1.2",
-		}
-		if err := survey.AskOne(prompt, &DNSValue); err != nil {
-			return err
-		}
-	}
-
-	if DNSValue == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
-	}
-
-	return nil
-}
-
-// EnsureZoneArgument ensures a zone argument is present.
-func EnsureZoneArgument(c *cli.Context) (string, error) {
-	zone := c.Args().First()
-
-	if zone == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name zone",
-			Default: "example.com",
-		}
-
-		if err := survey.AskOne(prompt, &zone); err != nil {
-			return zone, err
-		}
-	}
-
-	if zone == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
-	}
-
-	return zone, nil
-}
-
-// EnsureServerProvider ensures a 3rd party server provider is chosen.
-func EnsureServerProvider() error {
-	if ServerProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select server provider",
-			Options: []string{"capsul", "hetzner-cloud"},
-		}
-
-		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
-			return err
-		}
-	}
-
-	if ServerProvider == "" {
-		return fmt.Errorf("missing server provider?")
-	}
-
-	return nil
-}
-
-// EnsureNewCapsulVPSFlags ensure all flags are present.
-func EnsureNewCapsulVPSFlags(c *cli.Context) error {
-	if CapsulName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul name",
-		}
-		if err := survey.AskOne(prompt, &CapsulName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul instance URL",
-			Default: CapsulInstanceURL,
-		}
-		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul type",
-			Default: CapsulType,
-		}
-		if err := survey.AskOne(prompt, &CapsulType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul image",
-			Default: CapsulImage,
-		}
-		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
-			return err
-		}
-	}
-
-	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify capsul SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
-			return err
-		}
-		CapsulSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if CapsulAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("CAPSUL_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify capsul API token",
-			}
-			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
-				return err
-			}
-		} else {
-			CapsulAPIToken = token
-		}
-	}
-
-	if CapsulName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
-	}
-	if CapsulInstanceURL == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
-	}
-	if CapsulType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
-	}
-	if CapsulImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
-	}
-	if len(CapsulSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
-	}
-	if CapsulAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
-	}
-
-	return nil
-}
-
-// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
-func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
-	if HetznerCloudName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS type",
-			Default: HetznerCloudType,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS image",
-			Default: HetznerCloudImage,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
-			return err
-		}
-	}
-
-	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &sshKeys); err != nil {
-			return err
-		}
-		HetznerCloudSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS location",
-			Default: HetznerCloudLocation,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
-			return err
-		}
-	}
-
-	if HetznerCloudAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			HetznerCloudAPIToken = token
-		}
-	}
-
-	if HetznerCloudName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
-	}
-	if HetznerCloudType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
-	}
-	if HetznerCloudImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
-	}
-	if HetznerCloudLocation == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
-	}
-	if HetznerCloudAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
-	}
-
-	return nil
-}

cli/recipe/fetch.go

@@ -0,0 +1,44 @@
+package recipe
+
+import (
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeFetchCommand = cli.Command{
+	Name:        "fetch",
+	Usage:       "Fetch recipe(s)",
+	Aliases:     []string{"f"},
+	ArgsUsage:   "[<recipe>]",
+	Description: "Retrieves all recipes if no <recipe> argument is passed",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		if err := recipe.EnsureExists(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.EnsureUpToDate(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.EnsureLatest(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/lint.go

@@ -20,23 +20,59 @@ var recipeLintCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OnlyErrorFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, true)
+		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
-		tableCol := []string{"ref", "rule", "satisfied", "severity", "resolve"}
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		tableCol := []string{"ref", "rule", "severity", "satisfied", "skipped", "resolve"}
 		table := formatter.CreateTable(tableCol)
 
 		hasError := false
 		bar := formatter.CreateProgressbar(-1, "running recipe lint rules...")
 		for level := range lint.LintRules {
 			for _, rule := range lint.LintRules[level] {
+				if internal.OnlyErrors && rule.Level != "error" {
+					logrus.Debugf("skipping %s, does not have level \"error\"", rule.Ref)
+					continue
+				}
+
+				skipped := false
+				if rule.Skip(recipe) {
+					skipped = true
+				}
+
+				skippedOutput := "-"
+				if skipped {
+					skippedOutput = "yes"
+				}
+
+				satisfied := false
+				if !skipped {
 					ok, err := rule.Function(recipe)
 					if err != nil {
 						logrus.Warn(err)
@@ -46,22 +82,29 @@ var recipeLintCommand = cli.Command{
 						hasError = true
 					}
 
-				var result string
 					if ok {
-					result = "yes"
-				} else {
-					result = "NO"
+						satisfied = true
+					}
 				}
 
-				if internal.OnlyErrors {
-					if !ok && rule.Level == "error" {
-						table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
-						bar.Add(1)
+				satisfiedOutput := "yes"
+				if !satisfied {
+					satisfiedOutput = "NO"
+					if skipped {
+						satisfiedOutput = "-"
 					}
-				} else {
-					table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
-					bar.Add(1)
 				}
+
+				table.Append([]string{
+					rule.Ref,
+					rule.Description,
+					rule.Level,
+					satisfiedOutput,
+					skippedOutput,
+					rule.HowToResolve,
+				})
+
+				bar.Add(1)
 			}
 		}
 

cli/recipe/list.go

@@ -27,11 +27,13 @@ var recipeListCommand = cli.Command{
 	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.MachineReadableFlag,
 		patternFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err.Error())
 		}
@@ -66,11 +68,15 @@ var recipeListCommand = cli.Command{
 			}
 		}
 
-		table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
-
 		if table.NumLines() > 0 {
+			if internal.MachineReadable {
+				table.SetCaption(false, "")
+				table.JSONRender()
+			} else {
+				table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
 				table.Render()
 			}
+		}
 
 		return nil
 	},

cli/recipe/new.go

@@ -36,14 +36,15 @@ var recipeNewCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
 	Before:    internal.SubCommandBefore,
 	Usage:     "Create a new recipe",
 	ArgsUsage: "<recipe>",
 	Description: `
-This command creates a new recipe.
+Create a new recipe.
 
-Abra uses our built-in example repository which is available here:
+Abra uses the built-in example repository which is available here:
 
     https://git.coopcloud.tech/coop-cloud/example
 
@@ -111,7 +112,7 @@ In order to share your recipe, you can upload it the git repository to:
 
 If you're not sure how to do that, come chat with us:
 
-    https://docs.coopcloud.tech/contact
+    https://docs.coopcloud.tech/intro/contact
 
 See "abra recipe -h" for additional recipe maintainer commands.
 

cli/recipe/recipe.go

@@ -13,7 +13,8 @@ var RecipeCommand = cli.Command{
 	Description: `
 A recipe is a blueprint for an app. It is a bunch of config files which
 describe how to deploy and maintain an app. Recipes are maintained by the Co-op
-Cloud community and you can use Abra to read them and create apps for you.
+Cloud community and you can use Abra to read them, deploy them and create apps
+for you.
 
 Anyone who uses a recipe can become a maintainer. Maintainers typically make
 sure the recipe is in good working order and the config upgraded in a timely
@@ -21,12 +22,13 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 "abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands.
 `,
 	Subcommands: []cli.Command{
-		recipeListCommand,
-		recipeVersionCommand,
-		recipeReleaseCommand,
-		recipeNewCommand,
-		recipeUpgradeCommand,
-		recipeSyncCommand,
+		recipeFetchCommand,
 		recipeLintCommand,
+		recipeListCommand,
+		recipeNewCommand,
+		recipeReleaseCommand,
+		recipeSyncCommand,
+		recipeUpgradeCommand,
+		recipeVersionCommand,
 	},
 }

cli/recipe/release.go

@@ -27,17 +27,16 @@ var recipeReleaseCommand = cli.Command{
 	Usage:     "Release a new recipe version",
 	ArgsUsage: "<recipe> [<version>]",
 	Description: `
-This command is used to specify a new version of a recipe. These versions are
-then published on the Co-op Cloud recipe catalogue. These versions take the
-following form:
+Create a new version of a recipe. These versions are then published on the
+Co-op Cloud recipe catalogue. These versions take the following form:
 
     a.b.c+x.y.z
 
-Where the "a.b.c" part is a semantic version determined by the maintainer. And
-the "x.y.z" part is the image tag of the recipe "app" service (the main
-container which contains the software to be used).
+Where the "a.b.c" part is a semantic version determined by the maintainer. The
+"x.y.z" part is the image tag of the recipe "app" service (the main container
+which contains the software to be used, by naming convention).
 
-We maintain a semantic versioning scheme ("a.b.c") alongside the libre app
+We maintain a semantic versioning scheme ("a.b.c") alongside the recipe
 versioning scheme ("x.y.z") in order to maximise the chances that the nature of
 recipe updates are properly communicated. I.e. developers of an app might
 publish a minor version but that might lead to changes in the recipe which are
@@ -55,11 +54,12 @@ your SSH keys configured on your account.
 		internal.MinorFlag,
 		internal.PatchFlag,
 		internal.PublishFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, false)
+		recipe := internal.ValidateRecipe(c)
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
@@ -374,7 +374,7 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip
 	}
 
 	if lastGitTag.String() == tagString {
-		logrus.Fatalf("latest git tag (%s) and synced lable (%s) are the same?", lastGitTag, tagString)
+		logrus.Fatalf("latest git tag (%s) and synced label (%s) are the same?", lastGitTag, tagString)
 	}
 
 	if !internal.NoInput {

cli/recipe/sync.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -32,8 +31,8 @@ var recipeSyncCommand = cli.Command{
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
-This command will generate labels for the main recipe service (i.e. by
-convention, the service named 'app') which corresponds to the following format:
+Generate labels for the main recipe service (i.e. by convention, the service
+named "app") which corresponds to the following format:
 
     coop-cloud.${STACK_NAME}.version=<version>
 
@@ -41,12 +40,9 @@ Where <version> can be specifed on the command-line or Abra can attempt to
 auto-generate it for you. The <recipe> configuration will be updated on the
 local file system.
 `,
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, false)
-
-		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
-			logrus.Fatal(err)
-		}
+		recipe := internal.ValidateRecipe(c)
 
 		mainApp, err := internal.GetMainAppImage(recipe)
 		if err != nil {
@@ -68,6 +64,9 @@ local file system.
 		nextTag := c.Args().Get(1)
 		if len(tags) == 0 && nextTag == "" {
 			logrus.Warnf("no git tags found for %s", recipe.Name)
+			if internal.NoInput {
+				logrus.Fatalf("unable to continue, input required for initial version")
+			}
 			fmt.Println(fmt.Sprintf(`
 The following options are two types of initial semantic version that you can
 pick for %s that will be published in the recipe catalogue. This follows the
@@ -122,6 +121,7 @@ likely to change.
 			if err := iter.ForEach(func(ref *plumbing.Reference) error {
 				obj, err := repo.TagObject(ref.Hash())
 				if err != nil {
+					logrus.Fatal("Tag at commit ", ref.Hash(), " is unannotated or otherwise broken. Please fix it.")
 					return err
 				}
 
@@ -200,5 +200,4 @@ likely to change.
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }

cli/recipe/upgrade.go

@@ -2,6 +2,7 @@ package recipe
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -26,14 +27,23 @@ type imgPin struct {
 	version tagcmp.Tag
 }
 
+// anUpgrade represents a single service upgrade (as within a recipe), and the list of tags that it can be upgraded to,
+// for serialization purposes.
+type anUpgrade struct {
+	Service     string   `json:"service"`
+	Image       string   `json:"image"`
+	Tag         string   `json:"tag"`
+	UpgradeTags []string `json:"upgrades"`
+}
+
 var recipeUpgradeCommand = cli.Command{
 	Name:    "upgrade",
 	Aliases: []string{"u"},
 	Usage:   "Upgrade recipe image tags",
 	Description: `
-This command reads and attempts to parse all image tags within the given
-<recipe> configuration and prompt with more recent tags to upgrade to. It will
-update the relevant compose file tags on the local file system.
+Parse all image tags within the given <recipe> configuration and prompt with
+more recent tags to upgrade to. It will update the relevant compose file tags
+on the local file system.
 
 Some image tags cannot be parsed because they do not follow some sort of
 semver-like convention. In this case, all possible tags will be listed and it
@@ -47,7 +57,6 @@ You may invoke this command in "wizard" mode and be prompted for input:
 
     abra recipe upgrade
 `,
-	BashComplete: autocomplete.RecipeNameComplete,
 	ArgsUsage: "<recipe>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
@@ -55,11 +64,29 @@ You may invoke this command in "wizard" mode and be prompted for input:
 		internal.PatchFlag,
 		internal.MinorFlag,
 		internal.MajorFlag,
+		internal.MachineReadableFlag,
 		internal.AllTagsFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, true)
+		recipe := internal.ValidateRecipe(c)
+
+		if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
 
 		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
 		if bumpType != 0 {
@@ -69,6 +96,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
 			}
 		}
 
+		if internal.MachineReadable {
+			// -m implies -n in this case
+			internal.NoInput = true
+		}
+
+		upgradeList := make(map[string]anUpgrade)
+
 		// check for versions file and load pinned versions
 		versionsPresent := false
 		recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
@@ -160,7 +194,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				continue // skip on to the next tag and don't update any compose files
 			}
 
-			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name)
+			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -235,6 +269,28 @@ You may invoke this command in "wizard" mode and be prompted for input:
 						}
 					}
 
+					// there is always at least the item "skip" in compatibleStrings (a list of
+					// possible upgradable tags) and at least one other tag.
+					upgradableTags := compatibleStrings[1:]
+					upgrade := anUpgrade{
+						Service:     service.Name,
+						Image:       image,
+						Tag:         tag.String(),
+						UpgradeTags: make([]string, len(upgradableTags)),
+					}
+
+					for n, s := range upgradableTags {
+						var sb strings.Builder
+						if _, err := sb.WriteString(s); err != nil {
+						}
+						upgrade.UpgradeTags[n] = sb.String()
+					}
+
+					upgradeList[upgrade.Service] = upgrade
+
+					if internal.NoInput {
+						upgradeTag = "skip"
+					} else {
 						prompt := &survey.Select{
 							Message: msg,
 							Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
@@ -246,6 +302,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 						}
 					}
 				}
+			}
 			if upgradeTag != "skip" {
 				ok, err := recipe.UpdateTag(image, upgradeTag)
 				if err != nil {
@@ -255,10 +312,30 @@ You may invoke this command in "wizard" mode and be prompted for input:
 					logrus.Infof("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)
 				}
 			} else {
+				if !internal.NoInput {
 					logrus.Warnf("not upgrading %s, skipping as requested", image)
 				}
 			}
+		}
 
+		if internal.NoInput {
+			if internal.MachineReadable {
+				jsonstring, err := json.Marshal(upgradeList)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				fmt.Println(string(jsonstring))
+				return nil
+			}
+
+			for _, upgrade := range upgradeList {
+				logrus.Infof("can upgrade service: %s, image: %s, tag: %s ::\n", upgrade.Service, upgrade.Image, upgrade.Tag)
+				for _, utag := range upgrade.UpgradeTags {
+					logrus.Infof("    %s\n", utag)
+				}
+			}
+		}
 		return nil
 	},
 }

cli/recipe/version.go

@@ -1,54 +1,83 @@
 package recipe
 
 import (
+	"fmt"
+	"sort"
+
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/formatter"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
 var recipeVersionCommand = cli.Command{
 	Name:        "versions",
 	Aliases:     []string{"v"},
 	Usage:       "List recipe versions",
 	ArgsUsage:   "<recipe>",
+	Description: "Versions are read from the recipe catalogue.",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.MachineReadableFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, false)
+		recipe := internal.ValidateRecipe(c)
 
-		catalogue, err := recipePkg.ReadRecipeCatalogue()
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		recipeMeta, ok := catalogue[recipe.Name]
+		recipeMeta, ok := catl[recipe.Name]
 		if !ok {
-			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
+			logrus.Fatalf("%s is not published on the catalogue?", recipe.Name)
 		}
 
-		tableCol := []string{"Version", "Service", "Image", "Tag", "Digest"}
-		table := formatter.CreateTable(tableCol)
+		if len(recipeMeta.Versions) == 0 {
+			logrus.Fatalf("%s has no catalogue published versions?", recipe.Name)
+		}
 
 		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
-			for tag, meta := range recipeMeta.Versions[i] {
+			tableCols := []string{"version", "service", "image", "tag"}
+			table := formatter.CreateTable(tableCols)
+			for version, meta := range recipeMeta.Versions[i] {
+				var versions [][]string
 				for service, serviceMeta := range meta {
-					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag, serviceMeta.Digest})
-				}
-			}
+					versions = append(versions, []string{version, service, serviceMeta.Image, serviceMeta.Tag})
 				}
 
-		table.SetAutoMergeCells(true)
+				sort.Slice(versions, sortServiceByName(versions))
 
-		if table.NumLines() > 0 {
-			table.Render()
+				for _, version := range versions {
+					table.Append(version)
+				}
+
+				if internal.MachineReadable {
+					table.JSONRender()
 				} else {
-			logrus.Fatalf("%s has no published versions?", recipe.Name)
+					table.SetAutoMergeCellsByColumnIndex([]int{0})
+					table.SetAlignment(tablewriter.ALIGN_LEFT)
+					table.Render()
+					fmt.Println()
+				}
+			}
 		}
 
 		return nil

cli/record/list.go

@@ -1,82 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordListCommand lists domains.
-var RecordListCommand = cli.Command{
-	Name:      "list",
-	Usage:     "List domain name records",
-	Aliases:   []string{"ls"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.DNSProviderFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-This command lists all domain name records managed by a 3rd party provider for
-a specific zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-`,
-	Action: func(c *cli.Context) error {
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		for _, record := range records {
-			value := record.Value
-			if len(record.Value) > 30 {
-				value = fmt.Sprintf("%s...", record.Value[:30])
-			}
-
-			table.Append([]string{
-				record.Type,
-				record.Name,
-				value,
-				record.TTL.String(),
-				strconv.Itoa(record.Priority),
-			})
-		}
-
-		table.Render()
-
-		return nil
-	},
-}

cli/record/new.go

@@ -1,148 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/dns"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordNewCommand creates a new domain name record.
-var RecordNewCommand = cli.Command{
-	Name:      "new",
-	Usage:     "Create a new domain record",
-	Aliases:   []string{"n"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-		internal.DNSValueFlag,
-		internal.DNSTTLFlag,
-		internal.DNSPriorityFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-This command creates a new domain name record for a specific zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-
-Example:
-
-    abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSValueFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		ttl, err := dns.GetTTL(internal.DNSTTL)
-		if err != nil {
-			return err
-		}
-
-		record := libdns.Record{
-			Type:  internal.DNSType,
-			Name:  internal.DNSName,
-			Value: internal.DNSValue,
-			TTL:   ttl,
-		}
-
-		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
-			record.Priority = internal.DNSPriority
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, existingRecord := range records {
-			if existingRecord.Type == record.Type &&
-				existingRecord.Name == record.Name &&
-				existingRecord.Value == record.Value {
-				logrus.Fatalf("%s record for %s already exists?", record.Type, zone)
-			}
-		}
-
-		createdRecords, err := provider.SetRecords(
-			context.Background(),
-			zone,
-			[]libdns.Record{record},
-		)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(createdRecords) == 0 {
-			logrus.Fatal("provider library reports that no record was created?")
-		}
-
-		createdRecord := createdRecords[0]
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := createdRecord.Value
-		if len(createdRecord.Value) > 30 {
-			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
-		}
-
-		table.Append([]string{
-			createdRecord.Type,
-			createdRecord.Name,
-			value,
-			createdRecord.TTL.String(),
-			strconv.Itoa(createdRecord.Priority),
-		})
-
-		table.Render()
-
-		logrus.Info("record created")
-
-		return nil
-	},
-}

cli/record/record.go

@@ -1,37 +0,0 @@
-package record
-
-import (
-	"github.com/urfave/cli"
-)
-
-// RecordCommand supports managing DNS entries.
-var RecordCommand = cli.Command{
-	Name:      "record",
-	Usage:     "Manage domain name records",
-	Aliases:   []string{"rc"},
-	ArgsUsage: "<record>",
-	Description: `
-This command supports managing domain name records via 3rd party providers such
-as Gandi DNS. It supports listing, creating and removing all types of records
-that you might need for managing Co-op Cloud apps.
-
-The following providers are supported:
-
-    Gandi DNS https://www.gandi.net
-
-You need an account with such a provider already. Typically, you need to
-provide an API token on the Abra command-line when using these commands so that
-you can authenticate with your provider account.
-
-New providers can be integrated, we welcome change sets. See the underlying DNS
-library documentation for more. It supports many existing providers and allows
-to implement new provider support easily.
-
-    https://pkg.go.dev/github.com/libdns/libdns
-`,
-	Subcommands: []cli.Command{
-		RecordListCommand,
-		RecordNewCommand,
-		RecordRemoveCommand,
-	},
-}

cli/record/remove.go

@@ -1,136 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordRemoveCommand lists domains.
-var RecordRemoveCommand = cli.Command{
-	Name:      "remove",
-	Usage:     "Remove a domain name record",
-	Aliases:   []string{"rm"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-This command removes a domain name record for a specific zone.
-
-It uses the type of record and name to match existing records and choose one
-for deletion. You must specify a zone (e.g. example.com) under which your
-domain name records are listed. This zone must already be created on your
-provider account.
-
-Example:
-
-    abra record remove foo.com -p gandi -t A -n myapp
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record rm
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var toDelete libdns.Record
-		for _, record := range records {
-			if record.Type == internal.DNSType && record.Name == internal.DNSName {
-				toDelete = record
-				break
-			}
-		}
-
-		if (libdns.Record{}) == toDelete {
-			logrus.Fatal("provider library reports no matching record?")
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := toDelete.Value
-		if len(toDelete.Value) > 30 {
-			value = fmt.Sprintf("%s...", toDelete.Value[:30])
-		}
-
-		table.Append([]string{
-			toDelete.Type,
-			toDelete.Name,
-			value,
-			toDelete.TTL.String(),
-			strconv.Itoa(toDelete.Priority),
-		})
-
-		table.Render()
-
-		if !internal.NoInput {
-			response := false
-			prompt := &survey.Confirm{
-				Message: "continue with record deletion?",
-			}
-
-			if err := survey.AskOne(prompt, &response); err != nil {
-				return err
-			}
-
-			if !response {
-				logrus.Fatal("exiting as requested")
-			}
-		}
-
-		_, err = provider.DeleteRecords(context.Background(), zone, []libdns.Record{toDelete})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Info("record successfully deleted")
-
-		return nil
-	},
-}

cli/server/add.go

@@ -1,14 +1,9 @@
 package server
 
 import (
-	"context"
 	"errors"
-	"fmt"
 	"os"
-	"os/exec"
-	"os/user"
 	"path/filepath"
-	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
@@ -16,34 +11,11 @@ import (
 	contextPkg "coopcloud.tech/abra/pkg/context"
 	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/server"
-	"coopcloud.tech/abra/pkg/ssh"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/swarm"
-	dockerClient "github.com/docker/docker/client"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var (
-	dockerInstallMsg = `
-A docker installation cannot be found on %s. This is a required system
-dependency for running Co-op Cloud on your server. If you would like, Abra can
-attempt to install Docker for you using the upstream non-interactive
-installation script.
-
-See the following documentation for more:
-
-    https://docs.docker.com/engine/install/debian/#install-using-the-convenience-script
-
-N.B Docker doesn't recommend it for production environments but many use it for
-such purposes. Docker stable is now installed by default by this script. The
-source for this script can be seen here:
-
-    https://github.com/docker/docker-install
-`
-)
-
 var local bool
 var localFlag = &cli.BoolFlag{
 	Name:        "local, l",
@@ -51,106 +23,36 @@ var localFlag = &cli.BoolFlag{
 	Destination: &local,
 }
 
-var provision bool
-var provisionFlag = &cli.BoolFlag{
-	Name:        "provision, p",
-	Usage:       "Provision server so it can deploy apps",
-	Destination: &provision,
-}
-
-var sshAuth string
-var sshAuthFlag = &cli.StringFlag{
-	Name:        "ssh-auth, s",
-	Value:       "identity-file",
-	Usage:       "Select SSH authentication method (identity-file, password)",
-	Destination: &sshAuth,
-}
-
-var askSudoPass bool
-var askSudoPassFlag = &cli.BoolFlag{
-	Name:        "ask-sudo-pass, a",
-	Usage:       "Ask for sudo password",
-	Destination: &askSudoPass,
-}
-
 func cleanUp(domainName string) {
-	logrus.Warnf("cleaning up context for %s", domainName)
+	if domainName != "default" {
+		logrus.Infof("cleaning up context for %s", domainName)
 		if err := client.DeleteContext(domainName); err != nil {
 			logrus.Fatal(err)
 		}
+	}
 
-	logrus.Warnf("cleaning up server directory for %s", domainName)
-	if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, domainName)); err != nil {
+	logrus.Infof("attempting to clean up server directory for %s", domainName)
+
+	serverDir := filepath.Join(config.SERVERS_DIR, domainName)
+	files, err := config.GetAllFilesInDirectory(serverDir)
+	if err != nil {
+		logrus.Fatalf("unable to list files in %s: %s", serverDir, err)
+	}
+
+	if len(files) > 0 {
+		logrus.Warnf("aborting clean up of %s because it is not empty", serverDir)
+		return
+	}
+
+	if err := os.RemoveAll(serverDir); err != nil {
 		logrus.Fatal(err)
 	}
 }
 
-func installDockerLocal(c *cli.Context) error {
-	fmt.Println(fmt.Sprintf(dockerInstallMsg, "this local server"))
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: fmt.Sprintf("attempt install docker on local server?"),
-	}
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	for _, exe := range []string{"wget", "bash"} {
-		exists, err := ensureLocalExecutable(exe)
-		if err != nil {
-			return err
-		}
-		if !exists {
-			return fmt.Errorf("%s missing, please install it", exe)
-		}
-	}
-
-	cmd := exec.Command("bash", "-c", "wget -O- https://get.docker.com | bash")
-	if err := internal.RunCmd(cmd); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func newLocalServer(c *cli.Context, domainName string) error {
-	if err := createServerDir(domainName); err != nil {
-		return err
-	}
-
-	cl, err := newClient(c, domainName)
-	if err != nil {
-		return err
-	}
-
-	if provision {
-		exists, err := ensureLocalExecutable("docker")
-		if err != nil {
-			return err
-		}
-
-		if !exists {
-			if err := installDockerLocal(c); err != nil {
-				return err
-			}
-		}
-
-		if err := initSwarmLocal(c, cl, domainName); err != nil {
-			if !strings.Contains(err.Error(), "proxy already exists") {
-				logrus.Fatal(err)
-			}
-		}
-	}
-
-	logrus.Info("local server has been added")
-
-	return nil
-}
-
+// newContext creates a new internal Docker context for a server. This is how
+// Docker manages SSH connection details. These are stored to disk in
+// ~/.docker. Abra can manage this completely for the user, so it's an
+// implementation detail.
 func newContext(c *cli.Context, domainName, username, port string) error {
 	store := contextPkg.NewDefaultDockerContextStore()
 	contexts, err := store.Store.List()
@@ -174,187 +76,7 @@ func newContext(c *cli.Context, domainName, username, port string) error {
 	return nil
 }
 
-func newClient(c *cli.Context, domainName string) (*dockerClient.Client, error) {
-	cl, err := client.New(domainName)
-	if err != nil {
-		return &dockerClient.Client{}, err
-	}
-	return cl, nil
-}
-
-func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, domainName string) error {
-	exists, err := ensureRemoteExecutable("docker", sshCl)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		fmt.Println(fmt.Sprintf(dockerInstallMsg, domainName))
-
-		response := false
-		prompt := &survey.Confirm{
-			Message: fmt.Sprintf("attempt install docker on %s?", domainName),
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return err
-		}
-
-		if !response {
-			logrus.Fatal("exiting as requested")
-		}
-
-		exes := []string{"wget", "bash"}
-		if askSudoPass {
-			exes = append(exes, "ssh-askpass")
-		}
-
-		for _, exe := range exes {
-			exists, err := ensureRemoteExecutable(exe, sshCl)
-			if err != nil {
-				return err
-			}
-			if !exists {
-				return fmt.Errorf("%s missing on remote, please install it", exe)
-			}
-		}
-
-		var sudoPass string
-		if askSudoPass {
-			cmd := "wget -O- https://get.docker.com | bash"
-
-			prompt := &survey.Password{
-				Message: "sudo password?",
-			}
-
-			if err := survey.AskOne(prompt, &sudoPass); err != nil {
-				return err
-			}
-
-			logrus.Debugf("running %s on %s now with sudo password", cmd, domainName)
-
-			if sudoPass == "" {
-				return fmt.Errorf("missing sudo password but requested --ask-sudo-pass?")
-			}
-
-			logrus.Warn("installing docker, this could take some time...")
-
-			if err := ssh.RunSudoCmd(cmd, sudoPass, sshCl); err != nil {
-				fmt.Print(fmt.Sprintf(`
-Abra was unable to bootstrap Docker, see below for logs:
-
-
-%s
-
-If nothing works, you try running the Docker install script manually on your server:
-
-    wget -O- https://get.docker.com | bash
-
-`, string(err.Error())))
-				logrus.Fatal("Process exited with status 1")
-			}
-
-			logrus.Infof("docker is installed on %s", domainName)
-
-			remoteUser := sshCl.SSHClient.Conn.User()
-			logrus.Infof("adding %s to docker group", remoteUser)
-			permsCmd := fmt.Sprintf("sudo usermod -aG docker %s", remoteUser)
-			if err := ssh.RunSudoCmd(permsCmd, sudoPass, sshCl); err != nil {
-				return err
-			}
-		} else {
-			cmd := "wget -O- https://get.docker.com | bash"
-
-			logrus.Debugf("running %s on %s now without sudo password", cmd, domainName)
-
-			logrus.Warn("installing docker, this could take some time...")
-
-			if out, err := sshCl.Exec(cmd); err != nil {
-				fmt.Print(fmt.Sprintf(`
-Abra was unable to bootstrap Docker, see below for logs:
-
-
-%s
-
-This could be due to a number of things but one of the most common is that your
-server user account does not have sudo access, and if it does, you need to pass
-"--ask-sudo-pass" in order to supply Abra with your password.
-
-If nothing works, you try running the Docker install script manually on your server:
-
-    wget -O- https://get.docker.com | bash
-
-`, string(out)))
-				logrus.Fatal(err)
-			}
-
-			logrus.Infof("docker is installed on %s", domainName)
-		}
-	}
-
-	return nil
-}
-
-func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string) error {
-	initReq := swarm.InitRequest{ListenAddr: "0.0.0.0:2377"}
-	if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
-		if strings.Contains(err.Error(), "is already part of a swarm") ||
-			strings.Contains(err.Error(), "must specify a listening address") {
-			logrus.Infof("swarm mode already initialised on %s", domainName)
-		} else {
-			return err
-		}
-	} else {
-		logrus.Infof("initialised swarm mode on local server")
-	}
-
-	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
-	if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
-		if !strings.Contains(err.Error(), "proxy already exists") {
-			return err
-		}
-		logrus.Info("swarm overlay network already created on local server")
-	} else {
-		logrus.Infof("swarm overlay network created on local server")
-	}
-
-	return nil
-}
-
-func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error {
-	ipv4, err := dns.EnsureIPv4(domainName)
-	if err != nil {
-		return err
-	}
-
-	initReq := swarm.InitRequest{
-		ListenAddr:    "0.0.0.0:2377",
-		AdvertiseAddr: ipv4,
-	}
-	if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
-		if strings.Contains(err.Error(), "is already part of a swarm") ||
-			strings.Contains(err.Error(), "must specify a listening address") {
-			logrus.Infof("swarm mode already initialised on %s", domainName)
-		} else {
-			return err
-		}
-	} else {
-		logrus.Infof("initialised swarm mode on %s", domainName)
-	}
-
-	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
-	if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
-		if !strings.Contains(err.Error(), "proxy already exists") {
-			return err
-		}
-		logrus.Infof("swarm overlay network already created on %s", domainName)
-	} else {
-		logrus.Infof("swarm overlay network created on %s", domainName)
-	}
-
-	return nil
-}
-
+// createServerDir creates the ~/.abra/servers/... directory for a new server.
 func createServerDir(domainName string) error {
 	if err := server.CreateServerDir(domainName); err != nil {
 		if !os.IsExist(err) {
@@ -362,6 +84,7 @@ func createServerDir(domainName string) error {
 		}
 		logrus.Debugf("server dir for %s already created", domainName)
 	}
+
 	return nil
 }
 
@@ -370,34 +93,21 @@ var serverAddCommand = cli.Command{
 	Aliases: []string{"a"},
 	Usage:   "Add a server to your configuration",
 	Description: `
-This command adds a new server to your configuration so that it can be managed
-by Abra. This command can also provision your server ("--provision/-p") with a
-Docker installation so that it is capable of hosting Co-op Cloud apps.
+Add a new server to your configuration so that it can be managed by Abra.
 
-Abra will default to expecting that you have a running ssh-agent and are using
-SSH keys to connect to your new server. Abra will also read your SSH config
-(matching "Host" as <domain>). SSH connection details precedence follows as
-such: command-line > SSH config > guessed defaults.
+Abra uses the SSH command-line to discover connection details for your server.
+It is advised to configure an entry per-host in your ~/.ssh/config for each
+server. For example:
 
-If you have no SSH key configured for this host and are instead using password
-authentication, you may pass "--ssh-auth password" to have Abra ask you for the
-password. "--ask-sudo-pass" may be passed if you run your provisioning commands
-via sudo privilege escalation.
+Host example.com
+  Hostname example.com
+  User exampleUser
+  Port 12345
+  IdentityFile ~/.ssh/example@somewhere
 
-The <domain> argument must be a publicy accessible domain name which points to
-your server. You should working SSH access to this server already, Abra will
-assume port 22 and will use your current system username to make an initial
-connection. You can use the <user> and <port> arguments to adjust this.
+Abra can then load SSH connection details from this configuratiion with:
 
-Example:
-
-    abra server add varia.zone glodemodem 12345 -p
-
-Abra will construct the following SSH connection and Docker context:
-
-    ssh://globemodem@varia.zone:12345
-
-All communication between Abra and the server will use this SSH connection.
+    abra server add example.com
 
 If "--local" is passed, then Abra assumes that the current local server is
 intended as the target server. This is useful when you want to have your entire
@@ -408,104 +118,64 @@ developer machine.
 		internal.DebugFlag,
 		internal.NoInputFlag,
 		localFlag,
-		provisionFlag,
-		sshAuthFlag,
-		askSudoPassFlag,
 	},
 	Before:    internal.SubCommandBefore,
-	ArgsUsage: "<domain> [<user>] [<port>]",
+	ArgsUsage: "<domain>",
 	Action: func(c *cli.Context) error {
 		if len(c.Args()) > 0 && local || !internal.ValidateSubCmdFlags(c) {
 			err := errors.New("cannot use <domain> and --local together")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
-		if sshAuth != "password" && sshAuth != "identity-file" {
-			err := errors.New("--ssh-auth only accepts identity-file or password")
-			internal.ShowSubcommandHelpAndError(c, err)
+		var domainName string
+		if local {
+			domainName = "default"
+		} else {
+			domainName = internal.ValidateDomain(c)
 		}
 
-		domainName := internal.ValidateDomain(c)
-
 		if local {
-			if err := newLocalServer(c, "default"); err != nil {
+			if err := createServerDir(domainName); err != nil {
 				logrus.Fatal(err)
 			}
+
+			logrus.Infof("attempting to create client for %s", domainName)
+			if _, err := client.New(domainName); err != nil {
+				cleanUp(domainName)
+				logrus.Fatal(err)
+			}
+
+			logrus.Info("local server added")
+
 			return nil
 		}
 
-		username := c.Args().Get(1)
-		if username == "" {
-			systemUser, err := user.Current()
-			if err != nil {
-				return err
-			}
-			username = systemUser.Username
-		}
-
-		port := c.Args().Get(2)
-		if port == "" {
-			port = "22"
+		if _, err := dns.EnsureIPv4(domainName); err != nil {
+			logrus.Fatal(err)
 		}
 
 		if err := createServerDir(domainName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := newContext(c, domainName, username, port); err != nil {
+		hostConfig, err := sshPkg.GetHostConfig(domainName)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		cl, err := newClient(c, domainName)
-		if err != nil {
+		if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("attempting to create client for %s", domainName)
+		if _, err := client.New(domainName); err != nil {
 			cleanUp(domainName)
 			logrus.Debugf("failed to construct client for %s, saw %s", domainName, err.Error())
-			logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
+			logrus.Fatal(sshPkg.Fatal(domainName, err))
 		}
 
-		if provision {
-			logrus.Debugf("attempting to construct SSH client for %s", domainName)
-			sshCl, err := ssh.New(domainName, sshAuth, username, port)
-			if err != nil {
-				cleanUp(domainName)
-				logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
-			}
-			defer sshCl.Close()
-			logrus.Debugf("successfully created SSH client for %s", domainName)
-
-			if err := installDocker(c, cl, sshCl, domainName); err != nil {
-				logrus.Fatal(err)
-			}
-			if err := initSwarm(c, cl, domainName); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		if _, err := cl.Info(context.Background()); err != nil {
-			cleanUp(domainName)
-			logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
-		}
+		logrus.Infof("%s added", domainName)
 
 		return nil
 	},
 }
-
-// ensureLocalExecutable ensures that an executable is present on the local machine
-func ensureLocalExecutable(exe string) (bool, error) {
-	out, err := exec.Command("which", exe).Output()
-	if err != nil {
-		return false, err
-	}
-
-	return string(out) != "", nil
-}
-
-// ensureRemoteExecutable ensures that an executable is present on a remote machine
-func ensureRemoteExecutable(exe string, sshCl *ssh.Client) (bool, error) {
-	out, err := sshCl.Exec(fmt.Sprintf("which %s", exe))
-	if err != nil && string(out) != "" {
-		return false, err
-	}
-
-	return string(out) != "", nil
-}

cli/server/list.go

@@ -12,12 +12,23 @@ import (
 	"github.com/urfave/cli"
 )
 
+var problemsFilter bool
+
+var problemsFilterFlag = &cli.BoolFlag{
+	Name:        "problems, p",
+	Usage:       "Show only servers with potential connection problems",
+	Destination: &problemsFilter,
+}
+
 var serverListCommand = cli.Command{
 	Name:    "list",
 	Aliases: []string{"ls"},
 	Usage:   "List managed servers",
 	Flags: []cli.Flag{
+		problemsFilterFlag,
 		internal.DebugFlag,
+		internal.MachineReadableFlag,
+		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
@@ -29,7 +40,6 @@ var serverListCommand = cli.Command{
 
 		tableColumns := []string{"name", "host", "user", "port"}
 		table := formatter.CreateTable(tableColumns)
-		defer table.Render()
 
 		serverNames, err := config.ReadServerNames()
 		if err != nil {
@@ -44,14 +54,27 @@ var serverListCommand = cli.Command{
 					// No local context found, we can continue safely
 					continue
 				}
+
 				if ctx.Name == serverName {
 					sp, err := ssh.ParseURL(endpoint)
 					if err != nil {
 						logrus.Fatal(err)
 					}
+
+					if sp.Host == "" {
+						sp.Host = "unknown"
+					}
+					if sp.User == "" {
+						sp.User = "unknown"
+					}
+					if sp.Port == "" {
+						sp.Port = "unknown"
+					}
+
 					row = []string{serverName, sp.Host, sp.User, sp.Port}
 				}
 			}
+
 			if len(row) == 0 {
 				if serverName == "default" {
 					row = []string{serverName, "local", "n/a", "n/a"}
@@ -59,7 +82,27 @@ var serverListCommand = cli.Command{
 					row = []string{serverName, "unknown", "unknown", "unknown"}
 				}
 			}
+
+			if problemsFilter {
+				for _, val := range row {
+					if val == "unknown" {
 						table.Append(row)
+						break
+					}
+				}
+			} else {
+				table.Append(row)
+			}
+		}
+
+		if internal.MachineReadable {
+			table.JSONRender()
+		} else {
+			if problemsFilter && table.NumLines() == 0 {
+				logrus.Info("all servers wired up correctly 👏")
+			} else {
+				table.Render()
+			}
 		}
 
 		return nil

cli/server/new.go

@@ -1,262 +0,0 @@
-package server
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/libcapsul"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-func newHetznerCloudVPS(c *cli.Context) error {
-	if err := internal.EnsureNewHetznerCloudVPSFlags(c); err != nil {
-		return err
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	var sshKeysRaw []string
-	var sshKeys []*hcloud.SSHKey
-	for _, sshKey := range c.StringSlice("hetzner-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-
-		sshKey, _, err := client.SSHKey.GetByName(context.Background(), sshKey)
-		if err != nil {
-			return err
-		}
-		sshKeys = append(sshKeys, sshKey)
-		sshKeysRaw = append(sshKeysRaw, sshKey.Name)
-	}
-
-	serverOpts := hcloud.ServerCreateOpts{
-		Name:       internal.HetznerCloudName,
-		ServerType: &hcloud.ServerType{Name: internal.HetznerCloudType},
-		Image:      &hcloud.Image{Name: internal.HetznerCloudImage},
-		SSHKeys:    sshKeys,
-		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
-	}
-
-	sshKeyIDs := strings.Join(sshKeysRaw, "\n")
-	if sshKeyIDs == "" {
-		sshKeyIDs = "N/A (password auth)"
-	}
-
-	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.HetznerCloudName,
-		internal.HetznerCloudType,
-		internal.HetznerCloudImage,
-		sshKeyIDs,
-		internal.HetznerCloudLocation,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	res, _, err := client.Server.Create(context.Background(), serverOpts)
-	if err != nil {
-		return err
-	}
-
-	var rootPassword string
-	if len(sshKeys) > 0 {
-		rootPassword = "N/A (using SSH keys)"
-	} else {
-		rootPassword = res.RootPassword
-	}
-
-	ip := res.Server.PublicNet.IPv4.IP.String()
-
-	fmt.Println(fmt.Sprintf(`
-Your new Hetzner Cloud VPS has successfully been created! Here are the details:
-
-    name: %s
-    IP address: %s
-    root password: %s
-
-You can access this new VPS via SSH using the following command:
-
-    ssh root@%s
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record ("abra
-record new") and add the server to your Abra configuration ("abra server add")
-to have a working server that you can deploy Co-op Cloud apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A %s
-    *  1800 IN A %s
-
-	`,
-		internal.HetznerCloudName, ip, rootPassword,
-		ip, ip, ip,
-	))
-
-	return nil
-}
-
-func newCapsulVPS(c *cli.Context) error {
-	if err := internal.EnsureNewCapsulVPSFlags(c); err != nil {
-		return err
-	}
-
-	capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", internal.CapsulInstanceURL)
-
-	var sshKeys []string
-	for _, sshKey := range c.StringSlice("capsul-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-		sshKeys = append(sshKeys, sshKey)
-	}
-
-	tableColumns := []string{"instance", "name", "type", "image", "ssh-keys"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.CapsulInstanceURL,
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		strings.Join(sshKeys, "\n"),
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with capsul creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	capsulClient := libcapsul.New(capsulCreateURL, internal.CapsulAPIToken)
-	resp, err := capsulClient.Create(
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		sshKeys,
-	)
-	if err != nil {
-		return err
-	}
-
-	fmt.Println(fmt.Sprintf(`
-Your new Capsul has successfully been created! Here are the details:
-
-    Capsul name: %s
-    Capsul ID:   %v
-
-You will need to log into your Capsul instance web interface to retrieve the IP
-address. You can learn all about how to get SSH access to your new Capsul on:
-
-    %s/about-ssh
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record ("abra
-record new") and add the server to your Abra configuration ("abra server add")
-to have a working server that you can deploy Co-op Cloud apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A <your-capsul-ip>
-    *  1800 IN A <your-capsul-ip>
-
-	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
-
-	return nil
-}
-
-var serverNewCommand = cli.Command{
-	Name:    "new",
-	Aliases: []string{"n"},
-	Usage:   "Create a new server using a 3rd party provider",
-	Description: `
-This command creates a new server via a 3rd party provider.
-
-The following providers are supported:
-
-		Capsul https://git.cyberia.club/Cyberia/capsul-flask
-    Hetzner Cloud https://docs.hetzner.com/cloud
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-
-API tokens are read from the environment if specified, e.g.
-
-    export HCLOUD_TOKEN=...
-
-Where "$provider_TOKEN" is the expected env var format.
-`,
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.ServerProviderFlag,
-
-		// Capsul
-		internal.CapsulInstanceURLFlag,
-		internal.CapsulTypeFlag,
-		internal.CapsulImageFlag,
-		internal.CapsulSSHKeysFlag,
-		internal.CapsulAPITokenFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudTypeFlag,
-		internal.HetznerCloudImageFlag,
-		internal.HetznerCloudSSHKeysFlag,
-		internal.HetznerCloudLocationFlag,
-		internal.HetznerCloudAPITokenFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		if err := internal.EnsureServerProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		switch internal.ServerProvider {
-		case "capsul":
-			if err := newCapsulVPS(c); err != nil {
-				logrus.Fatal(err)
-			}
-		case "hetzner-cloud":
-			if err := newHetznerCloudVPS(c); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		return nil
-	},
-}

cli/server/prune.go

@@ -0,0 +1,103 @@
+package server
+
+import (
+	"context"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var allFilter bool
+
+var allFilterFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Usage:       "Remove all unused images not just dangling ones",
+	Destination: &allFilter,
+}
+
+var volumesFilter bool
+
+var volumesFilterFlag = &cli.BoolFlag{
+	Name:        "volumes, v",
+	Usage:       "Prune volumes. This will remove app data, Be Careful!",
+	Destination: &volumesFilter,
+}
+
+var serverPruneCommand = cli.Command{
+	Name:    "prune",
+	Aliases: []string{"p"},
+	Usage:   "Prune a managed server; Runs a docker system prune",
+	Description: `
+Prunes unused containers, networks, and dangling images.
+
+If passing "-v/--volumes" then volumes not connected to a deployed app will
+also be removed. This can result in unwanted data loss if not used carefully.
+	`,
+	ArgsUsage: "[<server>]",
+	Flags: []cli.Flag{
+		allFilterFlag,
+		volumesFilterFlag,
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.ServerNameComplete,
+	Action: func(c *cli.Context) error {
+		serverName := internal.ValidateServer(c)
+
+		cl, err := client.New(serverName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var args filters.Args
+
+		ctx := context.Background()
+		cr, err := cl.ContainersPrune(ctx, args)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		cntSpaceReclaimed := formatter.ByteCountSI(cr.SpaceReclaimed)
+		logrus.Infof("containers pruned: %d; space reclaimed: %s", len(cr.ContainersDeleted), cntSpaceReclaimed)
+
+		nr, err := cl.NetworksPrune(ctx, args)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("networks pruned: %d", len(nr.NetworksDeleted))
+
+		pruneFilters := filters.NewArgs()
+		if allFilter {
+			logrus.Debugf("removing all images, not only dangling ones")
+			pruneFilters.Add("dangling", "false")
+		}
+
+		ir, err := cl.ImagesPrune(ctx, pruneFilters)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
+		logrus.Infof("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed)
+
+		if volumesFilter {
+			vr, err := cl.VolumesPrune(ctx, args)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			volSpaceReclaimed := formatter.ByteCountSI(vr.SpaceReclaimed)
+			logrus.Infof("volumes pruned: %d; space reclaimed: %s", len(vr.VolumesDeleted), volSpaceReclaimed)
+		}
+
+		return nil
+	},
+}

cli/server/remove.go

@@ -1,174 +1,38 @@
 package server
 
 import (
-	"context"
-	"fmt"
 	"os"
 	"path/filepath"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var rmServer bool
-var rmServerFlag = &cli.BoolFlag{
-	Name:        "server, s",
-	Usage:       "remove the actual server also",
-	Destination: &rmServer,
-}
-
-func rmHetznerCloudVPS(c *cli.Context) error {
-	if internal.HetznerCloudName == "" && !internal.NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &internal.HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if internal.HetznerCloudAPIToken == "" && !internal.NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &internal.HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			internal.HetznerCloudAPIToken = token
-		}
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	server, _, err := client.Server.Get(context.Background(), internal.HetznerCloudName)
-	if err != nil {
-		return err
-	}
-
-	if server == nil {
-		logrus.Fatalf("library provider reports that %s doesn't exist?", internal.HetznerCloudName)
-	}
-
-	fmt.Println(fmt.Sprintf(`
-You have requested that Abra delete the following server (%s). Please be
-absolutely sure that this is indeed the server that you would like to have
-removed. There will be no going back once you confirm, the server will be
-destroyed.
-`, server.Name))
-
-	tableColumns := []string{"name", "type", "image", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		server.Name,
-		server.ServerType.Name,
-		server.Image.Name,
-		server.Datacenter.Name,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS removal?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	_, err = client.Server.Delete(context.Background(), server)
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof("%s has been deleted from your hetzner cloud account", internal.HetznerCloudName)
-
-	return nil
-}
-
 var serverRemoveCommand = cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
-	ArgsUsage: "[<server>]",
+	ArgsUsage: "<server>",
 	Usage:     "Remove a managed server",
-	Description: `
-This command removes a server from Abra management.
+	Description: `Remove a managed server.
 
-Depending on whether you used a 3rd party provider to create this server ("abra
-server new"), you can also destroy the virtual server as well. Pass
-"--server/-s" to tell Abra to try to delete this VPS.
-
-Otherwise, Abra will remove the internal bookkeeping (~/.abra/servers/...) and
-underlying client connection context. This server will then be lost in time,
-like tears in rain.
+Abra will remove the internal bookkeeping (~/.abra/servers/...) and underlying
+client connection context. This server will then be lost in time, like tears in
+rain.
 `,
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		rmServerFlag,
-		internal.ServerProviderFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudAPITokenFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.ServerNameComplete,
 	Action: func(c *cli.Context) error {
 		serverName := internal.ValidateServer(c)
 
-		warnMsg := `Did not pass -s/--server for actual server deletion, prompting!
-
-Abra doesn't currently know if it helped you create this server with one of the
-3rd party integrations (e.g. Capsul). You have a choice here to actually,
-really and finally destroy this server using those integrations. If you want to
-do this, choose Yes.
-
-If you just want to remove the server config files & context, choose No.
-`
-
-		if !rmServer {
-			logrus.Warn(fmt.Sprintf(warnMsg))
-
-			response := false
-			prompt := &survey.Confirm{
-				Message: "delete actual live server?",
-			}
-			if err := survey.AskOne(prompt, &response); err != nil {
-				logrus.Fatal(err)
-			}
-			if response {
-				logrus.Info("setting -s/--server and attempting to remove actual server")
-				rmServer = true
-			}
-		}
-
-		if rmServer {
-			if err := internal.EnsureServerProvider(); err != nil {
-				logrus.Fatal(err)
-			}
-
-			switch internal.ServerProvider {
-			case "capsul":
-				logrus.Warn("capsul provider does not support automatic removal yet, sorry!")
-			case "hetzner-cloud":
-				if err := rmHetznerCloudVPS(c); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
 		if err := client.DeleteContext(serverName); err != nil {
 			logrus.Fatal(err)
 		}

cli/server/server.go

@@ -9,19 +9,10 @@ var ServerCommand = cli.Command{
 	Name:    "server",
 	Aliases: []string{"s"},
 	Usage:   "Manage servers",
-	Description: `
-These commands support creating, managing and removing servers using 3rd party
-integrations. 
-
-Servers can be created from scratch using the "abra server new" command. If you
-already have a server, you can add it to your configuration using "abra server
-add". Abra can provision servers so that they are ready to deploy Co-op Cloud
-apps, see available flags on "server add" for more.
-`,
 	Subcommands: []cli.Command{
-		serverNewCommand,
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,
+		serverPruneCommand,
 	},
 }

cli/updater/updater.go

@@ -0,0 +1,498 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerclient "github.com/docker/docker/client"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+const SERVER = "localhost"
+
+var majorUpdate bool
+var majorFlag = &cli.BoolFlag{
+	Name:        "major, m",
+	Usage:       "Also check for major updates",
+	Destination: &majorUpdate,
+}
+
+var updateAll bool
+var allFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Usage:       "Update all deployed apps",
+	Destination: &updateAll,
+}
+
+// Notify checks for available upgrades
+var Notify = cli.Command{
+	Name:    "notify",
+	Aliases: []string{"n"},
+	Usage:   "Check for available upgrades",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		majorFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+It reads the deployed app versions and looks for new versions in the recipe
+catalogue. If a new patch/minor version is available, a notification is
+printed. To include major versions use the --major flag.
+`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if recipeName != "" {
+				_, err = getLatestUpgrade(cl, stackName, recipeName)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+// UpgradeApp upgrades apps.
+var UpgradeApp = cli.Command{
+	Name:      "upgrade",
+	Aliases:   []string{"u"},
+	Usage:     "Upgrade apps",
+	ArgsUsage: "<stack-name> <recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.ChaosFlag,
+		majorFlag,
+		allFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Upgrade an app by specifying its stack name and recipe. By passing "--all"
+instead, every deployed app is upgraded. For each apps with enabled auto
+updates the deployed version is compared with the current recipe catalogue
+version. If a new patch/minor version is available, the app is upgraded. To
+include major versions use the "--major" flag. Don't do that, it will probably
+break things. Only apps that are not deployed with "--chaos" are upgraded, to
+update chaos deployments use the "--chaos" flag. Use it with care.
+`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !updateAll {
+			stackName := c.Args().Get(0)
+			recipeName := c.Args().Get(1)
+			err = tryUpgrade(cl, stackName, recipeName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			return nil
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			err = tryUpgrade(cl, stackName, recipeName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+// getLabel reads docker labels from running services in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
+func getLabel(cl *dockerclient.Client, stackName string, label string) (string, error) {
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return "", err
+	}
+
+	for _, service := range services {
+		labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+		if labelValue, ok := service.Spec.Labels[labelKey]; ok {
+			return labelValue, nil
+		}
+	}
+
+	logrus.Debugf("no %s label found for %s", label, stackName)
+
+	return "", nil
+}
+
+// getBoolLabel reads a boolean docker label from running services
+func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool, error) {
+	lableValue, err := getLabel(cl, stackName, label)
+	if err != nil {
+		return false, err
+	}
+
+	if lableValue != "" {
+		value, err := strconv.ParseBool(lableValue)
+		if err != nil {
+			return false, err
+		}
+
+		return value, nil
+	}
+
+	logrus.Debugf("Boolean label %s could not be found for %s, set default to false.", label, stackName)
+
+	return false, nil
+}
+
+// getEnv reads env variables from docker services.
+func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) {
+	envMap := make(map[string]string)
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, service := range services {
+		envList := service.Spec.TaskTemplate.ContainerSpec.Env
+		for _, envString := range envList {
+			splitString := strings.SplitN(envString, "=", 2)
+			if len(splitString) != 2 {
+				logrus.Debugf("can't separate key from value: %s (this variable is probably unset)", envString)
+				continue
+			}
+			k := splitString[0]
+			v := splitString[1]
+			logrus.Debugf("For %s read env %s with value: %s from docker service", stackName, k, v)
+			envMap[k] = v
+		}
+	}
+
+	return envMap, nil
+}
+
+// getLatestUpgrade returns the latest available version for an app respecting
+// the "--major" flag if it is newer than the currently deployed version.
+func getLatestUpgrade(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	deployedVersion, err := getDeployedVersion(cl, stackName, recipeName)
+	if err != nil {
+		return "", err
+	}
+
+	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion)
+	if err != nil {
+		return "", err
+	}
+
+	if len(availableUpgrades) == 0 {
+		logrus.Debugf("no available upgrades for %s", stackName)
+		return "", nil
+	}
+
+	var chosenUpgrade string
+	if len(availableUpgrades) > 0 {
+		chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+		logrus.Infof("%s (%s) can be upgraded from version %s to %s", stackName, recipeName, deployedVersion, chosenUpgrade)
+	}
+
+	return chosenUpgrade, nil
+}
+
+// getDeployedVersion returns the currently deployed version of an app.
+func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	logrus.Debugf("Retrieve deployed version whether %s is already deployed", stackName)
+
+	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+	if err != nil {
+		return "", err
+	}
+
+	if !isDeployed {
+		return "", fmt.Errorf("%s is not deployed?", stackName)
+	}
+
+	if deployedVersion == "unknown" {
+		return "", fmt.Errorf("failed to determine deployed version of %s", stackName)
+	}
+
+	return deployedVersion, nil
+}
+
+// getAvailableUpgrades returns all available versions of an app that are newer
+// than the deployed version. It only includes major upgrades if the "--major"
+// flag is set.
+func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
+	deployedVersion string) ([]string, error) {
+	catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+	if err != nil {
+		return nil, err
+	}
+
+	versions, err := recipe.GetRecipeCatalogueVersions(recipeName, catl)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(versions) == 0 {
+		logrus.Warnf("no published releases for %s in the recipe catalogue?", recipeName)
+		return nil, nil
+	}
+
+	var availableUpgrades []string
+	for _, version := range versions {
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return nil, err
+		}
+
+		versionDelta, err := parsedDeployedVersion.UpgradeDelta(parsedVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		if 0 < versionDelta.UpgradeType() && (versionDelta.UpgradeType() < 4 || majorUpdate) {
+			availableUpgrades = append(availableUpgrades, version)
+		}
+	}
+
+	logrus.Debugf("Available updates for %s: %s", stackName, availableUpgrades)
+
+	return availableUpgrades, nil
+}
+
+// processRecipeRepoVersion clones, pulls, checks out the version and lints the
+// recipe repository.
+func processRecipeRepoVersion(recipeName, version string) error {
+	if err := recipe.EnsureExists(recipeName); err != nil {
+		return err
+	}
+
+	if err := recipe.EnsureUpToDate(recipeName); err != nil {
+		return err
+	}
+
+	if err := recipe.EnsureVersion(recipeName, version); err != nil {
+		return err
+	}
+
+	if r, err := recipe.Get(recipeName, internal.Offline); err != nil {
+		return err
+	} else if err := lint.LintForErrors(r); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// mergeAbraShEnv merges abra.sh env vars into the app env vars.
+func mergeAbraShEnv(recipeName string, env config.AppEnv) error {
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		return err
+	}
+
+	for k, v := range abraShEnv {
+		logrus.Debugf("read v:%s k: %s", v, k)
+		env[k] = v
+	}
+
+	return nil
+}
+
+// createDeployConfig merges and enriches the compose config for the deployment.
+func createDeployConfig(recipeName string, stackName string, env config.AppEnv) (*composetypes.Config, stack.Deploy, error) {
+	env["STACK_NAME"] = stackName
+
+	deployOpts := stack.Deploy{
+		Namespace:    stackName,
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+
+	composeFiles, err := config.GetAppComposeFiles(recipeName, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+
+	deployOpts.Composefiles = composeFiles
+	compose, err := config.GetAppComposeConfig(stackName, deployOpts, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+
+	config.ExposeAllEnv(stackName, compose, env)
+
+	// after the upgrade the deployment won't be in chaos state anymore
+	config.SetChaosLabel(compose, stackName, false)
+	config.SetRecipeLabel(compose, stackName, recipeName)
+	config.SetUpdateLabel(compose, stackName, env)
+
+	return compose, deployOpts, nil
+}
+
+// tryUpgrade performs the upgrade if all the requirements are fulfilled.
+func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string) error {
+	if recipeName == "" {
+		logrus.Debugf("don't update %s due to missing recipe name", stackName)
+		return nil
+	}
+
+	chaos, err := getBoolLabel(cl, stackName, "chaos")
+	if err != nil {
+		return err
+	}
+
+	if chaos && !internal.Chaos {
+		logrus.Debugf("don't update %s due to chaos deployment", stackName)
+		return nil
+	}
+
+	updatesEnabled, err := getBoolLabel(cl, stackName, "autoupdate")
+	if err != nil {
+		return err
+	}
+
+	if !updatesEnabled {
+		logrus.Debugf("don't update %s due to disabled auto updates or missing ENABLE_AUTO_UPDATE env", stackName)
+		return nil
+	}
+
+	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName)
+	if err != nil {
+		return err
+	}
+
+	if upgradeVersion == "" {
+		logrus.Debugf("don't update %s due to no new version", stackName)
+		return nil
+	}
+
+	err = upgrade(cl, stackName, recipeName, upgradeVersion)
+
+	return err
+}
+
+// upgrade performs all necessary steps to upgrade an app.
+func upgrade(cl *dockerclient.Client, stackName, recipeName,
+	upgradeVersion string) error {
+	env, err := getEnv(cl, stackName)
+	if err != nil {
+		return err
+	}
+
+	app := config.App{
+		Name:   stackName,
+		Recipe: recipeName,
+		Server: SERVER,
+		Env:    env,
+	}
+
+	if err = processRecipeRepoVersion(recipeName, upgradeVersion); err != nil {
+		return err
+	}
+
+	if err = mergeAbraShEnv(recipeName, app.Env); err != nil {
+		return err
+	}
+
+	compose, deployOpts, err := createDeployConfig(recipeName, stackName, app.Env)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
+
+	err = stack.RunDeploy(cl, deployOpts, compose, stackName, true)
+
+	return err
+}
+
+func newAbraApp(version, commit string) *cli.App {
+	app := &cli.App{
+		Name: "kadabra",
+		Usage: `The Co-op Cloud auto-updater
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []cli.Command{
+			Notify,
+			UpgradeApp,
+		},
+	}
+
+	app.Before = func(c *cli.Context) error {
+		logrus.Debugf("kadabra version %s, commit %s", version, commit)
+		return nil
+	}
+
+	return app
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := newAbraApp(version, commit)
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cmd/abra/main.go

@@ -5,10 +5,10 @@ import (
 	"coopcloud.tech/abra/cli"
 )
 
-// Version is the current version of Abra
+// Version is the current version of Abra.
 var Version string
 
-// Commit is the current git commit of Abra
+// Commit is the current git commit of Abra.
 var Commit string
 
 func main() {

cmd/kadabra/main.go

@@ -0,0 +1,23 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli/updater"
+)
+
+// Version is the current version of Kadabra.
+var Version string
+
+// Commit is the current git commit of Kadabra.
+var Commit string
+
+func main() {
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	updater.RunApp(Version, Commit)
+}

go.mod

@@ -4,51 +4,46 @@ go 1.16
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
-	github.com/AlecAivazis/survey/v2 v2.3.4
+	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.14+incompatible
-	github.com/docker/distribution v2.8.1+incompatible
-	github.com/docker/docker v20.10.14+incompatible
-	github.com/docker/go-units v0.4.0
-	github.com/go-git/go-git/v5 v5.4.2
-	github.com/hetznercloud/hcloud-go v1.33.1
+	github.com/docker/cli v24.0.6+incompatible
+	github.com/docker/distribution v2.8.2+incompatible
+	github.com/docker/docker v24.0.6+incompatible
+	github.com/docker/go-units v0.5.0
+	github.com/go-git/go-git/v5 v5.9.0
 	github.com/moby/sys/signal v0.7.0
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
+	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.8.6
-	github.com/schultz-is/passgen v1.0.1
-	github.com/sirupsen/logrus v1.8.1
-	gotest.tools/v3 v3.1.0
+	github.com/schollz/progressbar/v3 v3.13.1
+	github.com/sirupsen/logrus v1.9.3
+	gotest.tools/v3 v3.5.1
 )
 
 require (
-	coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
-	github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/buger/goterm v1.0.4
 	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/containers/image v3.0.2+incompatible
 	github.com/containers/storage v1.38.2 // indirect
+	github.com/decentral1se/passgen v1.0.1
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/gliderlabs/ssh v0.3.3
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.1
-	github.com/kevinburke/ssh_config v1.2.0
-	github.com/klauspost/pgzip v1.2.5
-	github.com/libdns/gandi v1.0.2
-	github.com/libdns/libdns v0.2.1
-	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4
+	github.com/klauspost/pgzip v1.2.6
+	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/stretchr/testify v1.8.4
 	github.com/theupdateframework/notary v0.7.0 // indirect
-	github.com/urfave/cli v1.22.5
-	github.com/xanzy/ssh-agent v0.3.1 // indirect
+	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
-	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
+	golang.org/x/sys v0.12.0
 )

go.sum

@@ -34,6 +34,7 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
@@ -46,13 +47,15 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e h1:o5OZInc5b9esiN4hlfjZY6u0r+qB2iSv/11jnMGuR38=
-coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e/go.mod h1:HEQ9pSJRsDKabMxPfYCCzpVpAreLoC4Gh4SkVyOaKvk=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52 h1:cyFFOl0tKe+dVHt8saejG8xoff33eQiHxFCVzRpPUjM=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/AlecAivazis/survey/v2 v2.3.4 h1:pchTU9rsLUSvWEl2Aq9Pv3k0IE2fkqtGxazskAMd9Ng=
-github.com/AlecAivazis/survey/v2 v2.3.4/go.mod h1:hrV6Y/kQCLhIZXGcriDCUBtB3wnN7156gMXJ3+b23xM=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
+github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
+github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
 github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg=
 github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -84,9 +87,10 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX
 github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
@@ -104,9 +108,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
-github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 h1:XcF0cTDJeiuZ5NU8w7WUDge0HRwwNRmxj/GGk6KSA6g=
-github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
+github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg=
+github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -114,15 +117,16 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
-github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
-github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
+github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -160,14 +164,16 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembj
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
@@ -185,6 +191,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e85keuznYcH5rqI438v41pKcBl4ZxQ=
 github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
+github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
+github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -209,7 +217,6 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1
 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
-github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
@@ -240,7 +247,6 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL
 github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
 github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
-github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
@@ -312,11 +318,13 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI=
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -325,6 +333,8 @@ github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7h
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/decentral1se/passgen v1.0.1 h1:j2AxK/kHKxDHWZZfkJj8Wgae9+O+DYEqR5sjKthIYKA=
+github.com/decentral1se/passgen v1.0.1/go.mod h1:530V+lNoPhKtkrX2fIVsIfLhkl47CuiOM7HRgi7C+SU=
 github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -332,16 +342,16 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v20.10.14+incompatible h1:dSBKJOVesDgHo7rbxlYjYsXe7gPzrTT+/cKQgpDAazg=
-github.com/docker/cli v20.10.14+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
+github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w=
-github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
+github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -354,8 +364,9 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6Uezg
 github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
 github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
-github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
@@ -365,10 +376,13 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
-github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -388,7 +402,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -402,18 +415,17 @@ github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYis
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/gliderlabs/ssh v0.3.3 h1:mBQ8NiOgDkINJrZtoizkC3nDNYgSaWtxyem6S2XHBtA=
-github.com/gliderlabs/ssh v0.3.3/go.mod h1:ZSS+CUoKHDrqVakTfTWUlKSr9MtMFkC4UvtQKD7O914=
-github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
-github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
-github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34=
+github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8=
-github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0=
-github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4=
-github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc=
+github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
+github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
+github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
+github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -421,11 +433,16 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -440,6 +457,8 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-sql-driver/mysql v1.3.0 h1:pgwjLi/dvffoP9aabwkT3AKpXQM93QARkjFhDDqC1UE=
 github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@@ -490,8 +509,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -508,8 +528,10 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
 github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -529,6 +551,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -583,8 +606,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
-github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -602,8 +625,6 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn
 github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ=
-github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -647,7 +668,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -658,8 +678,9 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
 github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
+github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -667,19 +688,15 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/libdns/gandi v1.0.2 h1:1Ts8UpI1x5PVKpOjKC7Dn4+EObndz9gm6vdZnloHSKQ=
-github.com/libdns/gandi v1.0.2/go.mod h1:hxpbQKcQFgQrTS5lV4tAgn6QoL6HcCnoBJaW5nOW4Sk=
-github.com/libdns/libdns v0.1.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
-github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
-github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
 github.com/magiconair/properties v1.5.3/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -706,19 +723,21 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
-github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -732,7 +751,6 @@ github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -741,19 +759,22 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
+github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
-github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM=
-github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM=
+github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=
+github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
+github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
+github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
 github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI=
 github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
-github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
-github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -770,6 +791,7 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -783,13 +805,45 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
+github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
+github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
+github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
+github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
+github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
+github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc=
+github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk=
+github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
+github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
+github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
+github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
+github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
+github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
+github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
+github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
+github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
+github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
+github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw=
+github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
+github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
+github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -830,6 +884,9 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -849,14 +906,18 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
 github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -864,8 +925,11 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -876,14 +940,21 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -893,10 +964,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/schollz/progressbar/v3 v3.8.6 h1:QruMUdzZ1TbEP++S1m73OqRJk20ON11m6Wqv4EoGg8c=
-github.com/schollz/progressbar/v3 v3.8.6/go.mod h1:W5IEwbJecncFGBvuEh4A7HT1nZZ6WNIL2i3qbnI0WKY=
-github.com/schultz-is/passgen v1.0.1 h1:wUINzqW1Xmmy3yREHR6YTj+83VlFYjj2DIDMHzIi5TQ=
-github.com/schultz-is/passgen v1.0.1/go.mod h1:NnqzT2aSfvyheNQvBtlLUa0YlPFLDj60Jw2DZVwqiJk=
+github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
+github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
@@ -912,8 +981,12 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
+github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -953,14 +1026,19 @@ github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -981,8 +1059,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
-github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw=
+github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
@@ -992,9 +1070,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
-github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo=
-github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w=
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b h1:6cLsL+2FW6dRAdl5iMtHgRogVCff0QpRi9653YmdcJA=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -1003,6 +1080,7 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1010,6 +1088,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
@@ -1028,7 +1108,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -1038,11 +1117,11 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -1058,12 +1137,17 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 h1:71vQrMauZZhcTVK6KdYM+rklehEEwb3E+ZhaE5jrPrE=
-golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1100,6 +1184,15 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1134,6 +1227,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -1147,14 +1241,32 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1172,6 +1284,8 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1183,6 +1297,12 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1258,6 +1378,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1271,7 +1392,6 @@ golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1286,18 +1406,48 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1306,8 +1456,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1371,6 +1529,7 @@ golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4X
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
@@ -1379,10 +1538,19 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -1491,7 +1659,6 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0=
 google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
@@ -1524,7 +1691,6 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -1539,8 +1705,11 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
@@ -1580,14 +1749,15 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk=
-gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1626,6 +1796,7 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=

pkg/app/app.go

@@ -1,13 +1,9 @@
 package app
 
 import (
-	"context"
-	"fmt"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/upstream/stack"
-	apiclient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
@@ -37,45 +33,6 @@ type deployedServiceSpec struct {
 // VersionSpec represents a deployed app and associated metadata.
 type VersionSpec map[string]deployedServiceSpec
 
-// DeployedVersions lists metadata (e.g. versions) for deployed
-func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App) (VersionSpec, bool, error) {
-	services, err := stack.GetStackServices(ctx, cl, app.StackName())
-	if err != nil {
-		return VersionSpec{}, false, err
-	}
-
-	appSpec := make(VersionSpec)
-	for _, service := range services {
-		serviceName := ParseServiceName(service.Spec.Name)
-		label := fmt.Sprintf("coop-cloud.%s.%s.version", app.StackName(), serviceName)
-		if deployLabel, ok := service.Spec.Labels[label]; ok {
-			version, _ := ParseVersionLabel(deployLabel)
-			appSpec[serviceName] = deployedServiceSpec{Name: serviceName, Version: version}
-		}
-	}
-
-	deployed := len(services) > 0
-
-	if deployed {
-		logrus.Debugf("detected %s as deployed versions of %s", appSpec, app.Name)
-	} else {
-		logrus.Debugf("detected %s as not deployed", app.Name)
-	}
-
-	return appSpec, len(services) > 0, nil
-}
-
-// ParseVersionLabel parses a $VERSION-$DIGEST app service label.
-func ParseVersionLabel(label string) (string, string) {
-	// versions may look like v4.2-abcd or v4.2-alpine-abcd
-	idx := strings.LastIndex(label, "-")
-	version := label[:idx]
-	digest := label[idx+1:]
-	logrus.Debugf("parsed %s as version from %s", version, label)
-	logrus.Debugf("parsed %s as digest from %s", digest, label)
-	return version, digest
-}
-
 // ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label.
 func ParseServiceName(label string) string {
 	idx := strings.LastIndex(label, "_")

pkg/autocomplete/autocomplete.go

@@ -9,7 +9,7 @@ import (
 	"github.com/urfave/cli"
 )
 
-// AppNameComplete copletes app names
+// AppNameComplete copletes app names.
 func AppNameComplete(c *cli.Context) {
 	appNames, err := config.GetAppNames()
 	if err != nil {
@@ -25,9 +25,9 @@ func AppNameComplete(c *cli.Context) {
 	}
 }
 
-// RecipeNameComplete completes recipe names
+// RecipeNameComplete completes recipe names.
 func RecipeNameComplete(c *cli.Context) {
-	catl, err := recipe.ReadRecipeCatalogue()
+	catl, err := recipe.ReadRecipeCatalogue(false)
 	if err != nil {
 		logrus.Warn(err)
 	}
@@ -41,7 +41,23 @@ func RecipeNameComplete(c *cli.Context) {
 	}
 }
 
-// SubcommandComplete completes subcommands.
+// ServerNameComplete completes server names.
+func ServerNameComplete(c *cli.Context) {
+	files, err := config.LoadAppFiles("")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if c.NArg() > 0 {
+		return
+	}
+
+	for _, appFile := range files {
+		fmt.Println(appFile.Server)
+	}
+}
+
+// SubcommandComplete completes sub-commands.
 func SubcommandComplete(c *cli.Context) {
 	if c.NArg() > 0 {
 		return
@@ -52,7 +68,6 @@ func SubcommandComplete(c *cli.Context) {
 		"autocomplete",
 		"catalogue",
 		"recipe",
-		"record",
 		"server",
 		"upgrade",
 	}

pkg/catalogue/catalogue.go

@@ -0,0 +1,127 @@
+package catalogue
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                         true,
+	"abra-apps":                    true,
+	"abra-aur":                     true,
+	"abra-bash":                    true,
+	"abra-capsul":                  true,
+	"abra-gandi":                   true,
+	"abra-hetzner":                 true,
+	"abra-integration-test-recipe": true,
+	"apps":                         true,
+	"aur-abra-git":                 true,
+	"auto-mirror":                  true,
+	"auto-recipes-catalogue-json":  true,
+	"backup-bot":                   true,
+	"backup-bot-two":               true,
+	"beta.coopcloud.tech":          true,
+	"comrade-renovate-bot":         true,
+	"coopcloud.tech":               true,
+	"coturn":                       true,
+	"docker-cp-deploy":             true,
+	"docker-dind-bats-kcov":        true,
+	"docs.coopcloud.tech":          true,
+	"drone-abra":                   true,
+	"example":                      true,
+	"gardening":                    true,
+	"go-abra":                      true,
+	"organising":                   true,
+	"pyabra":                       true,
+	"radicle-seed-node":            true,
+	"recipes-catalogue-json":       true,
+	"recipes-wishlist":             true,
+	"recipes.coopcloud.tech":       true,
+	"stack-ssh-deploy":             true,
+	"swarm-cronjob":                true,
+	"tagcmp":                       true,
+	"traefik-cert-dumper":          true,
+	"tyop":                         true,
+}
+
+// EnsureCatalogue ensures that the catalogue is cloned locally & present.
+func EnsureCatalogue() error {
+	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		logrus.Debugf("cloned catalogue repository to %s", catalogueDir)
+	}
+
+	return nil
+}
+
+// EnsureIsClean makes sure that the catalogue has no unstaged changes.
+func EnsureIsClean() error {
+	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		msg := "%s has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, config.CATALOGUE_DIR)
+	}
+
+	return nil
+}
+
+// EnsureUpToDate ensures that the local catalogue is up to date.
+func EnsureUpToDate() error {
+	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	remotes, err := repo.Remotes()
+	if err != nil {
+		return err
+	}
+
+	if len(remotes) == 0 {
+		msg := "cannot ensure %s is up-to-date, no git remotes configured"
+		logrus.Debugf(msg, config.CATALOGUE_DIR)
+		return nil
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	branch, err := gitPkg.CheckoutDefaultBranch(repo, config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	opts := &git.PullOptions{
+		Force:         true,
+		ReferenceName: branch,
+	}
+
+	if err := worktree.Pull(opts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return err
+		}
+	}
+
+	logrus.Debugf("fetched latest git changes for %s", config.CATALOGUE_DIR)
+
+	return nil
+}

pkg/client/client.go

@@ -2,24 +2,31 @@
 package client
 
 import (
+	"context"
+	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
 
 	contextPkg "coopcloud.tech/abra/pkg/context"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
 	"github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
-// New initiates a new Docker client.
-func New(contextName string) (*client.Client, error) {
+// New initiates a new Docker client. New client connections are validated so
+// that we ensure connections via SSH to the daemon can succeed. It takes into
+// account that you may only want the local client and not communicate via SSH.
+// For this use-case, please pass "default" as the contextName.
+func New(serverName string) (*client.Client, error) {
 	var clientOpts []client.Opt
 
-	if contextName != "default" {
-		context, err := GetContext(contextName)
+	if serverName != "default" {
+		context, err := GetContext(serverName)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("unknown server, run \"abra server add %s\"?", serverName)
 		}
 
 		ctxEndpoint, err := contextPkg.GetContextEndpoint(context)
@@ -33,7 +40,6 @@ func New(contextName string) (*client.Client, error) {
 		}
 
 		httpClient := &http.Client{
-			// No tls, no proxy
 			Transport: &http.Transport{
 				DialContext:     helper.Dialer,
 				IdleConnTimeout: 30 * time.Second,
@@ -59,7 +65,20 @@ func New(contextName string) (*client.Client, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("created client for %s", contextName)
+	logrus.Debugf("created client for %s", serverName)
+
+	info, err := cl.Info(context.Background())
+	if err != nil {
+		return cl, sshPkg.Fatal(serverName, err)
+	}
+
+	if info.Swarm.LocalNodeState == "inactive" {
+		if serverName != "default" {
+			return cl, fmt.Errorf("swarm mode not enabled on %s?", serverName)
+		} else {
+			return cl, errors.New("swarm mode not enabled on local server?")
+		}
+	}
 
 	return cl, nil
 }

pkg/client/registry.go

@@ -3,13 +3,10 @@ package client
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/containers/image/docker"
 	"github.com/containers/image/types"
 	"github.com/docker/distribution/reference"
-	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
 )
 
 // GetRegistryTags retrieves all tags of an image from a container registry.
@@ -29,29 +26,3 @@ func GetRegistryTags(img reference.Named) ([]string, error) {
 
 	return tags, nil
 }
-
-// GetTagDigest retrieves an image digest from a container registry.
-func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
-	target := fmt.Sprintf("//%s", reference.Path(image))
-
-	ref, err := docker.ParseReference(target)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
-	}
-
-	ctx := context.Background()
-	img, err := ref.NewImage(ctx, nil)
-	if err != nil {
-		logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
-		return "", fmt.Errorf("unable to read digest for %s", image)
-	}
-	defer img.Close()
-
-	digest := img.ConfigInfo().Digest.String()
-
-	if digest == "" {
-		return digest, fmt.Errorf("unable to read digest for %s", image)
-	}
-
-	return strings.Split(digest, ":")[1][:7], nil
-}

pkg/client/secret.go

@@ -4,20 +4,14 @@ import (
 	"context"
 
 	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
 )
 
-func StoreSecret(secretName, secretValue, server string) error {
-	cl, err := New(server)
-	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
+func StoreSecret(cl *client.Client, secretName, secretValue, server string) error {
 	ann := swarm.Annotations{Name: secretName}
 	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
 
-	// We don't bother with the secret IDs for now
-	if _, err := cl.SecretCreate(ctx, spec); err != nil {
+	if _, err := cl.SecretCreate(context.Background(), spec); err != nil {
 		return err
 	}
 

pkg/client/volumes.go

@@ -3,17 +3,14 @@ package client
 import (
 	"context"
 
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
 )
 
-func GetVolumes(ctx context.Context, server string, fs filters.Args) ([]*types.Volume, error) {
-	cl, err := New(server)
-	if err != nil {
-		return nil, err
-	}
-
-	volumeListOKBody, err := cl.VolumeList(ctx, fs)
+func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
+	volumeListOptions := volume.ListOptions{fs}
+	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
 		return volumeList, err
@@ -22,7 +19,7 @@ func GetVolumes(ctx context.Context, server string, fs filters.Args) ([]*types.V
 	return volumeList, nil
 }
 
-func GetVolumeNames(volumes []*types.Volume) []string {
+func GetVolumeNames(volumes []*volume.Volume) []string {
 	var volumeNames []string
 
 	for _, vol := range volumes {
@@ -32,12 +29,7 @@ func GetVolumeNames(volumes []*types.Volume) []string {
 	return volumeNames
 }
 
-func RemoveVolumes(ctx context.Context, server string, volumeNames []string, force bool) error {
-	cl, err := New(server)
-	if err != nil {
-		return err
-	}
-
+func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
 	for _, volName := range volumeNames {
 		err := cl.VolumeRemove(ctx, volName, force)
 		if err != nil {

pkg/compose/compose.go

@@ -122,7 +122,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 
 		discovered := false
 		for oldLabel, value := range service.Deploy.Labels {
-			if strings.HasPrefix(oldLabel, "coop-cloud") {
+			if strings.HasPrefix(oldLabel, "coop-cloud") && strings.Contains(oldLabel, "version") {
 				discovered = true
 
 				bytes, err := ioutil.ReadFile(composeFile)

pkg/config/app.go

@@ -2,12 +2,15 @@ package config
 
 import (
 	"fmt"
-	"html/template"
 	"io/ioutil"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 
+	"github.com/schollz/progressbar/v3"
+
+	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
@@ -44,9 +47,9 @@ type App struct {
 	Path   string
 }
 
-// StackName gets what the docker safe stack name is for the app. This should
-// not not shown to the user, use a.Name for that. Give the output of this
-// command to Docker only.
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
@@ -170,7 +173,7 @@ func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	if !exists {
 		recipe, exists = env["TYPE"]
 		if !exists {
-			return App{}, fmt.Errorf("%s is missing the RECIPE env var", name)
+			return App{}, fmt.Errorf("%s is missing the TYPE env var?", name)
 		}
 	}
 
@@ -184,7 +187,7 @@ func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	}, nil
 }
 
-// LoadAppFiles gets all app files for a given set of servers or all servers
+// LoadAppFiles gets all app files for a given set of servers or all servers.
 func LoadAppFiles(servers ...string) (AppFiles, error) {
 	appFiles := make(AppFiles)
 	if len(servers) == 1 {
@@ -193,7 +196,7 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 			var err error
 			servers, err = GetAllFoldersInDirectory(SERVERS_DIR)
 			if err != nil {
-				return nil, err
+				return appFiles, err
 			}
 		}
 	}
@@ -202,10 +205,11 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 
 	for _, server := range servers {
 		serverDir := path.Join(SERVERS_DIR, server)
-		files, err := getAllFilesInDirectory(serverDir)
+		files, err := GetAllFilesInDirectory(serverDir)
 		if err != nil {
-			return nil, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server)
+			return appFiles, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server)
 		}
+
 		for _, file := range files {
 			appName := strings.TrimSuffix(file.Name(), ".env")
 			appFilePath := path.Join(SERVERS_DIR, server, file.Name())
@@ -215,12 +219,13 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 			}
 		}
 	}
+
 	return appFiles, nil
 }
 
-// GetApp loads an apps settings, reading it from file, in preparation to use it
-//
-// ONLY use when ready to use the env file to keep IO down
+// GetApp loads an apps settings, reading it from file, in preparation to use
+// it. It should only be used when ready to use the env file to keep IO
+// operations down.
 func GetApp(apps AppFiles, name AppName) (App, error) {
 	appFile, exists := apps[name]
 	if !exists {
@@ -235,8 +240,9 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
 	return app, nil
 }
 
-// GetApps returns a slice of Apps with their env files read from a given slice of AppFiles
-func GetApps(appFiles AppFiles) ([]App, error) {
+// GetApps returns a slice of Apps with their env files read from a given
+// slice of AppFiles.
+func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) {
 	var apps []App
 
 	for name := range appFiles {
@@ -244,8 +250,15 @@ func GetApps(appFiles AppFiles) ([]App, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		if recipeFilter != "" {
+			if app.Recipe == recipeFilter {
 				apps = append(apps, app)
 			}
+		} else {
+			apps = append(apps, app)
+		}
+	}
 
 	return apps, nil
 }
@@ -291,7 +304,7 @@ func GetAppNames() ([]string, error) {
 		return appNames, err
 	}
 
-	apps, err := GetApps(appFiles)
+	apps, err := GetApps(appFiles, "")
 	if err != nil {
 		return appNames, err
 	}
@@ -303,7 +316,8 @@ func GetAppNames() ([]string, error) {
 	return appNames, nil
 }
 
-// TemplateAppEnvSample copies the example env file for the app into the users env files
+// TemplateAppEnvSample copies the example env file for the app into the users
+// env files.
 func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample")
 	envSample, err := ioutil.ReadFile(envSamplePath)
@@ -312,7 +326,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	}
 
 	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
-	if _, err := os.Stat(appEnvPath); os.IsExist(err) {
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
@@ -321,24 +335,15 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 		return err
 	}
 
-	file, err := os.OpenFile(appEnvPath, os.O_RDWR, 0664)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	tpl, err := template.ParseFiles(appEnvPath)
+	read, err := ioutil.ReadFile(appEnvPath)
 	if err != nil {
 		return err
 	}
 
-	type templateVars struct {
-		Name   string
-		Domain string
-	}
+	newContents := strings.Replace(string(read), recipeName+".example.com", domain, -1)
 
-	tvars := templateVars{Name: recipeName, Domain: domain}
-	if err := tpl.Execute(file, tvars); err != nil {
+	err = ioutil.WriteFile(appEnvPath, []byte(newContents), 0)
+	if err != nil {
 		return err
 	}
 
@@ -347,35 +352,49 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	return nil
 }
 
-// SanitiseAppName makes a app name usable with Docker by replacing illegal characters
+// SanitiseAppName makes a app name usable with Docker by replacing illegal
+// characters.
 func SanitiseAppName(name string) string {
 	return strings.ReplaceAll(name, ".", "_")
 }
 
-// GetAppStatuses queries servers to check the deployment status of given apps
-func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
+// GetAppStatuses queries servers to check the deployment status of given apps.
+func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) {
 	statuses := make(map[string]map[string]string)
 
-	var unique []string
 	servers := make(map[string]struct{})
-	for _, appFile := range appFiles {
-		if _, ok := servers[appFile.Server]; !ok {
-			servers[appFile.Server] = struct{}{}
-			unique = append(unique, appFile.Server)
+	for _, app := range apps {
+		if _, ok := servers[app.Server]; !ok {
+			servers[app.Server] = struct{}{}
 		}
 	}
 
-	bar := formatter.CreateProgressbar(len(servers), "querying remote servers...")
+	var bar *progressbar.ProgressBar
+	if !MachineReadable {
+		bar = formatter.CreateProgressbar(len(servers), "querying remote servers...")
+	}
+
 	ch := make(chan stack.StackStatus, len(servers))
 	for server := range servers {
+		cl, err := client.New(server)
+		if err != nil {
+			return statuses, err
+		}
+
 		go func(s string) {
-			ch <- stack.GetAllDeployedServices(s)
+			ch <- stack.GetAllDeployedServices(cl, s)
+			if !MachineReadable {
 				bar.Add(1)
+			}
 		}(server)
 	}
 
 	for range servers {
 		status := <-ch
+		if status.Err != nil {
+			return statuses, status.Err
+		}
+
 		for _, service := range status.Services {
 			result := make(map[string]string)
 			name := service.Spec.Labels[convert.LabelNamespace]
@@ -384,7 +403,25 @@ func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
 				result["status"] = "deployed"
 			}
 
-			labelKey := fmt.Sprintf("coop-cloud.%s.version", name)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name)
+			chaos, ok := service.Spec.Labels[labelKey]
+			if ok {
+				result["chaos"] = chaos
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name)
+			if chaosVersion, ok := service.Spec.Labels[labelKey]; ok {
+				result["chaosVersion"] = chaosVersion
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.autoupdate", name)
+			if autoUpdate, ok := service.Spec.Labels[labelKey]; ok {
+				result["autoUpdate"] = autoUpdate
+			} else {
+				result["autoUpdate"] = "false"
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.version", name)
 			if version, ok := service.Spec.Labels[labelKey]; ok {
 				result["version"] = version
 			} else {
@@ -438,3 +475,98 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp
 
 	return compose, nil
 }
+
+// ExposeAllEnv exposes all env variables to the app container
+func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("Add the following environment to the app service config of %s:", stackName)
+			for k, v := range appEnv {
+				_, exists := service.Environment[k]
+				if !exists {
+					value := v
+					service.Environment[k] = &value
+					logrus.Debugf("Add Key: %s Value: %s to %s", k, value, stackName)
+				}
+			}
+		}
+	}
+}
+
+// SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container
+// to signal which recipe is connected to the deployed app
+func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName)
+			service.Deploy.Labels[labelKey] = recipe
+		}
+	}
+}
+
+// SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container
+// to signal if the app is deployed in chaos mode
+func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName)
+			service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos)
+		}
+	}
+}
+
+// SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container
+func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName)
+			service.Deploy.Labels[labelKey] = chaosVersion
+		}
+	}
+}
+
+// SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the
+// auto update process for this app. The default if this variable is not set is to disable
+// the auto update process.
+func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"]
+			if !exists {
+				enable_auto_update = "false"
+			}
+			logrus.Debugf("set label 'coop-cloud.%s.autoupdate' to %s for %s", stackName, enable_auto_update, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.autoupdate", stackName)
+			service.Deploy.Labels[labelKey] = enable_auto_update
+		}
+	}
+}
+
+// GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files
+func GetLabel(compose *composetypes.Config, stackName string, label string) string {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+			logrus.Debugf("get label '%s'", labelKey)
+			if labelValue, ok := service.Deploy.Labels[labelKey]; ok {
+				return labelValue
+			}
+		}
+	}
+	logrus.Debugf("no %s label found for %s", label, stackName)
+	return ""
+}
+
+// GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value
+func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
+	var timeout = 50 // Default Timeout
+	var err error = nil
+	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
+		logrus.Debugf("timeout label: %s", timeoutLabel)
+		timeout, err = strconv.Atoi(timeoutLabel)
+	}
+	return timeout, err
+}

pkg/config/env.go

@@ -14,11 +14,21 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
+// getBaseDir retrieves the Abra base directory.
+func getBaseDir() string {
+	home := os.ExpandEnv("$HOME/.abra")
+	if customAbraDir, exists := os.LookupEnv("ABRA_DIR"); exists && customAbraDir != "" {
+		home = customAbraDir
+	}
+	return home
+}
+
+var ABRA_DIR = getBaseDir()
 var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
 var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
 var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
 var BACKUP_DIR = path.Join(ABRA_DIR, "backups")
+var CATALOGUE_DIR = path.Join(ABRA_DIR, "catalogue")
 var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json")
 var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
@@ -65,8 +75,8 @@ func ReadServerNames() ([]string, error) {
 	return serverNames, nil
 }
 
-// getAllFilesInDirectory returns filenames of all files in directory
-func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
+// GetAllFilesInDirectory returns filenames of all files in directory
+func GetAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
 	var realFiles []fs.FileInfo
 
 	files, err := ioutil.ReadDir(directory)

pkg/config/env_test.go

@@ -54,7 +54,7 @@ func TestGetAllFoldersInDirectory(t *testing.T) {
 }
 
 func TestGetAllFilesInDirectory(t *testing.T) {
-	files, err := getAllFilesInDirectory(testFolder)
+	files, err := GetAllFilesInDirectory(testFolder)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/context/context.go

@@ -8,22 +8,19 @@ import (
 	"github.com/docker/cli/cli/context"
 	contextStore "github.com/docker/cli/cli/context/store"
 	cliflags "github.com/docker/cli/cli/flags"
-	"github.com/moby/term"
 )
 
 func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
-	_, _, stderr := term.StdStreams()
-	dockerConfig := dConfig.LoadDefaultConfigFile(stderr)
 	contextDir := dConfig.ContextStoreDir()
 	storeConfig := command.DefaultContextStoreConfig()
 	store := newContextStore(contextDir, storeConfig)
 
-	opts := &cliflags.CommonOptions{Context: "default"}
+	opts := &cliflags.ClientOptions{Context: "default"}
 
 	dockerContextStore := &command.ContextStoreWithDefault{
 		Store: store,
 		Resolver: func() (*command.DefaultContext, error) {
-			return command.ResolveDefaultContext(opts, dockerConfig, storeConfig, stderr)
+			return command.ResolveDefaultContext(opts, storeConfig)
 		},
 	}
 

pkg/dns/common.go

@@ -1,103 +0,0 @@
-package dns
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"os"
-	"time"
-
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/sirupsen/logrus"
-)
-
-// NewToken constructs a new DNS provider token.
-func NewToken(provider, providerTokenEnvVar string) (string, error) {
-	if token, present := os.LookupEnv(providerTokenEnvVar); present {
-		return token, nil
-	}
-
-	logrus.Debugf("no %s in environment, asking via stdin", providerTokenEnvVar)
-
-	var token string
-	prompt := &survey.Input{
-		Message: fmt.Sprintf("%s API token?", provider),
-	}
-	if err := survey.AskOne(prompt, &token); err != nil {
-		return "", err
-	}
-
-	return token, nil
-}
-
-// EnsureIPv4 ensures that an ipv4 address is set for a domain name
-func EnsureIPv4(domainName string) (string, error) {
-	var ipv4 string
-
-	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
-	freifunkDNS := "5.1.66.255:53"
-
-	resolver := &net.Resolver{
-		PreferGo: false,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Millisecond * time.Duration(10000),
-			}
-			return d.DialContext(ctx, "udp", freifunkDNS)
-		},
-	}
-
-	ctx := context.Background()
-	ips, err := resolver.LookupIPAddr(ctx, domainName)
-	if err != nil {
-		return ipv4, err
-	}
-
-	if len(ips) == 0 {
-		return ipv4, fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
-	}
-
-	ipv4 = ips[0].IP.To4().String()
-	logrus.Debugf("%s points to %s (resolver: %s)", domainName, ipv4, freifunkDNS)
-
-	return ipv4, nil
-}
-
-// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address
-func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
-	var ipv4 string
-
-	domainIPv4, err := EnsureIPv4(domainName)
-	if err != nil {
-		return ipv4, err
-	}
-
-	if domainIPv4 == "" {
-		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", domainName)
-	}
-
-	serverIPv4, err := EnsureIPv4(server)
-	if err != nil {
-		return ipv4, err
-	}
-
-	if serverIPv4 == "" {
-		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", server)
-	}
-
-	if domainIPv4 != serverIPv4 {
-		err := "app domain %s (%s) does not appear to resolve to app server %s (%s)?"
-		return ipv4, fmt.Errorf(err, domainName, domainIPv4, server, serverIPv4)
-	}
-
-	return ipv4, nil
-}
-
-// GetTTL parses a ttl string into a duration
-func GetTTL(ttl string) (time.Duration, error) {
-	val, err := time.ParseDuration(ttl)
-	if err != nil {
-		return val, err
-	}
-	return val, nil
-}

pkg/dns/dns.go

@@ -0,0 +1,50 @@
+package dns
+
+import (
+	"fmt"
+	"net"
+)
+
+// EnsureIPv4 ensures that an ipv4 address is set for a domain name
+func EnsureIPv4(domainName string) (string, error) {
+	ipv4, err := net.ResolveIPAddr("ip", domainName)
+	if err != nil {
+		return "", err
+	}
+
+	return ipv4.String(), nil
+}
+
+// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address
+func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
+	if server == "default" || server == "local" {
+		return "", nil
+	}
+
+	var ipv4 string
+
+	domainIPv4, err := EnsureIPv4(domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if domainIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", domainName)
+	}
+
+	serverIPv4, err := EnsureIPv4(server)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if serverIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", server)
+	}
+
+	if domainIPv4 != serverIPv4 {
+		err := "app domain %s (%s) does not appear to resolve to app server %s (%s)?"
+		return ipv4, fmt.Errorf(err, domainName, domainIPv4, server, serverIPv4)
+	}
+
+	return ipv4, nil
+}

pkg/dns/gandi/gandi.go

@@ -1,15 +0,0 @@
-package gandi
-
-import (
-	"coopcloud.tech/abra/pkg/dns"
-	"github.com/libdns/gandi"
-)
-
-// New constructs a new DNs provider.
-func New() (gandi.Provider, error) {
-	token, err := dns.NewToken("Gandi", "GANDI_TOKEN")
-	if err != nil {
-		return gandi.Provider{}, err
-	}
-	return gandi.Provider{APIToken: token}, nil
-}

pkg/formatter/formatter.go

@@ -1,12 +1,14 @@
 package formatter
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/docker/go-units"
-	"github.com/olekukonko/tablewriter"
+	// "github.com/olekukonko/tablewriter"
+	"coopcloud.tech/abra/pkg/jsontable"
 	"github.com/schollz/progressbar/v3"
 	"github.com/sirupsen/logrus"
 )
@@ -32,8 +34,8 @@ func HumanDuration(timestamp int64) string {
 }
 
 // CreateTable prepares a table layout for output.
-func CreateTable(columns []string) *tablewriter.Table {
-	table := tablewriter.NewWriter(os.Stdout)
+func CreateTable(columns []string) *jsontable.JSONTable {
+	table := jsontable.NewJSONTable(os.Stdout)
 	table.SetAutoWrapText(false)
 	table.SetHeader(columns)
 	return table
@@ -69,3 +71,21 @@ func StripTagMeta(image string) string {
 
 	return image
 }
+
+// ByteCountSI presents a human friendly representation of a byte count. See
+// https://yourbasic.org/golang/formatting-byte-size-to-human-readable-format.
+func ByteCountSI(b uint64) string {
+	const unit = 1000
+
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+
+	div, exp := uint64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
+}

pkg/git/branch.go

@@ -1,11 +1,34 @@
 package git
 
 import (
+	"fmt"
+
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
 )
 
-// GetCurrentBranch retrieves the current branch of a repository
+// Check if a branch exists in a repo. Use this and not repository.Branch(),
+// because the latter does not actually check for existing branches. See
+// https://github.com/gogit/gogit/issues/518 for more.
+func HasBranch(repository *git.Repository, name string) bool {
+	var exist bool
+
+	if iter, err := repository.Branches(); err == nil {
+		iterFunc := func(reference *plumbing.Reference) error {
+			if name == reference.Name().Short() {
+				exist = true
+				return nil
+			}
+			return nil
+		}
+		_ = iter.ForEach(iterFunc)
+	}
+
+	return exist
+}
+
+// GetCurrentBranch retrieves the current branch of a repository.
 func GetCurrentBranch(repository *git.Repository) (string, error) {
 	branchRefs, err := repository.Branches()
 	if err != nil {
@@ -33,3 +56,45 @@ func GetCurrentBranch(repository *git.Repository) (string, error) {
 
 	return currentBranchName, nil
 }
+
+// GetDefaultBranch retrieves the default branch of a repository.
+func GetDefaultBranch(repo *git.Repository, repoPath string) (plumbing.ReferenceName, error) {
+	branch := "master"
+
+	if !HasBranch(repo, "master") {
+		if !HasBranch(repo, "main") {
+			return "", fmt.Errorf("failed to select default branch in %s", repoPath)
+		}
+		branch = "main"
+	}
+
+	return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
+}
+
+// CheckoutDefaultBranch checks out the default branch of a repository.
+func CheckoutDefaultBranch(repo *git.Repository, repoPath string) (plumbing.ReferenceName, error) {
+	branch, err := GetDefaultBranch(repo, repoPath)
+	if err != nil {
+		return plumbing.ReferenceName(""), err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return plumbing.ReferenceName(""), err
+	}
+
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: branch,
+	}
+
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out %s in %s", branch, repoPath)
+		return branch, err
+	}
+
+	logrus.Debugf("successfully checked out %v in %s", branch, repoPath)
+
+	return branch, nil
+}

pkg/git/clone.go

@@ -15,22 +15,32 @@ import (
 func Clone(dir, url string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
 		logrus.Debugf("%s does not exist, attempting to git clone from %s", dir, url)
-		_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
+
+		_, err := git.PlainClone(dir, false, &git.CloneOptions{
+			URL:           url,
+			Tags:          git.AllTags,
+			ReferenceName: plumbing.ReferenceName("refs/heads/master"),
+			SingleBranch:  true,
+		})
 		if err != nil {
 			logrus.Debugf("cloning %s default branch failed, attempting from main branch", url)
+
 			_, err := git.PlainClone(dir, false, &git.CloneOptions{
 				URL:           url,
 				Tags:          git.AllTags,
 				ReferenceName: plumbing.ReferenceName("refs/heads/main"),
+				SingleBranch:  true,
 			})
 			if err != nil {
 				if strings.Contains(err.Error(), "authentication required") {
 					name := filepath.Base(dir)
 					return fmt.Errorf("unable to clone %s, does %s exist?", name, url)
 				}
+
 				return err
 			}
 		}
+
 		logrus.Debugf("%s has been git cloned successfully", dir)
 	} else {
 		logrus.Debugf("%s already exists", dir)

pkg/jsontable/jsontable.go

@@ -0,0 +1,209 @@
+package jsontable
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+// A quick-and-dirty proxy/emulator of tablewriter to enable more easy machine readable output
+// - Does not strictly support types, just quoted or unquoted values
+// - Does not support nested values.
+// If a datalabel is set with SetDataLabel(true, "..."), that will be used as the key for teh data of the table,
+// otherwise if the caption is set with SetCaption(true, "..."), the data label will be set to the default of
+// "rows", otherwise the table will output as a JSON list.
+//
+// Proxys all actions through to the tablewriter except addrow and addbatch, which it does at render time
+//
+
+type JSONTable struct {
+	out             io.Writer
+	colsize         int
+	rows            [][]string
+	keys            []string
+	quoted          []bool // hack to do output typing, quoted vs. unquoted
+	hasDataLabel    bool
+	dataLabel       string
+	hasCaption      bool
+	caption         string // the actual caption
+	hasCaptionLabel bool
+	captionLabel    string // the key in the dictionary for the caption
+	tbl             *tablewriter.Table
+}
+
+func writeChar(w io.Writer, c byte) {
+	w.Write([]byte{c})
+
+}
+
+func NewJSONTable(writer io.Writer) *JSONTable {
+	t := &JSONTable{
+		out:             writer,
+		colsize:         0,
+		rows:            [][]string{},
+		keys:            []string{},
+		quoted:          []bool{},
+		hasDataLabel:    false,
+		dataLabel:       "rows",
+		hasCaption:      false,
+		caption:         "",
+		hasCaptionLabel: false,
+		captionLabel:    "caption",
+		tbl:             tablewriter.NewWriter(writer),
+	}
+	return t
+}
+
+func (t *JSONTable) NumLines() int {
+	// JSON only but reflects a shared state.
+	return len(t.rows)
+}
+
+func (t *JSONTable) SetHeader(keys []string) {
+	// Set the keys value which will assign each column to the keys.
+	// Note that we'll ignore values that are beyond the length of the keys list
+	t.colsize = len(keys)
+	t.keys = []string{}
+	for _, k := range keys {
+		t.keys = append(t.keys, k)
+		t.quoted = append(t.quoted, true)
+	}
+	t.tbl.SetHeader(keys)
+}
+
+func (t *JSONTable) SetColumnQuoting(quoting []bool) {
+	// Specify which columns are quoted or unquoted in output
+	// JSON only
+	for i := 0; i < t.colsize; i++ {
+		t.quoted[i] = quoting[i]
+	}
+}
+
+func (t *JSONTable) Append(row []string) {
+	// We'll just append whatever to the rows list. If they fix the keys after appending rows, it'll work as
+	// expected.
+	// We should detect if the row is narrower than the key list tho.
+	// JSON only (but we use the rows later when rendering a regular table)
+	t.rows = append(t.rows, row)
+}
+
+func (t *JSONTable) Render() {
+	// Load the table with rows and render.
+	// Proxy only
+	for _, row := range t.rows {
+		t.tbl.Append(row)
+	}
+
+	t.tbl.Render()
+}
+
+func (t *JSONTable) _JSONRenderInner() {
+	// JSON only
+	// Render the list of dictionaries to the writer.
+	//// inner render loop
+	writeChar(t.out, '[')
+	for rowidx, row := range t.rows {
+		if rowidx != 0 {
+			writeChar(t.out, ',')
+		}
+		writeChar(t.out, '{')
+		for keyidx, key := range t.keys {
+			value := "nil"
+			if keyidx < len(row) {
+				value = row[keyidx]
+			}
+			if keyidx != 0 {
+				writeChar(t.out, ',')
+			}
+			if t.quoted[keyidx] {
+				fmt.Fprintf(t.out, "\"%s\":\"%s\"", key, value)
+			} else {
+				fmt.Fprintf(t.out, "\"%s\":%s", key, value)
+			}
+		}
+		writeChar(t.out, '}')
+	}
+	writeChar(t.out, ']')
+
+}
+
+func (t *JSONTable) JSONRender() {
+	// write JSON table to output
+	// JSON only
+
+	if t.hasDataLabel || t.hasCaption {
+		// dict mode
+		writeChar(t.out, '{')
+
+		if t.hasCaption {
+			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
+
+		}
+		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
+
+	}
+
+	// write list
+	t._JSONRenderInner()
+
+	if t.hasDataLabel || t.hasCaption {
+		// dict mode
+		writeChar(t.out, '}')
+	}
+
+}
+
+func (t *JSONTable) SetCaption(caption bool, captionText ...string) {
+	t.hasCaption = caption
+	if len(captionText) == 1 {
+		t.caption = captionText[0]
+	}
+	t.tbl.SetCaption(caption, captionText...)
+}
+
+func (t *JSONTable) SetCaptionLabel(captionLabel bool, captionLabelText ...string) {
+	// JSON only
+	t.hasCaptionLabel = captionLabel
+	if len(captionLabelText) == 1 {
+		t.captionLabel = captionLabelText[0]
+	}
+}
+
+func (t *JSONTable) SetDataLabel(dataLabel bool, dataLabelText ...string) {
+	// JSON only
+	t.hasDataLabel = dataLabel
+	if len(dataLabelText) == 1 {
+		t.dataLabel = dataLabelText[0]
+	}
+}
+
+func (t *JSONTable) AppendBulk(rows [][]string) {
+	// JSON only but reflects shared state
+	for _, row := range rows {
+		t.Append(row)
+	}
+}
+
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAutoMergeCellsByColumnIndex(cols []int) {
+	// FIXME
+	t.tbl.SetAutoMergeCellsByColumnIndex(cols)
+}
+
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAlignment(align int) {
+	// FIXME
+	t.tbl.SetAlignment(align)
+}
+
+func (t *JSONTable) SetAutoMergeCells(auto bool) {
+	// FIXME
+	t.tbl.SetAutoMergeCells(auto)
+}
+
+// Stub functions
+func (t *JSONTable) SetAutoWrapText(auto bool) {
+	t.tbl.SetAutoWrapText(auto)
+	return
+}

pkg/jsontable/jsontable_test.go

@@ -0,0 +1,83 @@
+package jsontable
+
+import (
+	"testing"
+
+	"bytes"
+	"encoding/json"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+var TestLine = []string{"1", "2"}
+var TestGroup = [][]string{{"1", "2", "3"}, {"a", "teohunteohu", "c", "d"}, {"☺", "☹"}}
+var TestKeys = []string{"key0", "key1", "key2"}
+
+// test creation
+func TestNewTable(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+	if tbl.NumLines() != 0 {
+		t.Fatalf("Something went weird when making table (should have 0 lines)")
+	}
+}
+
+// test adding things
+func TestTableAdd(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+
+	tbl.Append(TestLine)
+	if tbl.NumLines() != 1 {
+		t.Fatalf("Appending a line does not result in a length of 1.")
+	}
+
+	tbl.AppendBulk(TestGroup)
+	numlines := tbl.NumLines()
+	if numlines != (len(TestGroup) + 1) {
+		t.Fatalf("Appending two lines does not result in a length of 4 (length is %d).", numlines)
+	}
+}
+
+// test JSON output is parsable
+func TestJsonParsable(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+
+	tbl.AppendBulk(TestGroup)
+	tbl.SetHeader(TestKeys)
+
+	tbl.JSONRender()
+
+	var son []map[string]interface{}
+
+	err := json.Unmarshal(b.Bytes(), &son)
+
+	if err != nil {
+		t.Fatalf("Did not produce parsable JSON: %s", err.Error())
+	}
+}
+
+// test identical commands to a tablewriter and jsontable produce the same rendered output
+func TestTableWriter(t *testing.T) {
+	var bjson bytes.Buffer
+	var btable bytes.Buffer
+
+	tbl := NewJSONTable(&bjson)
+
+	tbl.AppendBulk(TestGroup)
+	tbl.SetHeader(TestKeys)
+	tbl.Render()
+
+	wtbl := tablewriter.NewWriter(&btable)
+
+	wtbl.AppendBulk(TestGroup)
+	wtbl.SetHeader(TestKeys)
+	wtbl.Render()
+
+	if bytes.Compare(bjson.Bytes(), btable.Bytes()) != 0 {
+		t.Fatalf("JSON table and TableWriter produce non-identical outputs.\n%s\n%s", bjson.Bytes(), btable.Bytes())
+	}
+}
+
+/// FIXME test different output formats when captions etc. are added

pkg/lint/recipe.go

@@ -11,6 +11,8 @@ import (
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/docker/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
 
@@ -19,12 +21,40 @@ var Critical = "critical"
 
 type LintFunction func(recipe.Recipe) (bool, error)
 
+// SkipFunction determines whether the LintFunction is run or not. It should
+// not take the lint rule level into account because some rules are always an
+// error but may depend on some additional context of the recipe configuration.
+// This function aims to cover those additional cases.
+type SkipFunction func(recipe.Recipe) (bool, error)
+
+// LintRule is a linting rule which helps a recipe maintainer avoid common
+// problems in their recipe configurations. We aim to highlight things that
+// might result in critical errors or hours lost in debugging obscure
+// Docker-isms. Humans make the final call on these rules, please raise an
+// issue if you disagree.
 type LintRule struct {
-	Ref          string
-	Level        string
-	Description  string
-	HowToResolve string
-	Function     LintFunction
+	Ref           string       // Reference of the linting rule
+	Level         string       // Level of the warning
+	Description   string       // Description of the issue
+	HowToResolve  string       // Documentation for recipe maintainer
+	Function      LintFunction // Rule implementation
+	SkipCondition SkipFunction // Whether or not to execute the lint rule
+}
+
+// Skip implements the SkipFunction for the lint rule.
+func (l LintRule) Skip(recipe recipe.Recipe) bool {
+	if l.SkipCondition != nil {
+		ok, err := l.SkipCondition(recipe)
+		if err != nil {
+			logrus.Debugf("%s: skip condition: %s", l.Ref, err)
+		}
+		if ok {
+			logrus.Debugf("skipping %s based on skip condition", l.Ref)
+			return true
+		}
+	}
+
+	return false
 }
 
 var LintRules = map[string][]LintRule{
@@ -107,6 +137,7 @@ var LintRules = map[string][]LintRule{
 			Description:   "traefik routing enabled",
 			HowToResolve:  "include \"traefik.enable=true\" deploy label",
 			Function:      LintTraefikEnabled,
+			SkipCondition: LintTraefikEnabledSkipCondition,
 		},
 		{
 			Ref:          "R011",
@@ -122,9 +153,19 @@ var LintRules = map[string][]LintRule{
 			HowToResolve: "vendor config versions in an abra.sh",
 			Function:     LintAbraShVendors,
 		},
+		{
+			Ref:          "R014",
+			Level:        "error",
+			Description:  "only annotated tags used for recipe version",
+			HowToResolve: "replace lightweight tag with annotated tag",
+			Function:     LintValidTags,
+		},
 	},
 }
 
+// LintForErrors lints specifically for errors and not other levels. This is
+// used in code paths such as "app deploy" to avoid nasty surprises but not for
+// the typical linting commands, which do handle other levels.
 func LintForErrors(recipe recipe.Recipe) error {
 	logrus.Debugf("linting for critical errors in %s configs", recipe.Name)
 
@@ -132,7 +173,12 @@ func LintForErrors(recipe recipe.Recipe) error {
 		if level != "error" {
 			continue
 		}
+
 		for _, rule := range LintRules[level] {
+			if rule.Skip(recipe) {
+				continue
+			}
+
 			ok, err := rule.Function(recipe)
 			if err != nil {
 				return err
@@ -175,6 +221,24 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 	return false, nil
 }
 
+// LintTraefikEnabledSkipCondition signals a skip for this linting rule if it
+// confirms that there is no "DOMAIN=..." in the .env.sample configuration of
+// the recipe. This typically means that no domain is required to deploy and
+// therefore no matching traefik deploy label will be present.
+func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	sampleEnv, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
+	}
+
+	if _, ok := sampleEnv["DOMAIN"]; !ok {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func LintTraefikEnabled(recipe recipe.Recipe) (bool, error) {
 	for _, service := range recipe.Config.Services {
 		for label := range service.Deploy.Labels {
@@ -269,7 +333,7 @@ func LintImagePresent(recipe recipe.Recipe) (bool, error) {
 }
 
 func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
-	catl, err := recipePkg.ReadRecipeCatalogue()
+	catl, err := recipePkg.ReadRecipeCatalogue(false)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -336,3 +400,34 @@ func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
 
 	return true, nil
 }
+
+func LintValidTags(recipe recipe.Recipe) (bool, error) {
+	recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return false, fmt.Errorf("unable to open %s: %s", recipeDir, err)
+	}
+
+	iter, err := repo.Tags()
+	if err != nil {
+		logrus.Fatalf("unable to list local tags for %s", recipe.Name)
+	}
+
+	if err := iter.ForEach(func(ref *plumbing.Reference) error {
+		_, err := repo.TagObject(ref.Hash())
+		if err != nil {
+			switch err {
+			case plumbing.ErrObjectNotFound:
+				return fmt.Errorf("invalid lightweight tag detected")
+			default:
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}

pkg/recipe/recipe.go

@@ -7,17 +7,20 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
-	"time"
 
-	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/compose"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/limit"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/abra/pkg/web"
+	"coopcloud.tech/tagcmp"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
@@ -26,7 +29,7 @@ import (
 )
 
 // RecipeCatalogueURL is the only current recipe catalogue available.
-const RecipeCatalogueURL = "https://recipes.coopcloud.tech"
+const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
 
 // ReposMetadataURL is the recipe repository metadata
 const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
@@ -39,7 +42,6 @@ type service = string
 
 // ServiceMeta represents meta info associated with a service.
 type ServiceMeta struct {
-	Digest string `json:"digest"`
 	Image string `json:"image"`
 	Tag   string `json:"tag"`
 }
@@ -203,7 +205,7 @@ func (r Recipe) Tags() ([]string, error) {
 }
 
 // Get retrieves a recipe.
-func Get(recipeName string) (Recipe, error) {
+func Get(recipeName string, offline bool) (Recipe, error) {
 	if err := EnsureExists(recipeName); err != nil {
 		return Recipe{}, err
 	}
@@ -230,11 +232,12 @@ func Get(recipeName string) (Recipe, error) {
 		return Recipe{}, err
 	}
 
-	meta, err := GetRecipeMeta(recipeName)
+	meta, err := GetRecipeMeta(recipeName, offline)
 	if err != nil {
-		if strings.Contains(err.Error(), "does not exist") {
+		switch err.(type) {
+		case RecipeMissingFromCatalogue:
 			meta = RecipeMeta{}
-		} else {
+		default:
 			return Recipe{}, err
 		}
 	}
@@ -269,15 +272,6 @@ func EnsureExists(recipeName string) error {
 func EnsureVersion(recipeName, version string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
-	isClean, err := gitPkg.IsClean(recipeDir)
-	if err != nil {
-		return err
-	}
-
-	if !isClean {
-		return fmt.Errorf("%s has locally unstaged changes", recipeName)
-	}
-
 	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
@@ -304,11 +298,13 @@ func EnsureVersion(recipeName, version string) error {
 		return err
 	}
 
-	logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
+	joinedTags := strings.Join(parsedTags, ", ")
+	if joinedTags != "" {
+		logrus.Debugf("read %s as tags for recipe %s", joinedTags, recipeName)
+	}
 
 	if tagRef.String() == "" {
-		logrus.Warnf("no published release discovered for %s", recipeName)
-		return nil
+		return fmt.Errorf("the local copy of %s doesn't seem to have version %s available?", recipeName, version)
 	}
 
 	worktree, err := repo.Worktree()
@@ -330,25 +326,31 @@ func EnsureVersion(recipeName, version string) error {
 	return nil
 }
 
-// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
-func EnsureLatest(recipeName string) error {
+// EnsureIsClean makes sure that the recipe repository has no unstaged changes.
+func EnsureIsClean(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	isClean, err := gitPkg.IsClean(recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to check git clean status in %s: %s", recipeDir, err)
 	}
 
 	if !isClean {
-		return fmt.Errorf("%s has locally unstaged changes", recipeName)
+		msg := "%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, recipeName, recipeDir)
 	}
 
+	return nil
+}
+
+// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
+func EnsureLatest(recipeName string) error {
+	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+
 	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
 
-	logrus.Debugf("attempting to open git repository in %s", recipeDir)
-
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return err
@@ -359,7 +361,7 @@ func EnsureLatest(recipeName string) error {
 		return err
 	}
 
-	branch, err := GetDefaultBranch(repo, recipeName)
+	branch, err := gitPkg.GetDefaultBranch(repo, recipeDir)
 	if err != nil {
 		return err
 	}
@@ -420,7 +422,7 @@ func GetVersionLabelLocal(recipe Recipe) (string, error) {
 
 	for _, service := range recipe.Config.Services {
 		for label, value := range service.Deploy.Labels {
-			if strings.HasPrefix(label, "coop-cloud") {
+			if strings.HasPrefix(label, "coop-cloud") && strings.Contains(label, "version") {
 				return value, nil
 			}
 		}
@@ -463,6 +465,9 @@ func GetRecipeFeaturesAndCategory(recipeName string) (Features, string, error) {
 		"\n")
 
 	for _, val := range readmeLines {
+		if strings.Contains(val, "**Status**") {
+			feat.Status, _ = strconv.Atoi(strings.TrimSpace(strings.Split(strings.TrimPrefix(val, "* **Status**:"), ",")[0]))
+		}
 		if strings.Contains(val, "**Category**") {
 			category = strings.TrimSpace(
 				strings.TrimPrefix(val, "* **Category**:"),
@@ -566,23 +571,14 @@ func GetStringInBetween(recipeName, str, start, end string) (result string, err
 func EnsureUpToDate(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
-	isClean, err := gitPkg.IsClean(recipeDir)
-	if err != nil {
-		return err
-	}
-
-	if !isClean {
-		return fmt.Errorf("%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding", recipeName, recipeDir)
-	}
-
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to open %s: %s", recipeDir, err)
 	}
 
 	remotes, err := repo.Remotes()
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to read remotes in %s: %s", recipeDir, err)
 	}
 
 	if len(remotes) == 0 {
@@ -592,22 +588,30 @@ func EnsureUpToDate(recipeName string) error {
 
 	worktree, err := repo.Worktree()
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to open git work tree in %s: %s", recipeDir, err)
 	}
 
-	branch, err := CheckoutDefaultBranch(repo, recipeName)
+	branch, err := gitPkg.CheckoutDefaultBranch(repo, recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to check out default branch in %s: %s", recipeDir, err)
+	}
+
+	fetchOpts := &git.FetchOptions{Tags: git.AllTags}
+	if err := repo.Fetch(fetchOpts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return fmt.Errorf("unable to fetch tags in %s: %s", recipeDir, err)
+		}
 	}
 
 	opts := &git.PullOptions{
 		Force:         true,
 		ReferenceName: branch,
+		SingleBranch:  true,
 	}
 
 	if err := worktree.Pull(opts); err != nil {
 		if !strings.Contains(err.Error(), "already up-to-date") {
-			return err
+			return fmt.Errorf("unable to git pull in %s: %s", recipeDir, err)
 		}
 	}
 
@@ -616,110 +620,18 @@ func EnsureUpToDate(recipeName string) error {
 	return nil
 }
 
-func GetDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
-	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-
-	meta, _ := GetRecipeMeta(recipeName)
-	if meta.DefaultBranch != "" {
-		return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", meta.DefaultBranch)), nil
-	}
-
-	branch := "master"
-	if _, err := repo.Branch("master"); err != nil {
-		if _, err := repo.Branch("main"); err != nil {
-			return "", fmt.Errorf("failed to select default branch in %s", recipeDir)
-		}
-		branch = "main"
-	}
-
-	return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
-}
-
-func CheckoutDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
-	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-
-	branch, err := GetDefaultBranch(repo, recipeName)
-	if err != nil {
-		return plumbing.ReferenceName(""), err
-	}
-
-	worktree, err := repo.Worktree()
-	if err != nil {
-		return plumbing.ReferenceName(""), err
-	}
-
-	checkOutOpts := &git.CheckoutOptions{
-		Create: false,
-		Force:  true,
-		Branch: branch,
-	}
-
-	if err := worktree.Checkout(checkOutOpts); err != nil {
-		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-		logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
-		return branch, err
-	}
-
-	logrus.Debugf("successfully checked out %v in %s", branch, recipeDir)
-
-	return branch, nil
-}
-
-// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
-// is up to date.
-func recipeCatalogueFSIsLatest() (bool, error) {
-	httpClient := web.NewHTTPRetryClient()
-	res, err := httpClient.Head(RecipeCatalogueURL)
-	if err != nil {
-		return false, err
-	}
-
-	lastModified := res.Header["Last-Modified"][0]
-	parsed, err := time.Parse(time.RFC1123, lastModified)
-	if err != nil {
-		return false, err
-	}
-
-	info, err := os.Stat(config.RECIPES_JSON)
-	if err != nil {
-		if os.IsNotExist(err) {
-			logrus.Debugf("no recipe catalogue found in file system cache")
-			return false, nil
-		}
-		return false, err
-	}
-
-	localModifiedTime := info.ModTime().Unix()
-	remoteModifiedTime := parsed.Unix()
-
-	if localModifiedTime < remoteModifiedTime {
-		logrus.Debug("file system cached recipe catalogue is out-of-date")
-		return false, nil
-	}
-
-	logrus.Debug("file system cached recipe catalogue is up-to-date")
-
-	return true, nil
-}
-
 // ReadRecipeCatalogue reads the recipe catalogue.
-func ReadRecipeCatalogue() (RecipeCatalogue, error) {
+func ReadRecipeCatalogue(offline bool) (RecipeCatalogue, error) {
 	recipes := make(RecipeCatalogue)
 
-	if err := EnsureCatalogue(); err != nil {
+	if err := catalogue.EnsureCatalogue(); err != nil {
 		return nil, err
 	}
 
-	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
-	if err != nil {
+	if !offline {
+		if err := catalogue.EnsureUpToDate(); err != nil {
 			return nil, err
 		}
-
-	if !recipeFSIsLatest {
-		if err := readRecipeCatalogueWeb(&recipes); err != nil {
-			return nil, err
-		}
-		return recipes, nil
 	}
 
 	if err := readRecipeCatalogueFS(&recipes); err != nil {
@@ -745,31 +657,11 @@ func readRecipeCatalogueFS(target interface{}) error {
 	return nil
 }
 
-// readRecipeCatalogueWeb reads the catalogue from the web.
-func readRecipeCatalogueWeb(target interface{}) error {
-	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
-		return err
-	}
-
-	recipesJSON, err := json.MarshalIndent(target, "", "    ")
-	if err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
-		return err
-	}
-
-	logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL)
-
-	return nil
-}
-
 // VersionsOfService lists the version of a service.
-func VersionsOfService(recipe, serviceName string) ([]string, error) {
+func VersionsOfService(recipe, serviceName string, offline bool) ([]string, error) {
 	var versions []string
 
-	catalogue, err := ReadRecipeCatalogue()
+	catalogue, err := ReadRecipeCatalogue(offline)
 	if err != nil {
 		return nil, err
 	}
@@ -794,20 +686,26 @@ func VersionsOfService(recipe, serviceName string) ([]string, error) {
 	return versions, nil
 }
 
+// RecipeMissingFromCatalogue signifies a recipe is not present in the catalogue.
+type RecipeMissingFromCatalogue struct{ err string }
+
+// Error outputs the error message.
+func (r RecipeMissingFromCatalogue) Error() string {
+	return r.err
+}
+
 // GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
-func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
-	catl, err := ReadRecipeCatalogue()
+func GetRecipeMeta(recipeName string, offline bool) (RecipeMeta, error) {
+	catl, err := ReadRecipeCatalogue(offline)
 	if err != nil {
 		return RecipeMeta{}, err
 	}
 
 	recipeMeta, ok := catl[recipeName]
 	if !ok {
-		return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
+		return RecipeMeta{}, RecipeMissingFromCatalogue{
+			err: fmt.Sprintf("recipe %s does not exist?", recipeName),
 		}
-
-	if err := EnsureExists(recipeName); err != nil {
-		return RecipeMeta{}, err
 	}
 
 	logrus.Debugf("recipe metadata retrieved for %s", recipeName)
@@ -928,9 +826,8 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 }
 
 // GetRecipeVersions retrieves all recipe versions.
-func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
+func GetRecipeVersions(recipeName string, offline bool) (RecipeVersions, error) {
 	versions := RecipeVersions{}
-
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	logrus.Debugf("attempting to open git repository in %s", recipeDir)
@@ -967,17 +864,11 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 
 		logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
 
-		recipe, err := Get(recipeName)
+		recipe, err := Get(recipeName, offline)
 		if err != nil {
 			return err
 		}
 
-		cl, err := client.New("default") // only required for container registry calls
-		if err != nil {
-			return err
-		}
-
-		queryCache := make(map[reference.Named]string)
 		versionMeta := make(map[string]ServiceMeta)
 		for _, service := range recipe.Config.Services {
 
@@ -999,28 +890,9 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 				continue
 			}
 
-			var exists bool
-			var digest string
-			if digest, exists = queryCache[img]; !exists {
-				logrus.Debugf("cache miss: querying for image: %s, tag: %s", path, tag)
-
-				var err error
-				digest, err = client.GetTagDigest(cl, img)
-				if err != nil {
-					logrus.Warn(err)
-					digest = "unknown"
-				}
-
-				queryCache[img] = digest
-				logrus.Debugf("cached insert: %s, tag: %s, digest: %s", path, tag, digest)
-			} else {
-				logrus.Debugf("cache hit: image: %s, tag: %s, digest: %s", path, tag, digest)
-			}
-
 			versionMeta[service.Name] = ServiceMeta{
-				Digest: digest,
 				Image: path,
-				Tag:    img.(reference.NamedTagged).Tag(),
+				Tag:   tag,
 			}
 		}
 
@@ -1031,16 +903,56 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 		return versions, err
 	}
 
-	_, err = CheckoutDefaultBranch(repo, recipeName)
+	_, err = gitPkg.CheckoutDefaultBranch(repo, recipeDir)
 	if err != nil {
 		return versions, err
 	}
+	sortRecipeVersions(versions)
 
 	logrus.Debugf("collected %s for %s", versions, recipeName)
 
 	return versions, nil
 }
 
+// sortRecipeVersions sorts the recipe semver versions
+func sortRecipeVersions(versions RecipeVersions) {
+	sort.Slice(versions, func(i, j int) bool {
+		version1, err := tagcmp.Parse(getVersionString(versions[i]))
+		if err != nil {
+			panic(err)
+		}
+		version2, err := tagcmp.Parse(getVersionString(versions[j]))
+		if err != nil {
+			panic(err)
+		}
+		return version1.IsLessThan(version2)
+	})
+}
+
+// getVersionString returns the version string from RecipeVersions
+func getVersionString(versionMap map[string]map[string]ServiceMeta) string {
+	// Assuming there's only one key in versionMap
+	for k := range versionMap {
+		return k
+	}
+	return ""
+}
+
+// sortVersionStrings sorts a list of semver version strings
+func sortVersionStrings(versions []string) {
+	sort.Slice(versions, func(i, j int) bool {
+		version1, err := tagcmp.Parse(versions[i])
+		if err != nil {
+			panic(err)
+		}
+		version2, err := tagcmp.Parse(versions[j])
+		if err != nil {
+			panic(err)
+		}
+		return version1.IsLessThan(version2)
+	})
+}
+
 // GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
 func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
 	var versions []string
@@ -1053,19 +965,53 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri
 		}
 	}
 
+	sortVersionStrings(versions)
+
 	return versions, nil
 }
 
-// EnsureCatalogue ensures that the catalogue is cloned locally & present.
-func EnsureCatalogue() error {
-	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
-	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
-		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
-		if err := gitPkg.Clone(catalogueDir, url); err != nil {
-			return err
+// UpdateRepositories clones and updates all recipe repositories locally.
+func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
+	var barLength int
+	if recipeName != "" {
+		barLength = 1
+	} else {
+		barLength = len(repos)
 	}
 
-		logrus.Debugf("cloned catalogue repository to %s", catalogueDir)
+	cloneLimiter := limit.New(10)
+
+	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
+	ch := make(chan string, barLength)
+	for _, repoMeta := range repos {
+		go func(rm RepoMeta) {
+			cloneLimiter.Begin()
+			defer cloneLimiter.End()
+
+			if recipeName != "" && recipeName != rm.Name {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
+			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
+
+			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
+
+			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
+				logrus.Fatal(err)
+			}
+
+			ch <- rm.Name
+			retrieveBar.Add(1)
+		}(repoMeta)
+	}
+
+	for range repos {
+		<-ch // wait for everything
 	}
 
 	return nil

pkg/recipe/recipe_test.go

@@ -0,0 +1,31 @@
+package recipe
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetVersionLabelLocalDoesNotUseTimeoutLabel(t *testing.T) {
+	offline := true
+	recipe, err := Get("traefik", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 1; i < 1000; i++ {
+		label, err := GetVersionLabelLocal(recipe)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// NOTE(d1): this is potentially quite a brittle unit test as it needs to
+		// hardcode the default timeout label to ensure that the label parser never
+		// returns it. hopefully this won't fail too often! if you're here because
+		// of a failure, just update the `defaultTimeoutLabel` value & permalink
+		// below
+		// https://git.coopcloud.tech/coop-cloud/traefik/src/commit/ac3a47fe8ca3ef92db84f64cfedfbb348000faee/.env.sample#L2
+		defaultTimeoutLabel := "300"
+		assert.NotEqual(t, label, defaultTimeoutLabel)
+	}
+}

pkg/secret/secret.go

@@ -12,7 +12,8 @@ import (
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"github.com/schultz-is/passgen"
+	"github.com/decentral1se/passgen"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
@@ -117,7 +118,7 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
+func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
 	secrets := make(map[string]string)
 
 	var mutex sync.Mutex
@@ -146,7 +147,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					return
 				}
 
-				if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
 						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil
@@ -166,7 +167,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					return
 				}
 
-				if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
 						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil

pkg/ssh/ssh.go

@@ -1,37 +1,13 @@
 package ssh
 
 import (
-	"bufio"
-	"bytes"
-	"crypto/sha256"
-	"encoding/base64"
 	"fmt"
-	"io"
-	"net"
-	"os"
-	"os/user"
-	"path/filepath"
+	"os/exec"
 	"strings"
-	"sync"
-	"time"
 
-	"coopcloud.tech/abra/pkg/context"
-	"github.com/AlecAivazis/survey/v2"
-	dockerSSHPkg "github.com/docker/cli/cli/connhelper/ssh"
-	sshPkg "github.com/gliderlabs/ssh"
-	"github.com/kevinburke/ssh_config"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/agent"
-	"golang.org/x/crypto/ssh/knownhosts"
 )
 
-var KnownHostsPath = filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts")
-
-type Client struct {
-	SSHClient *ssh.Client
-}
-
 // HostConfig is a SSH host config.
 type HostConfig struct {
 	Host         string
@@ -40,509 +16,70 @@ type HostConfig struct {
 	User         string
 }
 
-// Exec cmd on the remote host and return stderr and stdout
-func (c *Client) Exec(cmd string) ([]byte, error) {
-	session, err := c.SSHClient.NewSession()
-	if err != nil {
-		return nil, err
-	}
-	defer session.Close()
-
-	return session.CombinedOutput(cmd)
-}
-
-// Close the underlying SSH connection
-func (c *Client) Close() error {
-	return c.SSHClient.Close()
-}
-
-// New creates a new SSH client connection.
-func New(domainName, sshAuth, username, port string) (*Client, error) {
-	var client *Client
-
-	ctxConnDetails, err := GetContextConnDetails(domainName)
-	if err != nil {
-		return client, nil
-	}
-
-	if sshAuth == "identity-file" {
-		var err error
-		client, err = connectWithAgentTimeout(
-			ctxConnDetails.Host,
-			ctxConnDetails.User,
-			ctxConnDetails.Port,
-			5*time.Second,
+// String presents a human friendly output for the HostConfig.
+func (h HostConfig) String() string {
+	return fmt.Sprintf(
+		"{host: %s, username: %s, port: %s, identityfile: %s}",
+		h.Host,
+		h.User,
+		h.Port,
+		h.IdentityFile,
 	)
-		if err != nil {
-			return client, err
-		}
-	} else {
-		password := ""
-		prompt := &survey.Password{
-			Message: "SSH password?",
-		}
-		if err := survey.AskOne(prompt, &password); err != nil {
-			return client, err
-		}
-
-		var err error
-		client, err = connectWithPasswordTimeout(
-			ctxConnDetails.Host,
-			ctxConnDetails.User,
-			ctxConnDetails.Port,
-			password,
-			5*time.Second,
-		)
-		if err != nil {
-			return client, err
-		}
-	}
-
-	return client, nil
-}
-
-// sudoWriter supports sudo command handling
-type sudoWriter struct {
-	b     bytes.Buffer
-	pw    string
-	stdin io.Writer
-	m     sync.Mutex
-}
-
-// Write satisfies the write interface for sudoWriter
-func (w *sudoWriter) Write(p []byte) (int, error) {
-	if strings.Contains(string(p), "sudo_password") {
-		w.stdin.Write([]byte(w.pw + "\n"))
-		w.pw = ""
-		return len(p), nil
-	}
-
-	w.m.Lock()
-	defer w.m.Unlock()
-
-	return w.b.Write(p)
-}
-
-// RunSudoCmd runs SSH commands and streams output
-func RunSudoCmd(cmd, passwd string, cl *Client) error {
-	session, err := cl.SSHClient.NewSession()
-	if err != nil {
-		return err
-	}
-	defer session.Close()
-
-	sudoCmd := fmt.Sprintf("SSH_ASKPASS=/usr/bin/ssh-askpass; sudo -p sudo_password -S %s", cmd)
-
-	w := &sudoWriter{pw: passwd}
-	w.stdin, err = session.StdinPipe()
-	if err != nil {
-		return err
-	}
-
-	session.Stdout = w
-	session.Stderr = w
-
-	modes := ssh.TerminalModes{
-		ssh.ECHO:          0,
-		ssh.TTY_OP_ISPEED: 14400,
-		ssh.TTY_OP_OSPEED: 14400,
-	}
-
-	err = session.RequestPty("xterm", 80, 40, modes)
-	if err != nil {
-		return err
-	}
-
-	if err := session.Run(sudoCmd); err != nil {
-		return fmt.Errorf("%s", string(w.b.Bytes()))
-	}
-
-	return nil
-}
-
-// EnsureKnowHostsFiles ensures that ~/.ssh/known_hosts is created
-func EnsureKnowHostsFiles() error {
-	if _, err := os.Stat(KnownHostsPath); os.IsNotExist(err) {
-		logrus.Debugf("missing %s, creating now", KnownHostsPath)
-		file, err := os.OpenFile(KnownHostsPath, os.O_CREATE, 0600)
-		if err != nil {
-			return err
-		}
-		file.Close()
-	}
-
-	return nil
-}
-
-// GetHostKey checks if a host key is registered in the ~/.ssh/known_hosts file
-func GetHostKey(hostname string) (bool, sshPkg.PublicKey, error) {
-	var hostKey sshPkg.PublicKey
-
-	ctxConnDetails, err := GetContextConnDetails(hostname)
-	if err != nil {
-		return false, hostKey, err
-	}
-
-	if err := EnsureKnowHostsFiles(); err != nil {
-		return false, hostKey, err
-	}
-
-	file, err := os.Open(KnownHostsPath)
-	if err != nil {
-		return false, hostKey, err
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		fields := strings.Split(scanner.Text(), " ")
-		if len(fields) != 3 {
-			continue
-		}
-
-		hostnameAndPort := fmt.Sprintf("%s:%s", ctxConnDetails.Host, ctxConnDetails.Port)
-		hashed := knownhosts.Normalize(hostnameAndPort)
-
-		if strings.Contains(fields[0], hashed) {
-			var err error
-			hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes())
-			if err != nil {
-				return false, hostKey, fmt.Errorf("error parsing server SSH host key %q: %v", fields[2], err)
-			}
-			break
-		}
-	}
-
-	if hostKey != nil {
-		logrus.Debugf("server SSH host key present in ~/.ssh/known_hosts for %s", hostname)
-		return true, hostKey, nil
-	}
-
-	return false, hostKey, nil
-}
-
-// InsertHostKey adds a new host key to the ~/.ssh/known_hosts file
-func InsertHostKey(hostname string, remote net.Addr, pubKey ssh.PublicKey) error {
-	file, err := os.OpenFile(KnownHostsPath, os.O_APPEND|os.O_WRONLY, 0600)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	hashedHostname := knownhosts.Normalize(hostname)
-	lineHostname := knownhosts.Line([]string{hashedHostname}, pubKey)
-	_, err = file.WriteString(fmt.Sprintf("%s\n", lineHostname))
-	if err != nil {
-		return err
-	}
-
-	hashedRemote := knownhosts.Normalize(remote.String())
-	lineRemote := knownhosts.Line([]string{hashedRemote}, pubKey)
-	_, err = file.WriteString(fmt.Sprintf("%s\n", lineRemote))
-	if err != nil {
-		return err
-	}
-
-	logrus.Debugf("SSH host key generated: %s", lineHostname)
-	logrus.Debugf("SSH host key generated: %s", lineRemote)
-
-	return nil
-}
-
-// HostKeyAddCallback ensures server ssh host keys are handled
-func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.PublicKey) error {
-	exists, _, err := GetHostKey(hostnameAndPort)
-	if err != nil {
-		return err
-	}
-
-	if exists {
-		hostname := strings.Split(hostnameAndPort, ":")[0]
-		logrus.Debugf("server SSH host key found for %s", hostname)
-		return nil
-	}
-
-	if !exists {
-		hostname := strings.Split(hostnameAndPort, ":")[0]
-		parsedPubKey := FingerprintSHA256(pubKey)
-
-		fmt.Printf(fmt.Sprintf(`
-You are attempting to make an SSH connection to a server but there is no entry
-in your ~/.ssh/known_hosts file which confirms that you have already validated
-that this is indeed the server you want to connect to. Please take a moment to
-validate the following SSH host key, it is important.
-
-    Host:        %s
-    Fingerprint: %s
-
-If this is confusing to you, you can read the article below and learn how to
-validate this fingerprint safely. Thanks to the comrades at cyberia.club for
-writing this extensive guide <3
-
-    https://sequentialread.com/understanding-the-secure-shell-protocol-ssh/
-
-`, hostname, parsedPubKey))
-
-		response := false
-		prompt := &survey.Confirm{
-			Message: "are you sure you trust this host key?",
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return err
-		}
-
-		if !response {
-			logrus.Fatal("exiting as requested")
-		}
-
-		logrus.Debugf("attempting to insert server SSH host key for %s, %s", hostnameAndPort, remote)
-
-		if err := InsertHostKey(hostnameAndPort, remote, pubKey); err != nil {
-			return err
-		}
-
-		logrus.Infof("successfully added server SSH host key for %s", hostname)
-	}
-
-	return nil
-}
-
-// connect makes the SSH connection
-func connect(username, host, port string, authMethod ssh.AuthMethod, timeout time.Duration) (*Client, error) {
-	config := &ssh.ClientConfig{
-		User:            username,
-		Auth:            []ssh.AuthMethod{authMethod},
-		HostKeyCallback: HostKeyAddCallback, // the main reason why we fork
-	}
-
-	hostnameAndPort := fmt.Sprintf("%s:%s", host, port)
-
-	logrus.Debugf("tcp dialing %s", hostnameAndPort)
-
-	var conn net.Conn
-	var err error
-	conn, err = net.DialTimeout("tcp", hostnameAndPort, timeout)
-	if err != nil {
-		logrus.Debugf("tcp dialing %s failed, trying via ~/.ssh/config", hostnameAndPort)
-		hostConfig, err := GetHostConfig(host, username, port, true)
-		if err != nil {
-			return nil, err
-		}
-		conn, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostConfig.Host, hostConfig.Port), timeout)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	sshConn, chans, reqs, err := ssh.NewClientConn(conn, hostnameAndPort, config)
-	if err != nil {
-		return nil, err
-	}
-
-	client := ssh.NewClient(sshConn, chans, reqs)
-	c := &Client{SSHClient: client}
-
-	return c, nil
-}
-
-func connectWithAgentTimeout(host, username, port string, timeout time.Duration) (*Client, error) {
-	logrus.Debugf("using ssh-agent to make an SSH connection for %s", host)
-
-	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
-	if err != nil {
-		return nil, err
-	}
-
-	agentCl := agent.NewClient(sshAgent)
-	authMethod := ssh.PublicKeysCallback(agentCl.Signers)
-
-	loadedKeys, err := agentCl.List()
-	if err != nil {
-		return nil, err
-	}
-
-	var convertedKeys []string
-	for _, key := range loadedKeys {
-		convertedKeys = append(convertedKeys, key.String())
-	}
-
-	if len(convertedKeys) > 0 {
-		logrus.Debugf("ssh-agent has these keys loaded: %s", strings.Join(convertedKeys, ","))
-	} else {
-		logrus.Debug("ssh-agent has no keys loaded")
-	}
-
-	return connect(username, host, port, authMethod, timeout)
-}
-
-func connectWithPasswordTimeout(host, username, port, pass string, timeout time.Duration) (*Client, error) {
-	authMethod := ssh.Password(pass)
-
-	return connect(username, host, port, authMethod, timeout)
-}
-
-// EnsureHostKey ensures that a host key trusted and added to the ~/.ssh/known_hosts file
-func EnsureHostKey(hostname string) error {
-	if hostname == "default" || hostname == "local" {
-		logrus.Debugf("not checking server SSH host key against local/default target")
-		return nil
-	}
-
-	exists, _, err := GetHostKey(hostname)
-	if err != nil {
-		return err
-	}
-	if exists {
-		return nil
-	}
-
-	ctxConnDetails, err := GetContextConnDetails(hostname)
-	if err != nil {
-		return err
-	}
-
-	_, err = connectWithAgentTimeout(
-		ctxConnDetails.Host,
-		ctxConnDetails.User,
-		ctxConnDetails.Port,
-		5*time.Second,
-	)
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// FingerprintSHA256 generates the SHA256 fingerprint for a server SSH host key
-func FingerprintSHA256(key ssh.PublicKey) string {
-	hash := sha256.Sum256(key.Marshal())
-	b64hash := base64.StdEncoding.EncodeToString(hash[:])
-	trimmed := strings.TrimRight(b64hash, "=")
-	return fmt.Sprintf("SHA256:%s", trimmed)
-}
-
-// GetContextConnDetails retrieves SSH connection details from a docker context endpoint
-func GetContextConnDetails(serverName string) (*dockerSSHPkg.Spec, error) {
-	dockerContextStore := context.NewDefaultDockerContextStore()
-	contexts, err := dockerContextStore.Store.List()
-	if err != nil {
-		return &dockerSSHPkg.Spec{}, err
-	}
-
-	if strings.Contains(serverName, ":") {
-		serverName = strings.Split(serverName, ":")[0]
-	}
-
-	for _, ctx := range contexts {
-		endpoint, err := context.GetContextEndpoint(ctx)
-		if err != nil && strings.Contains(err.Error(), "does not exist") {
-			// No local context found, we can continue safely
-			continue
-		}
-		if ctx.Name == serverName {
-			ctxConnDetails, err := dockerSSHPkg.ParseURL(endpoint)
-			if err != nil {
-				return &dockerSSHPkg.Spec{}, err
-			}
-			logrus.Debugf("found context connection details %v for %s", ctxConnDetails, serverName)
-			return ctxConnDetails, nil
-		}
-	}
-
-	hostConfig, err := GetHostConfig(serverName, "", "", false)
-	if err != nil {
-		return &dockerSSHPkg.Spec{}, err
-	}
-
-	logrus.Debugf("couldn't find a docker context matching %s", serverName)
-	logrus.Debugf("searching ~/.ssh/config for a Host entry for %s", serverName)
-
-	connDetails := &dockerSSHPkg.Spec{
-		Host: hostConfig.Host,
-		User: hostConfig.User,
-		Port: hostConfig.Port,
-	}
-
-	logrus.Debugf("using %v from ~/.ssh/config for connection details", connDetails)
-
-	return connDetails, nil
 }
 
-// GetHostConfig retrieves a ~/.ssh/config config for a host.
-func GetHostConfig(hostname, username, port string, override bool) (HostConfig, error) {
+// GetHostConfig retrieves a ~/.ssh/config config for a host using /usr/bin/ssh
+// directly. We therefore maintain consistent interop with this standard
+// tooling. This is useful because SSH confuses a lot of people and having to
+// learn how two tools (`ssh` and `abra`) handle SSH connection details instead
+// of one (just `ssh`) is Not Cool. Here's to less bug reports on this topic!
+func GetHostConfig(hostname string) (HostConfig, error) {
 	var hostConfig HostConfig
 
-	if hostname == "" || override {
-		if sshHost := ssh_config.Get(hostname, "Hostname"); sshHost != "" {
-			hostname = sshHost
-		}
-	}
-
-	if username == "" || override {
-		if sshUser := ssh_config.Get(hostname, "User"); sshUser != "" {
-			username = sshUser
-		} else {
-			systemUser, err := user.Current()
+	out, err := exec.Command("ssh", "-G", hostname).Output()
 	if err != nil {
 		return hostConfig, err
 	}
-			username = systemUser.Username
-		}
-	}
 
-	if port == "" || override {
-		if sshPort := ssh_config.Get(hostname, "Port"); sshPort != "" {
-			// skip override probably correct port with dummy default value from
-			// ssh_config which is 22. only when the original port number is empty
-			// should we try this default. this might not cover all cases
-			// unfortunately.
-			if port != "" && sshPort != "22" {
-				port = sshPort
+	for _, line := range strings.Split(string(out), "\n") {
+		entries := strings.Split(line, " ")
+		for idx, entry := range entries {
+			if entry == "hostname" {
+				hostConfig.Host = entries[idx+1]
+			}
+			if entry == "user" {
+				hostConfig.User = entries[idx+1]
+			}
+			if entry == "port" {
+				hostConfig.Port = entries[idx+1]
+			}
+			if entry == "identityfile" {
+				if hostConfig.IdentityFile == "" {
+					hostConfig.IdentityFile = entries[idx+1]
+				}
 			}
 		}
 	}
 
-	if idf := ssh_config.Get(hostname, "IdentityFile"); idf != "" && idf != "~/.ssh/identity" {
-		var err error
-		idf, err = identityFileAbsPath(idf)
-		if err != nil {
-			return hostConfig, err
-		}
-		hostConfig.IdentityFile = idf
-	} else {
-		hostConfig.IdentityFile = ""
-	}
-
-	hostConfig.Host = hostname
-	hostConfig.Port = port
-	hostConfig.User = username
-
-	logrus.Debugf("constructed SSH config %s for %s", hostConfig, hostname)
+	logrus.Debugf("retrieved ssh config for %s: %s", hostname, hostConfig.String())
 
 	return hostConfig, nil
 }
 
-func identityFileAbsPath(relPath string) (string, error) {
-	var err error
-	var absPath string
-
-	if strings.HasPrefix(relPath, "~/") {
-		systemUser, err := user.Current()
-		if err != nil {
-			return absPath, err
-		}
-		absPath = filepath.Join(systemUser.HomeDir, relPath[2:])
+// Fatal is a error output wrapper which aims to make SSH failures easier to
+// parse through re-wording.
+func Fatal(hostname string, err error) error {
+	out := err.Error()
+	if strings.Contains(out, "Host key verification failed.") {
+		return fmt.Errorf("SSH host key verification failed for %s", hostname)
+	} else if strings.Contains(out, "Could not resolve hostname") {
+		return fmt.Errorf("could not resolve hostname for %s", hostname)
+	} else if strings.Contains(out, "Connection timed out") {
+		return fmt.Errorf("connection timed out for %s", hostname)
+	} else if strings.Contains(out, "Permission denied") {
+		return fmt.Errorf("ssh auth: permission denied for %s", hostname)
+	} else if strings.Contains(out, "Network is unreachable") {
+		return fmt.Errorf("unable to connect to %s, network is unreachable?", hostname)
 	} else {
-		absPath, err = filepath.Abs(relPath)
-		if err != nil {
-			return absPath, err
+		return err
 	}
 }
-
-	logrus.Debugf("resolved %s to %s to read the ssh identity file", relPath, absPath)
-
-	return absPath, nil
-}

pkg/test/test.go

@@ -0,0 +1,38 @@
+package test
+
+import (
+	"log"
+	"os"
+
+	"github.com/sirupsen/logrus"
+)
+
+// RmServerAppRecipe deletes the test server / app / recipe.
+func RmServerAppRecipe() {
+	testAppLink := os.ExpandEnv("$HOME/.abra/servers/foo.com")
+	if err := os.Remove(testAppLink); err != nil {
+		logrus.Fatal(err)
+	}
+
+	testRecipeLink := os.ExpandEnv("$HOME/.abra/recipes/test")
+	if err := os.Remove(testRecipeLink); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+// MkServerAppRecipe symlinks the test server / app / recipe.
+func MkServerAppRecipe() {
+	RmServerAppRecipe()
+
+	testAppDir := os.ExpandEnv("$PWD/../../tests/resources/testapp")
+	testAppLink := os.ExpandEnv("$HOME/.abra/servers/foo.com")
+	if err := os.Symlink(testAppDir, testAppLink); err != nil {
+		log.Fatal(err)
+	}
+
+	testRecipeDir := os.ExpandEnv("$PWD/../../tests/resources/testrecipe")
+	testRecipeLink := os.ExpandEnv("$HOME/.abra/recipes/test")
+	if err := os.Symlink(testRecipeDir, testRecipeLink); err != nil {
+		log.Fatal(err)
+	}
+}

pkg/upstream/commandconn/connection.go

@@ -2,18 +2,15 @@ package commandconn
 
 import (
 	"context"
-	"fmt"
 	"net"
 	"net/url"
 
-	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	"github.com/docker/cli/cli/connhelper"
 	"github.com/docker/cli/cli/connhelper/ssh"
 	"github.com/docker/cli/cli/context/docker"
 	dCliContextStore "github.com/docker/cli/cli/context/store"
 	dClient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 // GetConnectionHelper returns Docker-specific connection helper for the given URL.
@@ -36,25 +33,6 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 			return nil, errors.Wrap(err, "ssh host connection is not valid")
 		}
 
-		if err := sshPkg.EnsureHostKey(ctxConnDetails.Host); err != nil {
-			return nil, err
-		}
-
-		hostConfig, err := sshPkg.GetHostConfig(
-			ctxConnDetails.Host,
-			ctxConnDetails.User,
-			ctxConnDetails.Port,
-			false,
-		)
-		if err != nil {
-			return nil, err
-		}
-		if hostConfig.IdentityFile != "" {
-			msg := "discovered %s as identity file for %s, using for ssh connection"
-			logrus.Debugf(msg, hostConfig.IdentityFile, ctxConnDetails.Host)
-			sshFlags = append(sshFlags, fmt.Sprintf("-o IdentityFile=%s", hostConfig.IdentityFile))
-		}
-
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)

pkg/upstream/stack/stack.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"strings"
+	"os"
+	"os/signal"
 	"time"
 
-	abraClient "coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"github.com/docker/cli/cli/command/service/progress"
+	"github.com/docker/cli/cli/command/stack/formatter"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -18,7 +19,7 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/client"
-	dockerclient "github.com/docker/docker/client"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -31,6 +32,9 @@ const (
 	ResolveImageNever    = "never"
 )
 
+// Timeout to wait until docker services converge, default is 50s (random choice)
+var WaitTimeout int = 50
+
 type StackStatus struct {
 	Services []swarm.Service
 	Err      error
@@ -57,20 +61,10 @@ func GetStackServices(ctx context.Context, dockerclient client.APIClient, namesp
 }
 
 // GetDeployedServicesByLabel filters services by label
-func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
-	cl, err := abraClient.New(contextName)
-	if err != nil {
-		if strings.Contains(err.Error(), "does not exist") {
-			// No local context found, bail out gracefully
-			return StackStatus{[]swarm.Service{}, nil}
-		}
-		return StackStatus{[]swarm.Service{}, err}
-	}
-
-	ctx := context.Background()
+func GetDeployedServicesByLabel(cl *dockerClient.Client, contextName string, label string) StackStatus {
 	filters := filters.NewArgs()
 	filters.Add("label", label)
-	services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: filters})
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filters})
 	if err != nil {
 		return StackStatus{[]swarm.Service{}, err}
 	}
@@ -78,18 +72,8 @@ func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
 	return StackStatus{services, nil}
 }
 
-func GetAllDeployedServices(contextName string) StackStatus {
-	cl, err := abraClient.New(contextName)
-	if err != nil {
-		if strings.Contains(err.Error(), "does not exist") {
-			// No local context found, bail out gracefully
-			return StackStatus{[]swarm.Service{}, nil}
-		}
-		return StackStatus{[]swarm.Service{}, err}
-	}
-
-	ctx := context.Background()
-	services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: getAllStacksFilter()})
+func GetAllDeployedServices(cl *dockerClient.Client, contextName string) StackStatus {
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: getAllStacksFilter()})
 	if err != nil {
 		return StackStatus{[]swarm.Service{}, err}
 	}
@@ -98,7 +82,7 @@ func GetAllDeployedServices(contextName string) StackStatus {
 }
 
 // GetDeployedServicesByName filters services by name
-func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, stackName, serviceName string) StackStatus {
+func GetDeployedServicesByName(ctx context.Context, cl *dockerClient.Client, stackName, serviceName string) StackStatus {
 	filters := filters.NewArgs()
 	filters.Add("name", fmt.Sprintf("%s_%s", stackName, serviceName))
 
@@ -111,7 +95,7 @@ func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, sta
 }
 
 // IsDeployed chekcks whether an appp is deployed or not.
-func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string) (bool, string, error) {
+func IsDeployed(ctx context.Context, cl *dockerClient.Client, stackName string) (bool, string, error) {
 	version := "unknown"
 	isDeployed := false
 
@@ -142,7 +126,7 @@ func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string)
 }
 
 // pruneServices removes services that are no longer referenced in the source
-func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, services map[string]struct{}) {
+func pruneServices(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, services map[string]struct{}) {
 	oldServices, err := GetStackServices(ctx, cl, namespace.Name())
 	if err != nil {
 		logrus.Infof("Failed to list services: %s\n", err)
@@ -158,9 +142,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve
 }
 
 // RunDeploy is the swarm implementation of docker stack deploy
-func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
-	ctx := context.Background()
-
+func RunDeploy(cl *dockerClient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
 	if err := validateResolveImageFlag(&opts); err != nil {
 		return err
 	}
@@ -170,7 +152,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, a
 		opts.ResolveImage = ResolveImageNever
 	}
 
-	return deployCompose(ctx, cl, opts, cfg, appName, dontWait)
+	return deployCompose(context.Background(), cl, opts, cfg, appName, dontWait)
 }
 
 // validateResolveImageFlag validates the opts.resolveImage command line option
@@ -183,7 +165,7 @@ func validateResolveImageFlag(opts *Deploy) error {
 	}
 }
 
-func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
+func deployCompose(ctx context.Context, cl *dockerClient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
 	namespace := convert.NewNamespace(opts.Namespace)
 
 	if opts.Prune {
@@ -241,7 +223,7 @@ func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) ma
 	return serviceNetworks
 }
 
-func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAPIClient, externalNetworks []string) error {
+func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAPIClient, externalNetworks []string) error {
 	for _, networkName := range externalNetworks {
 		if !container.NetworkMode(networkName).IsUserDefined() {
 			// Networks that are not user defined always exist on all nodes as
@@ -250,7 +232,7 @@ func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAP
 		}
 		network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
 		switch {
-		case dockerclient.IsErrNotFound(err):
+		case dockerClient.IsErrNotFound(err):
 			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
 		case err != nil:
 			return err
@@ -261,7 +243,7 @@ func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAP
 	return nil
 }
 
-func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm.SecretSpec) error {
+func createSecrets(ctx context.Context, cl *dockerClient.Client, secrets []swarm.SecretSpec) error {
 	for _, secretSpec := range secrets {
 		secret, _, err := cl.SecretInspectWithRaw(ctx, secretSpec.Name)
 		switch {
@@ -270,7 +252,7 @@ func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm
 			if err := cl.SecretUpdate(ctx, secret.ID, secret.Meta.Version, secretSpec); err != nil {
 				return errors.Wrapf(err, "failed to update secret %s", secretSpec.Name)
 			}
-		case dockerclient.IsErrNotFound(err):
+		case dockerClient.IsErrNotFound(err):
 			// secret does not exist, then we create a new one.
 			logrus.Infof("Creating secret %s\n", secretSpec.Name)
 			if _, err := cl.SecretCreate(ctx, secretSpec); err != nil {
@@ -283,7 +265,7 @@ func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm
 	return nil
 }
 
-func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm.ConfigSpec) error {
+func createConfigs(ctx context.Context, cl *dockerClient.Client, configs []swarm.ConfigSpec) error {
 	for _, configSpec := range configs {
 		config, _, err := cl.ConfigInspectWithRaw(ctx, configSpec.Name)
 		switch {
@@ -292,7 +274,7 @@ func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm
 			if err := cl.ConfigUpdate(ctx, config.ID, config.Meta.Version, configSpec); err != nil {
 				return errors.Wrapf(err, "failed to update config %s", configSpec.Name)
 			}
-		case dockerclient.IsErrNotFound(err):
+		case dockerClient.IsErrNotFound(err):
 			// config does not exist, then we create a new one.
 			logrus.Infof("Creating config %s\n", configSpec.Name)
 			if _, err := cl.ConfigCreate(ctx, configSpec); err != nil {
@@ -305,7 +287,7 @@ func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm
 	return nil
 }
 
-func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
+func createNetworks(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
 	existingNetworks, err := getStackNetworks(ctx, cl, namespace.Name())
 	if err != nil {
 		return err
@@ -335,7 +317,7 @@ func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace conv
 
 func deployServices(
 	ctx context.Context,
-	cl *dockerclient.Client,
+	cl *dockerClient.Client,
 	services map[string]swarm.ServiceSpec,
 	namespace convert.Namespace,
 	sendAuth bool,
@@ -433,7 +415,7 @@ func deployServices(
 		return nil
 	}
 
-	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
+	logrus.Infof("Waiting for %s to deploy... please hold 🤚", appName)
 	ch := make(chan error, len(serviceIDs))
 	for serviceID, serviceName := range serviceIDs {
 		logrus.Debugf("waiting on %s to converge", serviceName)
@@ -450,7 +432,7 @@ func deployServices(
 		logrus.Debugf("assuming %s converged successfully", serviceID)
 	}
 
-	logrus.Info("services converged 👌")
+	logrus.Infof("Successfully deployed %s", appName)
 
 	return nil
 }
@@ -469,21 +451,31 @@ func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespa
 
 // https://github.com/docker/cli/blob/master/cli/command/service/helpers.go
 // https://github.com/docker/cli/blob/master/cli/command/service/progress/progress.go
-func WaitOnService(ctx context.Context, cl *dockerclient.Client, serviceID, appName string) error {
+func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appName string) error {
 	errChan := make(chan error, 1)
 	pipeReader, pipeWriter := io.Pipe()
 
+	sigintChannel := make(chan os.Signal, 1)
+	signal.Notify(sigintChannel, os.Interrupt)
+	defer signal.Stop(sigintChannel)
+
 	go func() {
 		errChan <- progress.ServiceProgress(ctx, cl, serviceID, pipeWriter)
 	}()
 
 	go io.Copy(ioutil.Discard, pipeReader)
 
-	timeout := 50 * time.Second
+	timeout := time.Duration(WaitTimeout) * time.Second
 
 	select {
 	case err := <-errChan:
 		return err
+	case <-sigintChannel:
+		return fmt.Errorf(fmt.Sprintf(`
+Not waiting for %s to deploy. The deployment is ongoing...
+
+If you want to stop the deployment, try:
+  abra app undeploy %s`, appName, appName))
 	case <-time.After(timeout):
 		return fmt.Errorf(fmt.Sprintf(`
 %s has not converged (%s second timeout reached).
@@ -500,10 +492,44 @@ And inspect the logs with:
 
     abra app logs %s
 
-If a service is failing to even start, try smoke out the error with:
+If a service is failing to even start, try to smoke out the error with:
 
     abra app errors --watch %s
 
 `, appName, timeout, appName, appName, appName))
 	}
 }
+
+// Copypasta from https://github.com/docker/cli/blob/master/cli/command/stack/swarm/list.go
+// GetStacks lists the swarm stacks.
+func GetStacks(cl *dockerClient.Client) ([]*formatter.Stack, error) {
+	services, err := cl.ServiceList(
+		context.Background(),
+		types.ServiceListOptions{Filters: getAllStacksFilter()})
+	if err != nil {
+		return nil, err
+	}
+	m := make(map[string]*formatter.Stack)
+	for _, service := range services {
+		labels := service.Spec.Labels
+		name, ok := labels[convert.LabelNamespace]
+		if !ok {
+			return nil, errors.Errorf("cannot get label %s for service %s",
+				convert.LabelNamespace, service.ID)
+		}
+		ztack, ok := m[name]
+		if !ok {
+			m[name] = &formatter.Stack{
+				Name:     name,
+				Services: 1,
+			}
+		} else {
+			ztack.Services++
+		}
+	}
+	var stacks []*formatter.Stack
+	for _, stack := range m {
+		stacks = append(stacks, stack)
+	}
+	return stacks, nil
+}

renovate.json

@@ -1,3 +1,14 @@
 {
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json"
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "packageRules": [{
+    "matchUpdateTypes": ["minor", "patch"],
+    "automerge": true
+  }],
+  "postUpdateOptions": [
+    "gomodTidy"
+  ],
+  "ignoreDeps": [
+    "github.com/urfave/cli",
+    "goreleaser/goreleaser"
+  ]
 }

scripts/autocomplete/fish

@@ -0,0 +1,7 @@
+function complete_abra_args
+    set -l cmd (commandline -poc) --generate-bash-completion
+    $cmd
+end
+complete -c abra -f -n "not __fish_seen_subcommand_from -h --help -v --version complete_abra_args" -a "(complete_abra_args)"
+complete -c abra -f -s h -l help -d 'show help'
+complete -c abra -f -s v -l version -d 'print the version'

scripts/installer/installer

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.4.0-alpha"
+ABRA_VERSION="0.7.0-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.4.0-alpha"
+RC_VERSION="0.8.0-rc2-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do
@@ -44,8 +44,17 @@ function install_abra_release {
     exit 1
   fi
 
-
-  PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m)
+  ARCH=$(uname -m)
+  if [[ $ARCH =~ "aarch64" ]]; then
+      ARCH="arm64"
+  elif [[ $ARCH =~ "armv5l" ]]; then
+      ARCH="armv5"
+  elif [[ $ARCH =~ "armv6l" ]]; then
+      ARCH="armv6"
+  elif [[ $ARCH =~ "armv7l" ]]; then
+      ARCH="armv7"
+  fi
+  PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')_$ARCH
   FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM""
   sed_command_rel='s/.*"assets":\[\{[^]]*"name":"'$FILENAME'"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
   sed_command_checksums='s/.*"assets":\[\{[^\]*"name":"checksums.txt"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'

tests/README.md

@@ -0,0 +1,10 @@
+# Test suite
+
+* Unit testing is done in the packages themselves, run `find . -name
+  "*_test.go"` to find those. Use `make test` to run the entire unit test
+  suite. Some unit tests require mocked files which are located in
+  `./tests/resources.`
+
+* Integration tests are in `./tests/integration`. Please see [these
+  docs](https://docs.coopcloud.tech/abra/hack/#integration-tests) for
+  instructions and tips on how to run them.

tests/integration/.envrc.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

tests/integration/README.md

@@ -1,28 +0,0 @@
-# integration tests
-
-> You need to be a member of Autonomic Co-op to run these tests, sorry!
-
-`testfunctions.sh` contains the functions necessary to save and manipulate
-logs. Run `test_all.sh logdir` to run tests specified in that file and save the
-logs to `logdir`.
-
-When creating new tests, make sure the test command is a one-liner (you can use
-`;` to separate commands). Include `testfunctions.sh` and then write your tests
-like this:
-
-```
-run_test '$ABRA other stuff here'
-```
-
-By default, the testing script will ask after every command if the execution
-succeeded. If you reply `n`, it will log the test in the `logdir`. If you want
-all tests to run without questions, run `export logall=yes` before executing
-the test script.
-
-To run tests, you'll need to prepare your environment:
-
-```
-cp .envrc.sample .envrc # fill out values...
-direnv allow
-./test_all.sh logs
-```

tests/integration/app.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-source ./testfunctions.sh
-source ./common.sh
-
-run_test '$ABRA app ls'
-
-run_test '$ABRA app ls --status'
-
-run_test '$ABRA app ls --type wordpress'
-
-run_test '$ABRA app ls --type wordpress --server swarm.autonomic.zone'
-
-run_test '$ABRA app ls --type wordpress --server swarm.autonomic.zone --status'

tests/integration/app_backup.bats

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+setup_file() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file() {
+  _rm_app
+  _rm_server
+}
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "retrieve recipe if missing" {
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app backup "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'no containers matching'
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --chaos --no-input
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_success
+  assert_output --partial 'running backup for the app service'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "ensure recipe not up to date if --offline" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  refute [ -z "$latestCommit" ];
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug --offline
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "detect backup labels" {
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  assert_output --partial 'detected backup paths'
+  assert_output --partial 'detected pre-hook command'
+  assert_output --partial 'detected post-hook command'
+}
+
+@test "error if backups not enabled" {
+  run sed -i '/backupbot.backup=true/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_failure
+  assert_output --partial 'no backup config for app'
+
+  _checkout_recipe
+}
+
+@test "error if backup paths not configured" {
+  run sed -i '/backupbot.backup.path=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_failure
+  assert_output --partial 'backup paths are empty for app?'
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "backup single service" {
+  _deploy_app
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app
+  assert_success
+  assert_output --partial 'running backup for the app service'
+
+  sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
+  assert_output --partial "_$sanitisedDomainName_app"
+
+  assert_exists "$ABRA_DIR/backups"
+  assert bash -c "ls $ABRA_DIR/backups | grep -q $1_$sanitisedDomainName_app"
+
+  _undeploy_app
+}