fix: app version includes tags, sorts & tests

See coop-cloud/organising#442

.drone.yml

@@ -3,19 +3,19 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.20
+    image: golang:1.21
     commands:
       - make check
 
   - name: make build
-    image: golang:1.20
+    image: golang:1.21
     commands:
       - make build
     depends_on:
       - make check
 
   - name: make test
-    image: golang:1.20
+    image: golang:1.21
     commands:
       - make test
     depends_on:

.e2e.env.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

.envrc.sample

@@ -1,6 +1,7 @@
 go env -w GOPRIVATE=coopcloud.tech
 
 # export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
-# export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
-# export CAPSUL_TOKEN=...
-# export GITEA_TOKEN=...
+
+# export ABRA_DIR="$HOME/.abra_test"
+# export ABRA_TEST_DOMAIN=test.example.com
+# export ABRA_SKIP_TEARDOWN=1 # for faster feedback when developing tests

.gitignore

@@ -5,5 +5,5 @@
 /kadabra
 abra
 dist/
-tests/integration/.abra/catalogue
+tests/integration/.bats
 vendor/

AUTHORS.md

@@ -1,7 +1,7 @@
 # authors
 
 > If you're looking at this and you hack on `abra` and you're not listed here,
-> please do add yourself! This is a community project, let's show some :heart:
+> please do add yourself! This is a community project, let's show some 💞
 
 - 3wordchant
 - cassowary

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine AS build
+FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 

Makefile

@@ -9,15 +9,24 @@ export GOPRIVATE=coopcloud.tech
 
 all: format check build test
 
-run:
+run-abra:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
 
-install:
+run-kadabra:
+	@go run -ldflags=$(LDFLAGS) $(KADABRA)
+
+install-abra:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
-build-dev:
+install-kadaabra:
+	@go install -ldflags=$(LDFLAGS) $(KADABRA)
+
+build-abra:
 	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
 
+build-kadabra:
+	@go build -v -ldflags=$(LDFLAGS) $(KADABRA)
+
 build:
 	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
 	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
@@ -38,10 +47,3 @@ test:
 
 loc:
 	@find . -name "*.go" | xargs wc -l
-
-loc-author:
-	@git ls-files -z | \
-		xargs -0rn 1 -P "$$(nproc)" -I{} sh -c 'git blame -w -M -C -C --line-porcelain -- {} | grep -I --line-buffered "^author "' | \
-   	sort -f | \
-   	uniq -ic | \
-   	sort -n

README.md

@@ -8,6 +8,6 @@ The Co-op Cloud utility belt 🎩🐇
 
 <a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community 💖
 
 Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

cli/app/backup.go

@@ -15,8 +15,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -42,6 +41,7 @@ var appBackupCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
@@ -72,17 +72,27 @@ This file is a compressed archive which contains all backup paths. To see paths,
 This single file can be used to restore your app. See "abra app restore" for more.
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		recipe, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 		backupConfigs := make(map[string]backupConfig)
@@ -114,6 +124,11 @@ This single file can be used to restore your app. See "abra app restore" for mor
 			}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName != "" {
 			backupConfig, ok := backupConfigs[serviceName]

cli/app/check.go

@@ -8,7 +8,8 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -16,16 +17,37 @@ import (
 var appCheckCommand = cli.Command{
 	Name:      "check",
 	Aliases:   []string{"chk"},
-	Usage:     "Check if app is configured correctly",
+	Usage:     "Check if an app is configured correctly",
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.ChaosFlag,
 		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
 		if _, err := os.Stat(envSamplePath); err != nil {
@@ -56,5 +78,4 @@ var appCheckCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cmd.go

@@ -12,7 +12,8 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -40,18 +41,33 @@ Example:
 		internal.RemoteUserFlag,
 		internal.TtyFlag,
 		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	BashComplete: autocomplete.AppNameComplete,
 	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if internal.LocalCmd && internal.RemoteUser != "" {
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
 		}
@@ -67,6 +83,10 @@ Example:
 		}
 
 		if internal.LocalCmd {
+			if !(len(c.Args()) >= 2) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
 			cmdName := c.Args().Get(1)
 			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
 				logrus.Fatal(err)
@@ -78,6 +98,7 @@ Example:
 			for k, v := range app.Env {
 				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
 			}
+
 			var sourceAndExec string
 			if hasCmdArgs {
 				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
@@ -98,6 +119,10 @@ Example:
 				logrus.Fatal(err)
 			}
 		} else {
+			if !(len(c.Args()) >= 3) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
 			targetServiceName := c.Args().Get(1)
 
 			cmdName := c.Args().Get(2)
@@ -129,6 +154,11 @@ Example:
 				logrus.Debug("did not detect any command arguments")
 			}
 
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
 			if err := internal.RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
 				logrus.Fatal(err)
 			}

cli/app/config.go

@@ -20,9 +20,9 @@ var appConfigCommand = cli.Command{
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
-		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		appName := c.Args().First()
 
@@ -61,5 +61,4 @@ var appConfigCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cp.go

@@ -12,7 +12,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
@@ -28,10 +27,9 @@ var appCpCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
-	Usage:  "Copy files to/from a running app service",
+	Usage:  "Copy files to/from a deployed app service",
 	Description: `
 Copy files to and from any app service file system.
 
@@ -41,16 +39,11 @@ If you want to copy a myfile.txt to the root of the app service:
 
 And if you want to copy that file back to your current working directory locally:
 
-		abra app cp <domain> app:/myfile.txt .
+    abra app cp <domain> app:/myfile.txt .
 `,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
+		app := internal.ValidateApp(c)
 
 		src := c.Args().Get(1)
 		dst := c.Args().Get(2)
@@ -92,19 +85,23 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Debugf("assuming transfer is going TO the container")
 		}
 
-		if !isToContainer {
-			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist locally?", dstPath)
+		if isToContainer {
+			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", srcPath)
 			}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 func configureAndCp(
@@ -126,10 +123,6 @@ func configureAndCp(
 	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 	if isToContainer {
-		if _, err := os.Stat(srcPath); err != nil {
-			logrus.Fatalf("%s does not exist?", srcPath)
-		}
-
 		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
 		content, err := archive.TarWithOptions(srcPath, toTarOpts)
 		if err != nil {

cli/app/deploy.go

@@ -3,14 +3,9 @@ package app
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/runtime"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -20,8 +15,6 @@ import (
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/AlecAivazis/survey/v2"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -30,7 +23,7 @@ var appDeployCommand = cli.Command{
 	Name:      "deploy",
 	Aliases:   []string{"d"},
 	Usage:     "Deploy an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -54,22 +47,30 @@ recipes.
 `,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
-		cl, err := client.New(app.Server)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe, conf); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		r, err := recipe.Get(app.Recipe, conf)
+		r, err := recipe.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -80,6 +81,11 @@ recipes.
 
 		logrus.Debugf("checking whether %s is already deployed", stackName)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
@@ -93,10 +99,18 @@ recipes.
 			}
 		}
 
-		isLatestHash := false
 		version := deployedVersion
-		if version == "unknown" && !internal.Chaos {
-			catl, err := recipe.ReadRecipeCatalogue(conf)
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			version = specificVersion
+			logrus.Debugf("choosing %s as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if !internal.Chaos && specificVersion == "" {
+			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -104,7 +118,21 @@ recipes.
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			if len(versions) > 0 {
+
+			if len(versions) == 0 && !internal.Chaos {
+				logrus.Warn("no published versions in catalogue, trying local recipe repository")
+				recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+				if err != nil {
+					logrus.Warn(err)
+				}
+				for _, recipeVersion := range recipeVersions {
+					for version := range recipeVersion {
+						versions = append(versions, version)
+					}
+				}
+			}
+
+			if len(versions) > 0 && !internal.Chaos {
 				version = versions[len(versions)-1]
 				logrus.Debugf("choosing %s as version to deploy", version)
 				if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
@@ -115,25 +143,8 @@ recipes.
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				isLatestHash = true
 				version = formatter.SmallSHA(head.String())
 				logrus.Warn("no versions detected, using latest commit")
-				if err := recipe.EnsureLatest(app.Recipe, conf); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
-		if version == "unknown" && !internal.Chaos {
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		if version != "unknown" && !internal.Chaos && !isLatestHash {
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
 			}
 		}
 
@@ -159,6 +170,7 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+
 		deployOpts := stack.Deploy{
 			Composefiles: composeFiles,
 			Namespace:    stackName,
@@ -169,13 +181,14 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+
 		config.ExposeAllEnv(stackName, compose, app.Env)
 		config.SetRecipeLabel(compose, stackName, app.Recipe)
 		config.SetChaosLabel(compose, stackName, internal.Chaos)
 		config.SetChaosVersionLabel(compose, stackName, version)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
-		if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
+		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -205,175 +218,10 @@ recipes.
 		postDeployCmds, ok := app.Env["POST_DEPLOY_CMDS"]
 		if ok && !internal.DontWaitConverge {
 			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
-			if err := PostCmds(cl, app, postDeployCmds); err != nil {
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
 				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
 			}
 		}
 		return nil
 	},
 }
-
-// PostCmds parses a string of commands and executes them inside of the respective services
-// the commands string must have the following format:
-// "<service> <command> <arguments>|<service> <command> <arguments>|... "
-func PostCmds(cl *dockerClient.Client, app config.App, commands string) error {
-	abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
-	if _, err := os.Stat(abraSh); err != nil {
-		if os.IsNotExist(err) {
-			return fmt.Errorf(fmt.Sprintf("%s does not exist for %s?", abraSh, app.Name))
-		}
-		return err
-	}
-
-	for _, command := range strings.Split(commands, "|") {
-		commandParts := strings.Split(command, " ")
-		if len(commandParts) < 2 {
-			return fmt.Errorf(fmt.Sprintf("not enough arguments: %s", command))
-		}
-		targetServiceName := commandParts[0]
-		cmdName := commandParts[1]
-		parsedCmdArgs := ""
-		if len(commandParts) > 2 {
-			parsedCmdArgs = fmt.Sprintf("%s ", strings.Join(commandParts[2:], " "))
-		}
-		logrus.Infof("running post-command '%s %s' in container %s", cmdName, parsedCmdArgs, targetServiceName)
-
-		if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
-			return err
-		}
-
-		serviceNames, err := config.GetAppServiceNames(app.Name)
-		if err != nil {
-			return err
-		}
-
-		matchingServiceName := false
-		for _, serviceName := range serviceNames {
-			if serviceName == targetServiceName {
-				matchingServiceName = true
-			}
-		}
-
-		if !matchingServiceName {
-			return fmt.Errorf(fmt.Sprintf("no service %s for %s?", targetServiceName, app.Name))
-		}
-
-		logrus.Debugf("running command %s %s within the context of %s_%s", cmdName, parsedCmdArgs, app.StackName(), targetServiceName)
-
-		internal.Tty = true
-		if err := internal.RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// DeployOverview shows a deployment overview
-func DeployOverview(app config.App, version, message string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "version"}
-	table := formatter.CreateTable(tableCol)
-
-	deployConfig := "compose.yml"
-	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
-		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
-	}
-
-	server := app.Server
-	if app.Server == "default" {
-		server = "local"
-	}
-
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, version})
-	table.Render()
-
-	if internal.NoInput {
-		return nil
-	}
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: message,
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	return nil
-}
-
-// NewVersionOverview shows an upgrade or downgrade overview
-func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
-	table := formatter.CreateTable(tableCol)
-
-	deployConfig := "compose.yml"
-	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
-		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
-	}
-
-	server := app.Server
-	if app.Server == "default" {
-		server = "local"
-	}
-
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
-	table.Render()
-
-	if releaseNotes == "" {
-		var err error
-		releaseNotes, err = GetReleaseNotes(app.Recipe, newVersion)
-		if err != nil {
-			return err
-		}
-	}
-
-	if releaseNotes != "" && newVersion != "" {
-		fmt.Println()
-		fmt.Println(fmt.Sprintf("%s release notes:\n\n%s", newVersion, releaseNotes))
-	} else {
-		logrus.Warnf("no release notes available for %s", newVersion)
-	}
-
-	if internal.NoInput {
-		return nil
-	}
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with deployment?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	return nil
-}
-
-// GetReleaseNotes prints release notes for a recipe version
-func GetReleaseNotes(recipeName, version string) (string, error) {
-	if version == "" {
-		return "", nil
-	}
-
-	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
-
-	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
-		releaseNotes, err := ioutil.ReadFile(fpath)
-		if err != nil {
-			return "", err
-		}
-		return string(releaseNotes), nil
-	}
-
-	return "", nil
-}

cli/app/errors.go

@@ -12,7 +12,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -56,8 +55,7 @@ the logs.
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -74,25 +72,23 @@ the logs.
 		}
 
 		if !internal.Watch {
-			if err := checkErrors(c, cl, app, conf); err != nil {
+			if err := checkErrors(c, cl, app); err != nil {
 				logrus.Fatal(err)
 			}
 			return nil
 		}
 
 		for {
-			if err := checkErrors(c, cl, app, conf); err != nil {
+			if err := checkErrors(c, cl, app); err != nil {
 				logrus.Fatal(err)
 			}
 			time.Sleep(2 * time.Second)
 		}
-
-		return nil
 	},
 }
 
-func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App, conf *runtime.Config) error {
-	recipe, err := recipe.Get(app.Recipe, conf)
+func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
+	recipe, err := recipe.Get(app.Recipe, internal.Offline)
 	if err != nil {
 		return err
 	}

cli/app/list.go

@@ -11,7 +11,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -84,8 +83,6 @@ can take some time.
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-
 		appFiles, err := config.LoadAppFiles(listAppServer)
 		if err != nil {
 			logrus.Fatal(err)
@@ -113,7 +110,7 @@ can take some time.
 				logrus.Fatal(err)
 			}
 
-			catl, err = recipe.ReadRecipeCatalogue(conf)
+			catl, err = recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}

cli/app/logs.go

@@ -11,8 +11,8 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
@@ -79,23 +79,27 @@ var appLogsCommand = cli.Command{
 		internal.StdErrOnlyFlag,
 		internal.SinceLogsFlag,
 		internal.DebugFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(
-			runtime.WithOffline(internal.Offline),
-			runtime.WithEnsureRecipeExists(false),
-		)
-
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
 		logOpts.Since = internal.SinceLogs
 
 		serviceName := c.Args().Get(1)

cli/app/new.go

@@ -13,7 +13,6 @@ import (
 	"coopcloud.tech/abra/pkg/jsontable"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/AlecAivazis/survey/v2"
 	dockerClient "github.com/docker/docker/client"
@@ -55,17 +54,19 @@ var appNewCommand = cli.Command{
 		internal.SecretsFlag,
 		internal.OfflineFlag,
 	},
-	Before:    internal.SubCommandBefore,
-	ArgsUsage: "[<recipe>]",
+	Before:       internal.SubCommandBefore,
+	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(
-			runtime.WithOffline(internal.Offline),
-			runtime.WithEnsureRecipeUpToDate(false),
-		)
+		recipe := internal.ValidateRecipe(c)
 
-		recipe := internal.ValidateRecipeWithPrompt(c, conf)
+		if !internal.Offline {
+			if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name, conf); err != nil {
+		if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -93,14 +94,14 @@ var appNewCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		cl, err := client.New(internal.NewAppServer)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
+			cl, err := client.New(internal.NewAppServer)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
 			secrets, err := createSecrets(cl, sanitisedAppName)
 			if err != nil {
 				logrus.Fatal(err)
@@ -144,7 +145,6 @@ var appNewCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }
 
 // AppSecrets represents all app secrest

cli/app/ps.go

@@ -10,7 +10,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/buger/goterm"
@@ -30,13 +29,11 @@ var appPsCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.WatchFlag,
 		internal.DebugFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		cl, err := client.New(app.Server)
 		if err != nil {

cli/app/remove.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
@@ -49,8 +48,7 @@ flag.
 	BashComplete: autocomplete.AppNameComplete,
 	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		if !internal.Force && !internal.NoInput {
 			response := false

cli/app/restart.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/runtime"
 	upstream "coopcloud.tech/abra/pkg/upstream/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/sirupsen/logrus"
@@ -28,8 +27,7 @@ var appRestartCommand = cli.Command{
 	Description:  `This command restarts a service within a deployed app.`,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		serviceNameShort := c.Args().Get(1)
 		if serviceNameShort == "" {
@@ -42,6 +40,15 @@ var appRestartCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
 		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
 
 		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)

cli/app/restore.go

@@ -12,7 +12,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -36,6 +36,7 @@ var appRestoreCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
@@ -49,21 +50,37 @@ restoring the backup.
 Unlike "abra app backup", restore must be run on a per-service basis. You can
 not restore all services in one go. Backup files produced by Abra are
 compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic.
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
 
 Example:
 
     abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
@@ -80,11 +97,6 @@ Example:
 			}
 		}
 
-		recipe, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		restoreConfigs := make(map[string]restoreConfig)
 		for _, service := range recipe.Config.Services {
 			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
@@ -114,6 +126,11 @@ Example:
 			rsConfig = restoreConfig{}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
 			logrus.Fatal(err)
 		}
@@ -170,8 +187,8 @@ func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName
 		return err
 	}
 
-	// we use absolute paths so tar knows what to do. it will restore files
-	// according to the paths set in the compresed archive
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
 	restorePath := "/"
 
 	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}

cli/app/rollback.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 
@@ -23,7 +22,7 @@ var appRollbackCommand = cli.Command{
 	Name:      "rollback",
 	Aliases:   []string{"rl"},
 	Usage:     "Roll an app back to a previous version",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -49,17 +48,30 @@ recipes.
 `,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
 		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe, conf); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		r, err := recipe.Get(app.Recipe, conf)
+		r, err := recipe.Get(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -84,7 +96,7 @@ recipes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue(conf)
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -95,16 +107,41 @@ recipes.
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
 		}
 
 		var availableDowngrades []string
 		if deployedVersion == "unknown" {
 			availableDowngrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		if deployedVersion != "unknown" && !internal.Chaos {
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsGreaterThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not a downgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableDowngrades = append(availableDowngrades, specificVersion)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
 			for _, version := range versions {
 				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 				if err != nil {
@@ -114,12 +151,12 @@ recipes.
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				if parsedVersion != parsedDeployedVersion && parsedVersion.IsLessThan(parsedDeployedVersion) {
+				if parsedVersion.IsLessThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
 					availableDowngrades = append(availableDowngrades, version)
 				}
 			}
 
-			if len(availableDowngrades) == 0 {
+			if len(availableDowngrades) == 0 && !internal.Force {
 				logrus.Info("no available downgrades, you're on oldest ✌️")
 				return nil
 			}
@@ -127,9 +164,9 @@ recipes.
 
 		var chosenDowngrade string
 		if len(availableDowngrades) > 0 && !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force || internal.NoInput || specificVersion != "" {
 				chosenDowngrade = availableDowngrades[len(availableDowngrades)-1]
-				logrus.Debugf("choosing %s as version to downgrade to (--force)", chosenDowngrade)
+				logrus.Debugf("choosing %s as version to downgrade to (--force/--no-input)", chosenDowngrade)
 			} else {
 				prompt := &survey.Select{
 					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
@@ -185,7 +222,8 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, chosenDowngrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
-		if err := NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
+		// NOTE(d1): no release notes implemeneted for rolling back
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/run.go

@@ -9,7 +9,6 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -38,15 +37,13 @@ var appRunCommand = cli.Command{
 		internal.DebugFlag,
 		noTTYFlag,
 		userFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "<domain> <service> <args>...",
 	Usage:        "Run a command in a service container",
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		if len(c.Args()) < 2 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no <service> provided?"))

cli/app/secret.go

@@ -12,7 +12,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
@@ -43,18 +42,11 @@ var appSecretGenerateCommand = cli.Command{
 		internal.DebugFlag,
 		allSecretsFlag,
 		internal.PassFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
+		app := internal.ValidateApp(c)
 
 		if len(c.Args()) == 1 && !allSecrets {
 			err := errors.New("missing arguments <secret>/<version> or '--all'")
@@ -87,6 +79,11 @@ var appSecretGenerateCommand = cli.Command{
 			}
 		}
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
@@ -124,7 +121,6 @@ var appSecretInsertCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.PassFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "<domain> <secret-name> <version> <data>",
@@ -142,18 +138,17 @@ Example:
 
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
+
+		if len(c.Args()) != 4 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
+		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if len(c.Args()) != 4 {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
-		}
-
 		name := c.Args().Get(1)
 		version := c.Args().Get(2)
 		data := c.Args().Get(3)
@@ -203,7 +198,6 @@ var appSecretRmCommand = cli.Command{
 		internal.NoInputFlag,
 		rmAllSecretsFlag,
 		internal.PassRemoveFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "<domain> [<secret-name>]",
@@ -216,8 +210,7 @@ Example:
     abra app secret remove myapp db_pass
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		secrets := secret.ReadSecretEnvVars(app.Env)
 
 		if c.Args().Get(1) != "" && rmAllSecrets {
@@ -295,13 +288,12 @@ var appSecretLsCommand = cli.Command{
 	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
-		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
-	Usage:  "List all secrets",
+	Before:       internal.SubCommandBefore,
+	Usage:        "List all secrets",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		secrets := secret.ReadSecretEnvVars(app.Env)
 
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
@@ -350,7 +342,6 @@ var appSecretLsCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appSecretCommand = cli.Command{

cli/app/services.go

@@ -9,7 +9,6 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
@@ -24,13 +23,11 @@ var appServicesCommand = cli.Command{
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		cl, err := client.New(app.Server)
 		if err != nil {

cli/app/undeploy.go

@@ -10,7 +10,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
@@ -87,7 +86,6 @@ var appUndeployCommand = cli.Command{
 		internal.DebugFlag,
 		internal.NoInputFlag,
 		pruneFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "Undeploy an app",
@@ -101,8 +99,7 @@ any previously attached volumes as eligible for pruning once undeployed.
 Passing "-p/--prune" does not remove those volumes.
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
 		cl, err := client.New(app.Server)
@@ -121,7 +118,7 @@ Passing "-p/--prune" does not remove those volumes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		if err := DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
+		if err := internal.DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/upgrade.go

@@ -10,7 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -22,7 +22,7 @@ var appUpgradeCommand = cli.Command{
 	Name:      "upgrade",
 	Aliases:   []string{"up"},
 	Usage:     "Upgrade an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<version>]",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -51,33 +51,43 @@ Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe, conf); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		r, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("checking whether %s is already deployed", stackName)
-
 		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
@@ -87,37 +97,63 @@ recipes.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue(conf)
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		versions, err := recipePkg.GetRecipeCatalogueVersions(app.Recipe, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipePkg.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
 		}
 
 		var availableUpgrades []string
 		if deployedVersion == "unknown" {
 			availableUpgrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		if deployedVersion != "unknown" && !internal.Chaos {
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsLessThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not an upgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableUpgrades = append(availableUpgrades, specificVersion)
+		}
+
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
 			for _, version := range versions {
-				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
-				if err != nil {
-					logrus.Fatal(err)
-				}
 				parsedVersion, err := tagcmp.Parse(version)
 				if err != nil {
 					logrus.Fatal(err)
 				}
-				if parsedVersion.IsGreaterThan(parsedDeployedVersion) {
+				if parsedVersion.IsGreaterThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
 					availableUpgrades = append(availableUpgrades, version)
 				}
 			}
@@ -130,7 +166,7 @@ recipes.
 
 		var chosenUpgrade string
 		if len(availableUpgrades) > 0 && !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force || internal.NoInput || specificVersion != "" {
 				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
 				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
 			} else {
@@ -152,13 +188,30 @@ recipes.
 		// if release notes written after git tag published, read them before we
 		// check out the tag and then they'll appear to be missing. this covers
 		// when we obviously will forget to write release notes before publishing
-		releaseNotes, err := GetReleaseNotes(app.Recipe, chosenUpgrade)
-		if err != nil {
-			return err
+		var releaseNotes string
+		for _, version := range versions {
+			parsedVersion, err := tagcmp.Parse(version)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !(parsedVersion.Equals(parsedDeployedVersion)) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
+				note, err := internal.GetReleaseNotes(app.Recipe, version)
+				if err != nil {
+					return err
+				}
+				if note != "" {
+					releaseNotes += fmt.Sprintf("%s\n", note)
+				}
+			}
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
+			if err := recipePkg.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -166,7 +219,7 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenUpgrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenUpgrade, err = recipePkg.ChaosVersion(app.Recipe)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -201,7 +254,7 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
-		if err := NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -218,12 +271,11 @@ recipes.
 		postDeployCmds, ok := app.Env["POST_UPGRADE_CMDS"]
 		if ok && !internal.DontWaitConverge {
 			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
-			if err := PostCmds(cl, app, postDeployCmds); err != nil {
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
 				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
 			}
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/version.go

@@ -2,19 +2,30 @@ package app
 
 import (
 	"context"
+	"sort"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
+	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
 // getImagePath returns the image name
 func getImagePath(image string) (string, error) {
 	img, err := reference.ParseNormalizedNamed(image)
@@ -40,16 +51,11 @@ var appVersionCommand = cli.Command{
 		internal.NoInputFlag,
 		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Show app versions",
-	Description: `
-Show all information about versioning related to a deployed app. This includes
-the individual image names, tags and digests. But also the Co-op Cloud recipe
-version.
-`,
+	Before:       internal.SubCommandBefore,
+	Usage:        "Show version info of a deployed app",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
 		cl, err := client.New(app.Server)
@@ -64,15 +70,15 @@ version.
 			logrus.Fatal(err)
 		}
 
-		if deployedVersion == "unknown" {
-			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
-		}
-
 		if !isDeployed {
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, conf)
+		if deployedVersion == "unknown" {
+			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
+		}
+
+		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -88,17 +94,24 @@ version.
 			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"version", "service", "image"}
+		tableCol := []string{"version", "service", "image", "tag"}
 		table := formatter.CreateTable(tableCol)
-		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
+		var versions [][]string
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{deployedVersion, serviceName, versionMeta.Image})
+			versions = append(versions, []string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Tag})
 		}
 
+		sort.Slice(versions, sortServiceByName(versions))
+
+		for _, version := range versions {
+			table.Append(version)
+		}
+
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+		table.SetAlignment(tablewriter.ALIGN_LEFT)
 		table.Render()
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/volume.go

@@ -7,7 +7,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/runtime"
+	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -20,14 +20,12 @@ var appVolumeListCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "List volumes associated with an app",
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -83,18 +81,26 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		internal.DebugFlag,
 		internal.NoInputFlag,
 		internal.ForceFlag,
-		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		app := internal.ValidateApp(c, conf)
+		app := internal.ValidateApp(c)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+		}
+
 		filters, err := app.Filters(false, true)
 		if err != nil {
 			logrus.Fatal(err)
@@ -124,16 +130,19 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 			volumesToRemove = volumeNames
 		}
 
-		err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
-		if err != nil {
-			logrus.Fatal(err)
-		}
+		if len(volumesToRemove) > 0 {
+			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
+			if err != nil {
+				logrus.Fatal(err)
+			}
 
-		logrus.Info("volumes removed successfully")
+			logrus.Info("volumes removed successfully")
+		} else {
+			logrus.Info("no volumes removed")
+		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appVolumeCommand = cli.Command{

cli/catalogue/catalogue.go

@@ -13,7 +13,6 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -29,6 +28,7 @@ var catalogueGenerateCommand = cli.Command{
 		internal.PublishFlag,
 		internal.DryFlag,
 		internal.SkipUpdatesFlag,
+		internal.ChaosFlag,
 		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
@@ -53,20 +53,22 @@ Push your new release to git.coopcloud.tech with "-p/--publish". This requires
 that you have permission to git push to these repositories and have your SSH
 keys configured on your account.
 `,
-	ArgsUsage: "[<recipe>]",
+	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
 		recipeName := c.Args().First()
 
 		if recipeName != "" {
-			internal.ValidateRecipe(c, conf)
+			internal.ValidateRecipe(c)
 		}
 
-		if err := catalogue.EnsureUpToDate(conf); err != nil {
-			logrus.Fatal(err)
+		if !internal.Chaos {
+			if err := catalogue.EnsureIsClean(); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		repos, err := recipe.ReadReposMetadata(conf)
+		repos, err := recipe.ReadReposMetadata()
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -83,7 +85,7 @@ keys configured on your account.
 
 		if !internal.SkipUpdates {
 			logrus.Warn(logMsg)
-			if err := recipe.UpdateRepositories(repos, recipeName, conf); err != nil {
+			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -101,7 +103,7 @@ keys configured on your account.
 				continue
 			}
 
-			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, conf)
+			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
 			}
@@ -137,7 +139,7 @@ keys configured on your account.
 				logrus.Fatal(err)
 			}
 		} else {
-			catlFS, err := recipe.ReadRecipeCatalogue(conf)
+			catlFS, err := recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -211,7 +213,6 @@ keys configured on your account.
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }
 
 // CatalogueCommand defines the `abra catalogue` command and sub-commands.

cli/cli.go

@@ -12,10 +12,10 @@ import (
 	"coopcloud.tech/abra/cli/catalogue"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/cli/recipe"
-	"coopcloud.tech/abra/cli/record"
 	"coopcloud.tech/abra/cli/server"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -158,7 +158,6 @@ func newAbraApp(version, commit string) *cli.App {
 			server.ServerCommand,
 			recipe.RecipeCommand,
 			catalogue.CatalogueCommand,
-			record.RecordCommand,
 			UpgradeCommand,
 			AutoCompleteCommand,
 		},
@@ -170,10 +169,10 @@ func newAbraApp(version, commit string) *cli.App {
 	app.Before = func(c *cli.Context) error {
 		paths := []string{
 			config.ABRA_DIR,
-			path.Join(config.SERVERS_DIR),
-			path.Join(config.RECIPES_DIR),
-			path.Join(config.VENDOR_DIR),
-			path.Join(config.BACKUP_DIR),
+			config.SERVERS_DIR,
+			config.RECIPES_DIR,
+			config.VENDOR_DIR,
+			config.BACKUP_DIR,
 		}
 
 		for _, path := range paths {
@@ -185,6 +184,14 @@ func newAbraApp(version, commit string) *cli.App {
 			}
 		}
 
+		if _, err := os.Stat(config.CATALOGUE_DIR); os.IsNotExist(err) {
+			url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+			logrus.Warnf("local recipe catalogue is missing, retrieving now")
+			if err := git.Clone(config.CATALOGUE_DIR, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		logrus.Debugf("abra version %s, commit %s", version, commit)
 
 		return nil

cli/internal/cli.go

@@ -68,17 +68,6 @@ var TtyFlag = &cli.BoolFlag{
 	Destination: &Tty,
 }
 
-// DNSProvider specifies a DNS provider.
-var DNSProvider string
-
-// DNSProviderFlag selects a DNS provider.
-var DNSProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Value:       "",
-	Usage:       "DNS provider",
-	Destination: &DNSProvider,
-}
-
 var NoInput bool
 var NoInputFlag = &cli.BoolFlag{
 	Name:        "no-input, n",
@@ -86,163 +75,6 @@ var NoInputFlag = &cli.BoolFlag{
 	Destination: &NoInput,
 }
 
-var DNSType string
-
-var DNSTypeFlag = &cli.StringFlag{
-	Name:        "record-type, rt",
-	Value:       "",
-	Usage:       "Domain name record type (e.g. A)",
-	Destination: &DNSType,
-}
-
-var DNSName string
-
-var DNSNameFlag = &cli.StringFlag{
-	Name:        "record-name, rn",
-	Value:       "",
-	Usage:       "Domain name record name (e.g. mysubdomain)",
-	Destination: &DNSName,
-}
-
-var DNSValue string
-
-var DNSValueFlag = &cli.StringFlag{
-	Name:        "record-value, rv",
-	Value:       "",
-	Usage:       "Domain name record value (e.g. 192.168.1.1)",
-	Destination: &DNSValue,
-}
-
-var DNSTTL string
-var DNSTTLFlag = &cli.StringFlag{
-	Name:        "record-ttl, rl",
-	Value:       "600s",
-	Usage:       "Domain name TTL value (seconds)",
-	Destination: &DNSTTL,
-}
-
-var DNSPriority int
-
-var DNSPriorityFlag = &cli.IntFlag{
-	Name:        "record-priority, rp",
-	Value:       10,
-	Usage:       "Domain name priority value",
-	Destination: &DNSPriority,
-}
-
-var ServerProvider string
-
-var ServerProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Usage:       "3rd party server provider",
-	Destination: &ServerProvider,
-}
-
-var CapsulInstanceURL string
-
-var CapsulInstanceURLFlag = &cli.StringFlag{
-	Name:        "capsul-url, cu",
-	Value:       "yolo.servers.coop",
-	Usage:       "capsul instance URL",
-	Destination: &CapsulInstanceURL,
-}
-
-var CapsulName string
-
-var CapsulNameFlag = &cli.StringFlag{
-	Name:        "capsul-name, cn",
-	Value:       "",
-	Usage:       "capsul name",
-	Destination: &CapsulName,
-}
-
-var CapsulType string
-
-var CapsulTypeFlag = &cli.StringFlag{
-	Name:        "capsul-type, ct",
-	Value:       "f1-xs",
-	Usage:       "capsul type",
-	Destination: &CapsulType,
-}
-
-var CapsulImage string
-
-var CapsulImageFlag = &cli.StringFlag{
-	Name:        "capsul-image, ci",
-	Value:       "debian10",
-	Usage:       "capsul image",
-	Destination: &CapsulImage,
-}
-
-var CapsulSSHKeys cli.StringSlice
-var CapsulSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "capsul-ssh-keys, cs",
-	Usage: "capsul SSH key",
-	Value: &CapsulSSHKeys,
-}
-
-var CapsulAPIToken string
-
-var CapsulAPITokenFlag = &cli.StringFlag{
-	Name:        "capsul-token, ca",
-	Usage:       "capsul API token",
-	EnvVar:      "CAPSUL_TOKEN",
-	Destination: &CapsulAPIToken,
-}
-
-var HetznerCloudName string
-
-var HetznerCloudNameFlag = &cli.StringFlag{
-	Name:        "hetzner-name, hn",
-	Value:       "",
-	Usage:       "hetzner cloud name",
-	Destination: &HetznerCloudName,
-}
-
-var HetznerCloudType string
-
-var HetznerCloudTypeFlag = &cli.StringFlag{
-	Name:        "hetzner-type, ht",
-	Usage:       "hetzner cloud type",
-	Destination: &HetznerCloudType,
-	Value:       "cx11",
-}
-
-var HetznerCloudImage string
-
-var HetznerCloudImageFlag = &cli.StringFlag{
-	Name:        "hetzner-image, hi",
-	Usage:       "hetzner cloud image",
-	Value:       "debian-10",
-	Destination: &HetznerCloudImage,
-}
-
-var HetznerCloudSSHKeys cli.StringSlice
-
-var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "hetzner-ssh-keys, hs",
-	Usage: "hetzner cloud SSH keys (e.g. me@foo.com)",
-	Value: &HetznerCloudSSHKeys,
-}
-
-var HetznerCloudLocation string
-
-var HetznerCloudLocationFlag = &cli.StringFlag{
-	Name:        "hetzner-location, hl",
-	Usage:       "hetzner cloud server location",
-	Value:       "hel1",
-	Destination: &HetznerCloudLocation,
-}
-
-var HetznerCloudAPIToken string
-
-var HetznerCloudAPITokenFlag = &cli.StringFlag{
-	Name:        "hetzner-token, ha",
-	Usage:       "hetzner cloud API token",
-	EnvVar:      "HCLOUD_TOKEN",
-	Destination: &HetznerCloudAPIToken,
-}
-
 // Debug stores the variable from DebugFlag.
 var Debug bool
 

cli/internal/deploy.go

@@ -0,0 +1,173 @@
+package internal
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// NewVersionOverview shows an upgrade or downgrade overview
+func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
+	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
+	table := formatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
+	table.Render()
+
+	if releaseNotes != "" && newVersion != "" {
+		fmt.Println()
+		fmt.Print(releaseNotes)
+	} else {
+		logrus.Warnf("no release notes available for %s", newVersion)
+	}
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with deployment?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}
+
+// GetReleaseNotes prints release notes for a recipe version
+func GetReleaseNotes(recipeName, version string) (string, error) {
+	if version == "" {
+		return "", nil
+	}
+
+	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
+
+	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
+		releaseNotes, err := ioutil.ReadFile(fpath)
+		if err != nil {
+			return "", err
+		}
+		withTitle := fmt.Sprintf("%s release notes:\n%s", version, string(releaseNotes))
+		return withTitle, nil
+	}
+
+	return "", nil
+}
+
+// PostCmds parses a string of commands and executes them inside of the respective services
+// the commands string must have the following format:
+// "<service> <command> <arguments>|<service> <command> <arguments>|... "
+func PostCmds(cl *dockerClient.Client, app config.App, commands string) error {
+	abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
+	if _, err := os.Stat(abraSh); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf(fmt.Sprintf("%s does not exist for %s?", abraSh, app.Name))
+		}
+		return err
+	}
+
+	for _, command := range strings.Split(commands, "|") {
+		commandParts := strings.Split(command, " ")
+		if len(commandParts) < 2 {
+			return fmt.Errorf(fmt.Sprintf("not enough arguments: %s", command))
+		}
+		targetServiceName := commandParts[0]
+		cmdName := commandParts[1]
+		parsedCmdArgs := ""
+		if len(commandParts) > 2 {
+			parsedCmdArgs = fmt.Sprintf("%s ", strings.Join(commandParts[2:], " "))
+		}
+		logrus.Infof("running post-command '%s %s' in container %s", cmdName, parsedCmdArgs, targetServiceName)
+
+		if err := EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			return err
+		}
+
+		serviceNames, err := config.GetAppServiceNames(app.Name)
+		if err != nil {
+			return err
+		}
+
+		matchingServiceName := false
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
+			}
+		}
+
+		if !matchingServiceName {
+			return fmt.Errorf(fmt.Sprintf("no service %s for %s?", targetServiceName, app.Name))
+		}
+
+		logrus.Debugf("running command %s %s within the context of %s_%s", cmdName, parsedCmdArgs, app.StackName(), targetServiceName)
+
+		Tty = true
+		if err := RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DeployOverview shows a deployment overview
+func DeployOverview(app config.App, version, message string) error {
+	tableCol := []string{"server", "recipe", "config", "domain", "version"}
+	table := formatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, version})
+	table.Render()
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: message,
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}

cli/internal/validate.go

@@ -2,60 +2,24 @@ package internal
 
 import (
 	"errors"
-	"fmt"
-	"os"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
 // ValidateRecipe ensures the recipe arg is valid.
-func ValidateRecipe(c *cli.Context, conf *runtime.Config) recipe.Recipe {
-	recipeName := c.Args().First()
-
-	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
-	}
-
-	chosenRecipe, err := recipe.Get(recipeName, conf)
-	if err != nil {
-		if c.Command.Name == "generate" {
-			if strings.Contains(err.Error(), "missing a compose") {
-				logrus.Fatal(err)
-			}
-			logrus.Warn(err)
-		} else {
-			if strings.Contains(err.Error(), "template_driver is not allowed") {
-				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
-			}
-			logrus.Fatalf("unable to validate recipe: %s", err)
-		}
-	}
-
-	if err := recipe.EnsureLatest(recipeName, conf); err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("validated %s as recipe argument", recipeName)
-
-	return chosenRecipe
-}
-
-// ValidateRecipeWithPrompt ensures a recipe argument is present before
-// validating, asking for input if required.
-func ValidateRecipeWithPrompt(c *cli.Context, conf *runtime.Config) recipe.Recipe {
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
 
 	if recipeName == "" && !NoInput {
 		var recipes []string
 
-		catl, err := recipe.ReadRecipeCatalogue(conf)
+		catl, err := recipe.ReadRecipeCatalogue(Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -93,13 +57,19 @@ func ValidateRecipeWithPrompt(c *cli.Context, conf *runtime.Config) recipe.Recip
 		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName, conf)
+	chosenRecipe, err := recipe.Get(recipeName, Offline)
 	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := recipe.EnsureLatest(recipeName, conf); err != nil {
-		logrus.Fatal(err)
+		if c.Command.Name == "generate" {
+			if strings.Contains(err.Error(), "missing a compose") {
+				logrus.Fatal(err)
+			}
+			logrus.Warn(err)
+		} else {
+			if strings.Contains(err.Error(), "template_driver is not allowed") {
+				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
+			}
+			logrus.Fatalf("unable to validate recipe: %s", err)
+		}
 	}
 
 	logrus.Debugf("validated %s as recipe argument", recipeName)
@@ -108,7 +78,7 @@ func ValidateRecipeWithPrompt(c *cli.Context, conf *runtime.Config) recipe.Recip
 }
 
 // ValidateApp ensures the app name arg is valid.
-func ValidateApp(c *cli.Context, conf *runtime.Config) config.App {
+func ValidateApp(c *cli.Context) config.App {
 	appName := c.Args().First()
 
 	if appName == "" {
@@ -120,10 +90,6 @@ func ValidateApp(c *cli.Context, conf *runtime.Config) config.App {
 		logrus.Fatal(err)
 	}
 
-	if err := recipe.EnsureExists(app.Recipe, conf); err != nil {
-		logrus.Fatal(err)
-	}
-
 	logrus.Debugf("validated %s as app argument", appName)
 
 	return app
@@ -192,309 +158,15 @@ func ValidateServer(c *cli.Context) string {
 		}
 	}
 
-	if !matched {
-		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
-	}
-
 	if serverName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
 	}
 
+	if !matched {
+		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
+	}
+
 	logrus.Debugf("validated %s as server argument", serverName)
 
 	return serverName
 }
-
-// EnsureDNSProvider ensures a DNS provider is chosen.
-func EnsureDNSProvider() error {
-	if DNSProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select DNS provider",
-			Options: []string{"gandi"},
-		}
-
-		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
-			return err
-		}
-	}
-
-	if DNSProvider == "" {
-		return fmt.Errorf("missing DNS provider?")
-	}
-
-	return nil
-}
-
-// EnsureDNSTypeFlag ensures a DNS type flag is present.
-func EnsureDNSTypeFlag(c *cli.Context) error {
-	if DNSType == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record type",
-			Default: "A",
-		}
-		if err := survey.AskOne(prompt, &DNSType); err != nil {
-			return err
-		}
-	}
-
-	if DNSType == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSNameFlag ensures a DNS name flag is present.
-func EnsureDNSNameFlag(c *cli.Context) error {
-	if DNSName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record name",
-			Default: "mysubdomain",
-		}
-		if err := survey.AskOne(prompt, &DNSName); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSValueFlag ensures a DNS value flag is present.
-func EnsureDNSValueFlag(c *cli.Context) error {
-	if DNSValue == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record value",
-			Default: "192.168.1.2",
-		}
-		if err := survey.AskOne(prompt, &DNSValue); err != nil {
-			return err
-		}
-	}
-
-	if DNSValue == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
-	}
-
-	return nil
-}
-
-// EnsureZoneArgument ensures a zone argument is present.
-func EnsureZoneArgument(c *cli.Context) (string, error) {
-	zone := c.Args().First()
-
-	if zone == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name zone",
-			Default: "example.com",
-		}
-
-		if err := survey.AskOne(prompt, &zone); err != nil {
-			return zone, err
-		}
-	}
-
-	if zone == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
-	}
-
-	return zone, nil
-}
-
-// EnsureServerProvider ensures a 3rd party server provider is chosen.
-func EnsureServerProvider() error {
-	if ServerProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select server provider",
-			Options: []string{"capsul", "hetzner-cloud"},
-		}
-
-		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
-			return err
-		}
-	}
-
-	if ServerProvider == "" {
-		return fmt.Errorf("missing server provider?")
-	}
-
-	return nil
-}
-
-// EnsureNewCapsulVPSFlags ensure all flags are present.
-func EnsureNewCapsulVPSFlags(c *cli.Context) error {
-	if CapsulName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul name",
-		}
-		if err := survey.AskOne(prompt, &CapsulName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul instance URL",
-			Default: CapsulInstanceURL,
-		}
-		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul type",
-			Default: CapsulType,
-		}
-		if err := survey.AskOne(prompt, &CapsulType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul image",
-			Default: CapsulImage,
-		}
-		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
-			return err
-		}
-	}
-
-	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify capsul SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
-			return err
-		}
-		CapsulSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if CapsulAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("CAPSUL_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify capsul API token",
-			}
-			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
-				return err
-			}
-		} else {
-			CapsulAPIToken = token
-		}
-	}
-
-	if CapsulName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
-	}
-	if CapsulInstanceURL == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
-	}
-	if CapsulType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
-	}
-	if CapsulImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
-	}
-	if len(CapsulSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
-	}
-	if CapsulAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
-	}
-
-	return nil
-}
-
-// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
-func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
-	if HetznerCloudName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS type",
-			Default: HetznerCloudType,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS image",
-			Default: HetznerCloudImage,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
-			return err
-		}
-	}
-
-	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &sshKeys); err != nil {
-			return err
-		}
-		HetznerCloudSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS location",
-			Default: HetznerCloudLocation,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
-			return err
-		}
-	}
-
-	if HetznerCloudAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			HetznerCloudAPIToken = token
-		}
-	}
-
-	if HetznerCloudName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
-	}
-	if HetznerCloudType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
-	}
-	if HetznerCloudImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
-	}
-	if HetznerCloudLocation == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
-	}
-	if HetznerCloudAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
-	}
-
-	return nil
-}

cli/recipe/fetch.go

@@ -4,38 +4,38 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
 var recipeFetchCommand = cli.Command{
 	Name:        "fetch",
-	Usage:       "Fetch recipe local copies",
+	Usage:       "Fetch recipe(s)",
 	Aliases:     []string{"f"},
 	ArgsUsage:   "[<recipe>]",
-	Description: "Fetchs all recipes without arguments.",
+	Description: "Retrieves all recipes if no <recipe> argument is passed",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
-		internal.OfflineFlag,
+		internal.NoInputFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
 		recipeName := c.Args().First()
 
 		if recipeName != "" {
-			internal.ValidateRecipe(c, conf)
-			return nil // ValidateRecipe ensures latest checkout
+			internal.ValidateRecipe(c)
 		}
 
-		repos, err := recipe.ReadReposMetadata(conf)
-		if err != nil {
+		if err := recipe.EnsureExists(recipeName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := recipe.UpdateRepositories(repos, recipeName, conf); err != nil {
+		if err := recipe.EnsureUpToDate(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.EnsureLatest(recipeName); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/recipe/lint.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/lint"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -22,17 +21,34 @@ var recipeLintCommand = cli.Command{
 		internal.DebugFlag,
 		internal.OnlyErrorFlag,
 		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		recipe := internal.ValidateRecipe(c, conf)
+		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name, conf); err != nil {
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		tableCol := []string{"ref", "rule", "severity", "satisfied", "skipped", "resolve"}
 		table := formatter.CreateTable(tableCol)
 

cli/recipe/list.go

@@ -9,7 +9,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -34,9 +33,7 @@ var recipeListCommand = cli.Command{
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-
-		catl, err := recipe.ReadRecipeCatalogue(conf)
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err.Error())
 		}

cli/recipe/release.go

@@ -13,7 +13,6 @@ import (
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
@@ -60,12 +59,7 @@ your SSH keys configured on your account.
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(
-			runtime.WithOffline(internal.Offline),
-			runtime.WithEnsureRecipeUpToDate(false),
-		)
-
-		recipe := internal.ValidateRecipeWithPrompt(c, conf)
+		recipe := internal.ValidateRecipe(c)
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {

cli/recipe/sync.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -29,7 +28,6 @@ var recipeSyncCommand = cli.Command{
 		internal.MajorFlag,
 		internal.MinorFlag,
 		internal.PatchFlag,
-		internal.OfflineFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -42,13 +40,9 @@ Where <version> can be specifed on the command-line or Abra can attempt to
 auto-generate it for you. The <recipe> configuration will be updated on the
 local file system.
 `,
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(
-			runtime.WithOffline(internal.Offline),
-			runtime.WithEnsureRecipeUpToDate(false),
-		)
-
-		recipe := internal.ValidateRecipeWithPrompt(c, conf)
+		recipe := internal.ValidateRecipe(c)
 
 		mainApp, err := internal.GetMainAppImage(recipe)
 		if err != nil {
@@ -70,6 +64,9 @@ local file system.
 		nextTag := c.Args().Get(1)
 		if len(tags) == 0 && nextTag == "" {
 			logrus.Warnf("no git tags found for %s", recipe.Name)
+			if internal.NoInput {
+				logrus.Fatalf("unable to continue, input required for initial version")
+			}
 			fmt.Println(fmt.Sprintf(`
 The following options are two types of initial semantic version that you can
 pick for %s that will be published in the recipe catalogue. This follows the
@@ -203,5 +200,4 @@ likely to change.
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }

cli/recipe/upgrade.go

@@ -15,7 +15,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
@@ -58,8 +57,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 
     abra recipe upgrade
 `,
-	BashComplete: autocomplete.RecipeNameComplete,
-	ArgsUsage:    "<recipe>",
+	ArgsUsage: "<recipe>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
@@ -68,14 +66,25 @@ You may invoke this command in "wizard" mode and be prompted for input:
 		internal.MajorFlag,
 		internal.MachineReadableFlag,
 		internal.AllTagsFlag,
-		internal.OfflineFlag,
 	},
-	Before: internal.SubCommandBefore,
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-		recipe := internal.ValidateRecipeWithPrompt(c, conf)
+		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name, conf); err != nil {
+		if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -185,7 +194,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				continue // skip on to the next tag and don't update any compose files
 			}
 
-			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, conf)
+			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, internal.Offline)
 			if err != nil {
 				logrus.Fatal(err)
 			}

cli/recipe/version.go

@@ -1,63 +1,85 @@
 package recipe
 
 import (
+	"fmt"
+	"sort"
+
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/formatter"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
+	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
 var recipeVersionCommand = cli.Command{
-	Name:      "versions",
-	Aliases:   []string{"v"},
-	Usage:     "List recipe versions",
-	ArgsUsage: "<recipe>",
+	Name:        "versions",
+	Aliases:     []string{"v"},
+	Usage:       "List recipe versions",
+	ArgsUsage:   "<recipe>",
+	Description: "Versions are read from the recipe catalogue.",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.MachineReadableFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(
-			runtime.WithOffline(internal.Offline),
-			runtime.WithEnsureRecipeUpToDate(false),
-		)
+		recipe := internal.ValidateRecipe(c)
 
-		recipe := internal.ValidateRecipe(c, conf)
-
-		catalogue, err := recipePkg.ReadRecipeCatalogue(conf)
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		recipeMeta, ok := catalogue[recipe.Name]
+		recipeMeta, ok := catl[recipe.Name]
 		if !ok {
-			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
+			logrus.Fatalf("%s is not published on the catalogue?", recipe.Name)
 		}
 
-		tableCol := []string{"Version", "Service", "Image", "Tag"}
-		table := formatter.CreateTable(tableCol)
+		if len(recipeMeta.Versions) == 0 {
+			logrus.Fatalf("%s has no catalogue published versions?", recipe.Name)
+		}
 
 		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
-			for tag, meta := range recipeMeta.Versions[i] {
+			tableCols := []string{"version", "service", "image", "tag"}
+			table := formatter.CreateTable(tableCols)
+			for version, meta := range recipeMeta.Versions[i] {
+				var versions [][]string
 				for service, serviceMeta := range meta {
-					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag})
+					versions = append(versions, []string{version, service, serviceMeta.Image, serviceMeta.Tag})
+				}
+
+				sort.Slice(versions, sortServiceByName(versions))
+
+				for _, version := range versions {
+					table.Append(version)
+				}
+
+				if internal.MachineReadable {
+					table.JSONRender()
+				} else {
+					table.SetAutoMergeCellsByColumnIndex([]int{0})
+					table.SetAlignment(tablewriter.ALIGN_LEFT)
+					table.Render()
+					fmt.Println()
 				}
 			}
 		}
 
-		table.SetAutoMergeCells(true)
-
-		if table.NumLines() > 0 {
-			table.Render()
-		} else {
-			logrus.Fatalf("%s has no published versions?", recipe.Name)
-		}
-
 		return nil
 	},
 }

cli/record/list.go

@@ -1,83 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordListCommand lists domains.
-var RecordListCommand = cli.Command{
-	Name:      "list",
-	Usage:     "List domain name records",
-	Aliases:   []string{"ls"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.DNSProviderFlag,
-		internal.OfflineFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-List all domain name records managed by a 3rd party provider for a specific
-zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-`,
-	Action: func(c *cli.Context) error {
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		for _, record := range records {
-			value := record.Value
-			if len(record.Value) > 30 {
-				value = fmt.Sprintf("%s...", record.Value[:30])
-			}
-
-			table.Append([]string{
-				record.Type,
-				record.Name,
-				value,
-				record.TTL.String(),
-				strconv.Itoa(record.Priority),
-			})
-		}
-
-		table.Render()
-
-		return nil
-	},
-}

cli/record/new.go

@@ -1,149 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/dns"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordNewCommand creates a new domain name record.
-var RecordNewCommand = cli.Command{
-	Name:      "new",
-	Usage:     "Create a new domain record",
-	Aliases:   []string{"n"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-		internal.DNSValueFlag,
-		internal.DNSTTLFlag,
-		internal.DNSPriorityFlag,
-		internal.OfflineFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Create a new domain name record for a specific zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-
-Example:
-
-    abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSValueFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		ttl, err := dns.GetTTL(internal.DNSTTL)
-		if err != nil {
-			return err
-		}
-
-		record := libdns.Record{
-			Type:  internal.DNSType,
-			Name:  internal.DNSName,
-			Value: internal.DNSValue,
-			TTL:   ttl,
-		}
-
-		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
-			record.Priority = internal.DNSPriority
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, existingRecord := range records {
-			if existingRecord.Type == record.Type &&
-				existingRecord.Name == record.Name &&
-				existingRecord.Value == record.Value {
-				logrus.Fatalf("%s record for %s already exists?", record.Type, zone)
-			}
-		}
-
-		createdRecords, err := provider.SetRecords(
-			context.Background(),
-			zone,
-			[]libdns.Record{record},
-		)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(createdRecords) == 0 {
-			logrus.Fatal("provider library reports that no record was created?")
-		}
-
-		createdRecord := createdRecords[0]
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := createdRecord.Value
-		if len(createdRecord.Value) > 30 {
-			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
-		}
-
-		table.Append([]string{
-			createdRecord.Type,
-			createdRecord.Name,
-			value,
-			createdRecord.TTL.String(),
-			strconv.Itoa(createdRecord.Priority),
-		})
-
-		table.Render()
-
-		logrus.Info("record created")
-
-		return nil
-	},
-}

cli/record/record.go

@@ -1,37 +0,0 @@
-package record
-
-import (
-	"github.com/urfave/cli"
-)
-
-// RecordCommand supports managing DNS entries.
-var RecordCommand = cli.Command{
-	Name:      "record",
-	Usage:     "Manage domain name records",
-	Aliases:   []string{"rc"},
-	ArgsUsage: "<record>",
-	Description: `
-Manage domain name records via 3rd party providers such as Gandi DNS. It
-supports listing, creating and removing all types of records that you might
-need for managing Co-op Cloud apps.
-
-The following providers are supported:
-
-    Gandi DNS https://www.gandi.net
-
-You need an account with such a provider already. Typically, you need to
-provide an API token on the Abra command-line when using these commands so that
-you can authenticate with your provider account.
-
-New providers can be integrated, we welcome change sets. See the underlying DNS
-library documentation for more. It supports many existing providers and allows
-to implement new provider support easily.
-
-    https://pkg.go.dev/github.com/libdns/libdns
-`,
-	Subcommands: []cli.Command{
-		RecordListCommand,
-		RecordNewCommand,
-		RecordRemoveCommand,
-	},
-}

cli/record/remove.go

@@ -1,137 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordRemoveCommand lists domains.
-var RecordRemoveCommand = cli.Command{
-	Name:      "remove",
-	Usage:     "Remove a domain name record",
-	Aliases:   []string{"rm"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-		internal.OfflineFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Remove a domain name record for a specific zone.
-
-It uses the type of record and name to match existing records and choose one
-for deletion. You must specify a zone (e.g. example.com) under which your
-domain name records are listed. This zone must already be created on your
-provider account.
-
-Example:
-
-    abra record remove foo.com -p gandi -t A -n myapp
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record rm
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var toDelete libdns.Record
-		for _, record := range records {
-			if record.Type == internal.DNSType && record.Name == internal.DNSName {
-				toDelete = record
-				break
-			}
-		}
-
-		if (libdns.Record{}) == toDelete {
-			logrus.Fatal("provider library reports no matching record?")
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := toDelete.Value
-		if len(toDelete.Value) > 30 {
-			value = fmt.Sprintf("%s...", toDelete.Value[:30])
-		}
-
-		table.Append([]string{
-			toDelete.Type,
-			toDelete.Name,
-			value,
-			toDelete.TTL.String(),
-			strconv.Itoa(toDelete.Priority),
-		})
-
-		table.Render()
-
-		if !internal.NoInput {
-			response := false
-			prompt := &survey.Confirm{
-				Message: "continue with record deletion?",
-			}
-
-			if err := survey.AskOne(prompt, &response); err != nil {
-				return err
-			}
-
-			if !response {
-				logrus.Fatal("exiting as requested")
-			}
-		}
-
-		_, err = provider.DeleteRecords(context.Background(), zone, []libdns.Record{toDelete})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Info("record successfully deleted")
-
-		return nil
-	},
-}

cli/server/add.go

@@ -118,7 +118,6 @@ developer machine.
 		internal.DebugFlag,
 		internal.NoInputFlag,
 		localFlag,
-		internal.OfflineFlag,
 	},
 	Before:    internal.SubCommandBefore,
 	ArgsUsage: "<domain>",

cli/server/list.go

@@ -60,6 +60,17 @@ var serverListCommand = cli.Command{
 					if err != nil {
 						logrus.Fatal(err)
 					}
+
+					if sp.Host == "" {
+						sp.Host = "unknown"
+					}
+					if sp.User == "" {
+						sp.User = "unknown"
+					}
+					if sp.Port == "" {
+						sp.Port = "unknown"
+					}
+
 					row = []string{serverName, sp.Host, sp.User, sp.Port}
 				}
 			}
@@ -73,8 +84,11 @@ var serverListCommand = cli.Command{
 			}
 
 			if problemsFilter {
-				if row[1] == "unknown" {
-					table.Append(row)
+				for _, val := range row {
+					if val == "unknown" {
+						table.Append(row)
+						break
+					}
 				}
 			} else {
 				table.Append(row)

cli/server/new.go

@@ -1,261 +0,0 @@
-package server
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/libcapsul"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-func newHetznerCloudVPS(c *cli.Context) error {
-	if err := internal.EnsureNewHetznerCloudVPSFlags(c); err != nil {
-		return err
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	var sshKeysRaw []string
-	var sshKeys []*hcloud.SSHKey
-	for _, sshKey := range c.StringSlice("hetzner-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-
-		sshKey, _, err := client.SSHKey.GetByName(context.Background(), sshKey)
-		if err != nil {
-			return err
-		}
-		sshKeys = append(sshKeys, sshKey)
-		sshKeysRaw = append(sshKeysRaw, sshKey.Name)
-	}
-
-	serverOpts := hcloud.ServerCreateOpts{
-		Name:       internal.HetznerCloudName,
-		ServerType: &hcloud.ServerType{Name: internal.HetznerCloudType},
-		Image:      &hcloud.Image{Name: internal.HetznerCloudImage},
-		SSHKeys:    sshKeys,
-		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
-	}
-
-	sshKeyIDs := strings.Join(sshKeysRaw, "\n")
-	if sshKeyIDs == "" {
-		sshKeyIDs = "N/A (password auth)"
-	}
-
-	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.HetznerCloudName,
-		internal.HetznerCloudType,
-		internal.HetznerCloudImage,
-		sshKeyIDs,
-		internal.HetznerCloudLocation,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	res, _, err := client.Server.Create(context.Background(), serverOpts)
-	if err != nil {
-		return err
-	}
-
-	var rootPassword string
-	if len(sshKeys) > 0 {
-		rootPassword = "N/A (using SSH keys)"
-	} else {
-		rootPassword = res.RootPassword
-	}
-
-	ip := res.Server.PublicNet.IPv4.IP.String()
-
-	fmt.Println(fmt.Sprintf(`
-Your new Hetzner Cloud VPS has successfully been created! Here are the details:
-
-    name: %s
-    IP address: %s
-    root password: %s
-
-You can access this new VPS via SSH using the following command:
-
-    ssh root@%s
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record (manually
-or by using "abra record new") and add the server to your Abra configuration
-("abra server add") to have a working server that you can deploy Co-op Cloud
-apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A %s
-    *  1800 IN A %s
-	`,
-		internal.HetznerCloudName, ip, rootPassword,
-		ip, ip, ip,
-	))
-
-	return nil
-}
-
-func newCapsulVPS(c *cli.Context) error {
-	if err := internal.EnsureNewCapsulVPSFlags(c); err != nil {
-		return err
-	}
-
-	capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", internal.CapsulInstanceURL)
-
-	var sshKeys []string
-	for _, sshKey := range c.StringSlice("capsul-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-		sshKeys = append(sshKeys, sshKey)
-	}
-
-	tableColumns := []string{"instance", "name", "type", "image", "ssh-keys"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.CapsulInstanceURL,
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		strings.Join(sshKeys, "\n"),
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with capsul creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	capsulClient := libcapsul.New(capsulCreateURL, internal.CapsulAPIToken)
-	resp, err := capsulClient.Create(
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		sshKeys,
-	)
-	if err != nil {
-		return err
-	}
-
-	fmt.Println(fmt.Sprintf(`
-Your new Capsul has successfully been created! Here are the details:
-
-    Capsul name: %s
-    Capsul ID:   %v
-
-You will need to log into your Capsul instance web interface to retrieve the IP
-address. You can learn all about how to get SSH access to your new Capsul on:
-
-    %s/about-ssh
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record (manually
-or by using "abra record new") and add the server to your Abra configuration
-("abra server add") to have a working server that you can deploy Co-op Cloud
-apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A <your-capsul-ip>
-    *  1800 IN A <your-capsul-ip>
-	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
-
-	return nil
-}
-
-var serverNewCommand = cli.Command{
-	Name:    "new",
-	Aliases: []string{"n"},
-	Usage:   "Create a new server using a 3rd party provider",
-	Description: `
-Create a new server via a 3rd party provider.
-
-The following providers are supported:
-
-		Capsul https://git.cyberia.club/Cyberia/capsul-flask
-    Hetzner Cloud https://docs.hetzner.com/cloud
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-
-API tokens are read from the environment if specified, e.g.
-
-    export HCLOUD_TOKEN=...
-`,
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.ServerProviderFlag,
-		internal.OfflineFlag,
-
-		// Capsul
-		internal.CapsulInstanceURLFlag,
-		internal.CapsulTypeFlag,
-		internal.CapsulImageFlag,
-		internal.CapsulSSHKeysFlag,
-		internal.CapsulAPITokenFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudTypeFlag,
-		internal.HetznerCloudImageFlag,
-		internal.HetznerCloudSSHKeysFlag,
-		internal.HetznerCloudLocationFlag,
-		internal.HetznerCloudAPITokenFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		if err := internal.EnsureServerProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		switch internal.ServerProvider {
-		case "capsul":
-			if err := newCapsulVPS(c); err != nil {
-				logrus.Fatal(err)
-			}
-		case "hetzner-cloud":
-			if err := newHetznerCloudVPS(c); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		return nil
-	},
-}

cli/server/prune.go

@@ -20,12 +20,12 @@ var allFilterFlag = &cli.BoolFlag{
 	Destination: &allFilter,
 }
 
-var volunesFilter bool
+var volumesFilter bool
 
 var volumesFilterFlag = &cli.BoolFlag{
 	Name:        "volumes, v",
 	Usage:       "Prune volumes. This will remove app data, Be Careful!",
-	Destination: &volunesFilter,
+	Destination: &volumesFilter,
 }
 
 var serverPruneCommand = cli.Command{
@@ -35,7 +35,7 @@ var serverPruneCommand = cli.Command{
 	Description: `
 Prunes unused containers, networks, and dangling images.
 
-If passing "-v/--volumes" then volumes not connected with a deployed app will
+If passing "-v/--volumes" then volumes not connected to a deployed app will
 also be removed. This can result in unwanted data loss if not used carefully.
 	`,
 	ArgsUsage: "[<server>]",
@@ -44,12 +44,11 @@ also be removed. This can result in unwanted data loss if not used carefully.
 		volumesFilterFlag,
 		internal.DebugFlag,
 		internal.OfflineFlag,
+		internal.NoInputFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.ServerNameComplete,
 	Action: func(c *cli.Context) error {
-		var args filters.Args
-
 		serverName := internal.ValidateServer(c)
 
 		cl, err := client.New(serverName)
@@ -57,6 +56,8 @@ also be removed. This can result in unwanted data loss if not used carefully.
 			logrus.Fatal(err)
 		}
 
+		var args filters.Args
+
 		ctx := context.Background()
 		cr, err := cl.ContainersPrune(ctx, args)
 		if err != nil {
@@ -75,6 +76,7 @@ also be removed. This can result in unwanted data loss if not used carefully.
 
 		pruneFilters := filters.NewArgs()
 		if allFilter {
+			logrus.Debugf("removing all images, not only dangling ones")
 			pruneFilters.Add("dangling", "false")
 		}
 
@@ -86,7 +88,7 @@ also be removed. This can result in unwanted data loss if not used carefully.
 		imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
 		logrus.Infof("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed)
 
-		if volunesFilter {
+		if volumesFilter {
 			vr, err := cl.VolumesPrune(ctx, args)
 			if err != nil {
 				logrus.Fatal(err)

cli/server/remove.go

@@ -1,8 +1,6 @@
 package server
 
 import (
-	"context"
-	"fmt"
 	"os"
 	"path/filepath"
 
@@ -10,168 +8,31 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var rmServer bool
-var rmServerFlag = &cli.BoolFlag{
-	Name:        "server, s",
-	Usage:       "remove the actual server also",
-	Destination: &rmServer,
-}
-
-func rmHetznerCloudVPS(c *cli.Context) error {
-	if internal.HetznerCloudName == "" && !internal.NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &internal.HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if internal.HetznerCloudAPIToken == "" && !internal.NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &internal.HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			internal.HetznerCloudAPIToken = token
-		}
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	server, _, err := client.Server.Get(context.Background(), internal.HetznerCloudName)
-	if err != nil {
-		return err
-	}
-
-	if server == nil {
-		logrus.Fatalf("library provider reports that %s doesn't exist?", internal.HetznerCloudName)
-	}
-
-	fmt.Println(fmt.Sprintf(`
-You have requested that Abra delete the following server (%s). Please be
-absolutely sure that this is indeed the server that you would like to have
-removed. There will be no going back once you confirm, the server will be
-destroyed.
-`, server.Name))
-
-	tableColumns := []string{"name", "type", "image", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		server.Name,
-		server.ServerType.Name,
-		server.Image.Name,
-		server.Datacenter.Name,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS removal?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	_, err = client.Server.Delete(context.Background(), server)
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof("%s has been deleted from your hetzner cloud account", internal.HetznerCloudName)
-
-	return nil
-}
-
 var serverRemoveCommand = cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
-	ArgsUsage: "[<server>]",
+	ArgsUsage: "<server>",
 	Usage:     "Remove a managed server",
-	Description: `
-Remova a server from Abra management.
+	Description: `Remove a managed server.
 
-Depending on whether you used a 3rd party provider to create this server ("abra
-server new"), you can also destroy the virtual server as well. Pass
-"--server/-s" to tell Abra to try to delete this VPS.
-
-Otherwise, Abra will remove the internal bookkeeping (~/.abra/servers/...) and
-underlying client connection context. This server will then be lost in time,
-like tears in rain.
+Abra will remove the internal bookkeeping (~/.abra/servers/...) and underlying
+client connection context. This server will then be lost in time, like tears in
+rain.
 `,
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		rmServerFlag,
-		internal.ServerProviderFlag,
 		internal.OfflineFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudAPITokenFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.ServerNameComplete,
 	Action: func(c *cli.Context) error {
 		serverName := internal.ValidateServer(c)
 
-		warnMsg := `Did not pass -s/--server for actual server deletion, prompting!
-
-Abra doesn't currently know if it helped you create this server with one of the
-3rd party integrations (e.g. Capsul). You have a choice here to actually,
-really and finally destroy this server using those integrations. If you want to
-do this, choose Yes.
-
-If you just want to remove the server config files & context, choose No.
-`
-
-		if !rmServer {
-			logrus.Warn(fmt.Sprintf(warnMsg))
-
-			response := false
-			prompt := &survey.Confirm{
-				Message: "delete actual live server?",
-			}
-			if err := survey.AskOne(prompt, &response); err != nil {
-				logrus.Fatal(err)
-			}
-			if response {
-				logrus.Info("setting -s/--server and attempting to remove actual server")
-				rmServer = true
-			}
-		}
-
-		if rmServer {
-			if err := internal.EnsureServerProvider(); err != nil {
-				logrus.Fatal(err)
-			}
-
-			switch internal.ServerProvider {
-			case "capsul":
-				logrus.Warn("capsul provider does not support automatic removal yet, sorry!")
-			case "hetzner-cloud":
-				if err := rmHetznerCloudVPS(c); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
 		if err := client.DeleteContext(serverName); err != nil {
 			logrus.Fatal(err)
 		}

cli/server/server.go

@@ -9,16 +9,7 @@ var ServerCommand = cli.Command{
 	Name:    "server",
 	Aliases: []string{"s"},
 	Usage:   "Manage servers",
-	Description: `
-Create, manage and remove servers using 3rd party integrations. 
-
-Servers can be created from scratch using the "abra server new" command. If you
-already have a server, you can add it to your configuration using "abra server
-add". Abra can provision servers so that they are ready to deploy Co-op Cloud
-recipes, see available flags on "abra server add" for more.
-`,
 	Subcommands: []cli.Command{
-		serverNewCommand,
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,

cli/updater/updater.go

@@ -12,7 +12,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
@@ -58,8 +57,6 @@ catalogue. If a new patch/minor version is available, a notification is
 printed. To include major versions use the --major flag.
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-
 		cl, err := client.New("default")
 		if err != nil {
 			logrus.Fatal(err)
@@ -78,7 +75,7 @@ printed. To include major versions use the --major flag.
 			}
 
 			if recipeName != "" {
-				_, err = getLatestUpgrade(cl, stackName, recipeName, conf)
+				_, err = getLatestUpgrade(cl, stackName, recipeName)
 				if err != nil {
 					logrus.Fatal(err)
 				}
@@ -113,8 +110,6 @@ break things. Only apps that are not deployed with "--chaos" are upgraded, to
 update chaos deployments use the "--chaos" flag. Use it with care.
 `,
 	Action: func(c *cli.Context) error {
-		conf := runtime.New(runtime.WithOffline(internal.Offline))
-
 		cl, err := client.New("default")
 		if err != nil {
 			logrus.Fatal(err)
@@ -123,7 +118,7 @@ update chaos deployments use the "--chaos" flag. Use it with care.
 		if !updateAll {
 			stackName := c.Args().Get(0)
 			recipeName := c.Args().Get(1)
-			err = tryUpgrade(cl, stackName, recipeName, conf)
+			err = tryUpgrade(cl, stackName, recipeName)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -143,7 +138,7 @@ update chaos deployments use the "--chaos" flag. Use it with care.
 				logrus.Fatal(err)
 			}
 
-			err = tryUpgrade(cl, stackName, recipeName, conf)
+			err = tryUpgrade(cl, stackName, recipeName)
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -227,14 +222,13 @@ func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) {
 
 // getLatestUpgrade returns the latest available version for an app respecting
 // the "--major" flag if it is newer than the currently deployed version.
-func getLatestUpgrade(cl *dockerclient.Client, stackName string,
-	recipeName string, conf *runtime.Config) (string, error) {
+func getLatestUpgrade(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
 	deployedVersion, err := getDeployedVersion(cl, stackName, recipeName)
 	if err != nil {
 		return "", err
 	}
 
-	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion, conf)
+	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion)
 	if err != nil {
 		return "", err
 	}
@@ -277,8 +271,8 @@ func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName st
 // than the deployed version. It only includes major upgrades if the "--major"
 // flag is set.
 func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
-	deployedVersion string, conf *runtime.Config) ([]string, error) {
-	catl, err := recipe.ReadRecipeCatalogue(conf)
+	deployedVersion string) ([]string, error) {
+	catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 	if err != nil {
 		return nil, err
 	}
@@ -322,12 +316,12 @@ func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName
 
 // processRecipeRepoVersion clones, pulls, checks out the version and lints the
 // recipe repository.
-func processRecipeRepoVersion(recipeName, version string, conf *runtime.Config) error {
-	if err := recipe.EnsureExists(recipeName, conf); err != nil {
+func processRecipeRepoVersion(recipeName, version string) error {
+	if err := recipe.EnsureExists(recipeName); err != nil {
 		return err
 	}
 
-	if err := recipe.EnsureUpToDate(recipeName, conf); err != nil {
+	if err := recipe.EnsureUpToDate(recipeName); err != nil {
 		return err
 	}
 
@@ -335,7 +329,7 @@ func processRecipeRepoVersion(recipeName, version string, conf *runtime.Config)
 		return err
 	}
 
-	if r, err := recipe.Get(recipeName, conf); err != nil {
+	if r, err := recipe.Get(recipeName, internal.Offline); err != nil {
 		return err
 	} else if err := lint.LintForErrors(r); err != nil {
 		return err
@@ -392,7 +386,7 @@ func createDeployConfig(recipeName string, stackName string, env config.AppEnv)
 }
 
 // tryUpgrade performs the upgrade if all the requirements are fulfilled.
-func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *runtime.Config) error {
+func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string) error {
 	if recipeName == "" {
 		logrus.Debugf("don't update %s due to missing recipe name", stackName)
 		return nil
@@ -418,7 +412,7 @@ func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *run
 		return nil
 	}
 
-	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName, conf)
+	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName)
 	if err != nil {
 		return err
 	}
@@ -428,14 +422,14 @@ func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *run
 		return nil
 	}
 
-	err = upgrade(cl, stackName, recipeName, upgradeVersion, conf)
+	err = upgrade(cl, stackName, recipeName, upgradeVersion)
 
 	return err
 }
 
 // upgrade performs all necessary steps to upgrade an app.
 func upgrade(cl *dockerclient.Client, stackName, recipeName,
-	upgradeVersion string, conf *runtime.Config) error {
+	upgradeVersion string) error {
 	env, err := getEnv(cl, stackName)
 	if err != nil {
 		return err
@@ -448,7 +442,7 @@ func upgrade(cl *dockerclient.Client, stackName, recipeName,
 		Env:    env,
 	}
 
-	if err = processRecipeRepoVersion(recipeName, upgradeVersion, conf); err != nil {
+	if err = processRecipeRepoVersion(recipeName, upgradeVersion); err != nil {
 		return err
 	}
 

go.mod

@@ -7,22 +7,21 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v24.0.5+incompatible
+	github.com/docker/cli v24.0.6+incompatible
 	github.com/docker/distribution v2.8.2+incompatible
-	github.com/docker/docker v24.0.5+incompatible
+	github.com/docker/docker v24.0.6+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.8.1
+	github.com/go-git/go-git/v5 v5.9.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/sirupsen/logrus v1.9.3
-	gotest.tools/v3 v3.5.0
+	gotest.tools/v3 v3.5.1
 )
 
 require (
-	coopcloud.tech/libcapsul v0.0.0-20230605070824-878af473f07b
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/buger/goterm v1.0.4
 	github.com/containerd/containerd v1.5.9 // indirect
@@ -35,18 +34,16 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.4
-	github.com/hetznercloud/hcloud-go v1.48.0
 	github.com/klauspost/pgzip v1.2.6
-	github.com/libdns/gandi v1.0.2
-	github.com/libdns/libdns v0.2.1
 	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/stretchr/testify v1.8.4
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/sys v0.10.0
+	golang.org/x/sys v0.12.0
 )

go.sum

@@ -47,8 +47,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-coopcloud.tech/libcapsul v0.0.0-20230605070824-878af473f07b h1:ORxAmzrd6SSlSGm/RdPM2TSTTe5+QWp7rqzjEY/pIKA=
-coopcloud.tech/libcapsul v0.0.0-20230605070824-878af473f07b/go.mod h1:6u7ekg+v+yL07QtU7E+K+WqK9LKDDqTF4s+PrIXZ+QM=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52 h1:cyFFOl0tKe+dVHt8saejG8xoff33eQiHxFCVzRpPUjM=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@@ -110,8 +108,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
-github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg=
+github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -324,8 +322,9 @@ github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
-github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -343,16 +342,16 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc=
-github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
+github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
-github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
+github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -377,8 +376,8 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
-github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -421,12 +420,12 @@ github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4x
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
-github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
+github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
+github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
-github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
+github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
+github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -442,6 +441,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -456,6 +457,8 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-sql-driver/mysql v1.3.0 h1:pgwjLi/dvffoP9aabwkT3AKpXQM93QARkjFhDDqC1UE=
 github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@@ -548,6 +551,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -621,8 +625,6 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn
 github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hetznercloud/hcloud-go v1.48.0 h1:b6x0ABNYJr8zVX9sE1kCNMU/sndzI8vmZjxEWEr+Gn0=
-github.com/hetznercloud/hcloud-go v1.48.0/go.mod h1:VzDWThl47lOnZXY0q5/LPFD+M62pfe/52TV+mOrpp9Q=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -695,11 +697,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/libdns/gandi v1.0.2 h1:1Ts8UpI1x5PVKpOjKC7Dn4+EObndz9gm6vdZnloHSKQ=
-github.com/libdns/gandi v1.0.2/go.mod h1:hxpbQKcQFgQrTS5lV4tAgn6QoL6HcCnoBJaW5nOW4Sk=
-github.com/libdns/libdns v0.1.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
-github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
-github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
 github.com/magiconair/properties v1.5.3/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -794,6 +791,7 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -807,13 +805,45 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
+github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
+github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
+github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
+github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
+github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
+github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc=
+github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk=
+github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
+github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
+github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
+github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
+github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
+github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
+github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
+github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
+github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
+github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
+github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw=
+github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
+github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
+github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -1058,6 +1088,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
@@ -1114,8 +1145,9 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1152,11 +1184,15 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
 golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1191,6 +1227,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -1206,24 +1243,30 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1257,8 +1300,9 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1334,6 +1378,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1361,12 +1406,15 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1376,24 +1424,30 @@ golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1404,11 +1458,14 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1472,6 +1529,7 @@ golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4X
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
@@ -1480,10 +1538,16 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
-golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1692,8 +1756,8 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY=
-gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/autocomplete/autocomplete.go

@@ -5,7 +5,6 @@ import (
 
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -28,12 +27,7 @@ func AppNameComplete(c *cli.Context) {
 
 // RecipeNameComplete completes recipe names.
 func RecipeNameComplete(c *cli.Context) {
-	// defaults since we can't take arguments here... this means auto-completion
-	// of recipe names always access the network if e.g. the catalogue needs
-	// cloning / updating
-	conf := runtime.New()
-
-	catl, err := recipe.ReadRecipeCatalogue(conf)
+	catl, err := recipe.ReadRecipeCatalogue(false)
 	if err != nil {
 		logrus.Warn(err)
 	}
@@ -74,7 +68,6 @@ func SubcommandComplete(c *cli.Context) {
 		"autocomplete",
 		"catalogue",
 		"recipe",
-		"record",
 		"server",
 		"upgrade",
 	}

pkg/catalogue/catalogue.go

@@ -8,58 +8,54 @@ import (
 
 	"coopcloud.tech/abra/pkg/config"
 	gitPkg "coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 )
 
 // CatalogueSkipList is all the repos that are not recipes.
 var CatalogueSkipList = map[string]bool{
-	"abra":                        true,
-	"abra-apps":                   true,
-	"abra-aur":                    true,
-	"abra-bash":                   true,
-	"abra-capsul":                 true,
-	"abra-gandi":                  true,
-	"abra-hetzner":                true,
-	"apps":                        true,
-	"aur-abra-git":                true,
-	"auto-recipes-catalogue-json": true,
-	"auto-mirror":                 true,
-	"backup-bot":                  true,
-	"backup-bot-two":              true,
-	"beta.coopcloud.tech":         true,
-	"comrade-renovate-bot":        true,
-	"coopcloud.tech":              true,
-	"coturn":                      true,
-	"docker-cp-deploy":            true,
-	"docker-dind-bats-kcov":       true,
-	"docs.coopcloud.tech":         true,
-	"drone-abra":                  true,
-	"example":                     true,
-	"gardening":                   true,
-	"go-abra":                     true,
-	"organising":                  true,
-	"pyabra":                      true,
-	"radicle-seed-node":           true,
-	"recipes-catalogue-json":      true,
-	"recipes-wishlist":            true,
-	"recipes.coopcloud.tech":      true,
-	"stack-ssh-deploy":            true,
-	"swarm-cronjob":               true,
-	"tagcmp":                      true,
-	"traefik-cert-dumper":         true,
-	"tyop":                        true,
+	"abra":                         true,
+	"abra-apps":                    true,
+	"abra-aur":                     true,
+	"abra-bash":                    true,
+	"abra-capsul":                  true,
+	"abra-gandi":                   true,
+	"abra-hetzner":                 true,
+	"abra-integration-test-recipe": true,
+	"apps":                         true,
+	"aur-abra-git":                 true,
+	"auto-mirror":                  true,
+	"auto-recipes-catalogue-json":  true,
+	"backup-bot":                   true,
+	"backup-bot-two":               true,
+	"beta.coopcloud.tech":          true,
+	"comrade-renovate-bot":         true,
+	"coopcloud.tech":               true,
+	"coturn":                       true,
+	"docker-cp-deploy":             true,
+	"docker-dind-bats-kcov":        true,
+	"docs.coopcloud.tech":          true,
+	"drone-abra":                   true,
+	"example":                      true,
+	"gardening":                    true,
+	"go-abra":                      true,
+	"organising":                   true,
+	"pyabra":                       true,
+	"radicle-seed-node":            true,
+	"recipes-catalogue-json":       true,
+	"recipes-wishlist":             true,
+	"recipes.coopcloud.tech":       true,
+	"stack-ssh-deploy":             true,
+	"swarm-cronjob":                true,
+	"tagcmp":                       true,
+	"traefik-cert-dumper":          true,
+	"tyop":                         true,
 }
 
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
-func EnsureCatalogue(conf *runtime.Config) error {
+func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
-		if conf.Offline {
-			return fmt.Errorf("no local copy of the catalogue available, network access required")
-		}
-
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
 		if err := gitPkg.Clone(catalogueDir, url); err != nil {
 			return err
@@ -71,9 +67,8 @@ func EnsureCatalogue(conf *runtime.Config) error {
 	return nil
 }
 
-// EnsureUpToDate ensures that the local catalogue has no unstaged changes as
-// is up to date. This is useful to run before doing catalogue generation.
-func EnsureUpToDate(conf *runtime.Config) error {
+// EnsureIsClean makes sure that the catalogue has no unstaged changes.
+func EnsureIsClean() error {
 	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
 	if err != nil {
 		return err
@@ -84,11 +79,11 @@ func EnsureUpToDate(conf *runtime.Config) error {
 		return fmt.Errorf(msg, config.CATALOGUE_DIR)
 	}
 
-	if conf.Offline {
-		logrus.Debug("attempting to use local catalogue without access network (\"--offline\")")
-		return nil
-	}
+	return nil
+}
 
+// EnsureUpToDate ensures that the local catalogue is up to date.
+func EnsureUpToDate() error {
 	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
 	if err != nil {
 		return err

pkg/compose/compose.go

@@ -122,7 +122,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 
 		discovered := false
 		for oldLabel, value := range service.Deploy.Labels {
-			if strings.HasPrefix(oldLabel, "coop-cloud") {
+			if strings.HasPrefix(oldLabel, "coop-cloud") && strings.Contains(oldLabel, "version") {
 				discovered = true
 
 				bytes, err := ioutil.ReadFile(composeFile)

pkg/config/app.go

@@ -326,7 +326,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	}
 
 	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
-	if _, err := os.Stat(appEnvPath); os.IsExist(err) {
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 

pkg/config/env.go

@@ -14,7 +14,16 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
+// getBaseDir retrieves the Abra base directory.
+func getBaseDir() string {
+	home := os.ExpandEnv("$HOME/.abra")
+	if customAbraDir, exists := os.LookupEnv("ABRA_DIR"); exists && customAbraDir != "" {
+		home = customAbraDir
+	}
+	return home
+}
+
+var ABRA_DIR = getBaseDir()
 var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
 var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
 var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")

pkg/dns/dns.go

@@ -3,32 +3,8 @@ package dns
 import (
 	"fmt"
 	"net"
-	"os"
-	"time"
-
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/sirupsen/logrus"
 )
 
-// NewToken constructs a new DNS provider token.
-func NewToken(provider, providerTokenEnvVar string) (string, error) {
-	if token, present := os.LookupEnv(providerTokenEnvVar); present {
-		return token, nil
-	}
-
-	logrus.Debugf("no %s in environment, asking via stdin", providerTokenEnvVar)
-
-	var token string
-	prompt := &survey.Input{
-		Message: fmt.Sprintf("%s API token?", provider),
-	}
-	if err := survey.AskOne(prompt, &token); err != nil {
-		return "", err
-	}
-
-	return token, nil
-}
-
 // EnsureIPv4 ensures that an ipv4 address is set for a domain name
 func EnsureIPv4(domainName string) (string, error) {
 	ipv4, err := net.ResolveIPAddr("ip", domainName)
@@ -72,12 +48,3 @@ func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
 
 	return ipv4, nil
 }
-
-// GetTTL parses a ttl string into a duration
-func GetTTL(ttl string) (time.Duration, error) {
-	val, err := time.ParseDuration(ttl)
-	if err != nil {
-		return val, err
-	}
-	return val, nil
-}

pkg/dns/gandi/gandi.go

@@ -1,15 +0,0 @@
-package gandi
-
-import (
-	"coopcloud.tech/abra/pkg/dns"
-	"github.com/libdns/gandi"
-)
-
-// New constructs a new DNs provider.
-func New() (gandi.Provider, error) {
-	token, err := dns.NewToken("Gandi", "GANDI_TOKEN")
-	if err != nil {
-		return gandi.Provider{}, err
-	}
-	return gandi.Provider{APIToken: token}, nil
-}

pkg/jsontable/jsontable.go

@@ -191,6 +191,12 @@ func (t *JSONTable) SetAutoMergeCellsByColumnIndex(cols []int) {
 	t.tbl.SetAutoMergeCellsByColumnIndex(cols)
 }
 
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAlignment(align int) {
+	// FIXME
+	t.tbl.SetAlignment(align)
+}
+
 func (t *JSONTable) SetAutoMergeCells(auto bool) {
 	// FIXME
 	t.tbl.SetAutoMergeCells(auto)

pkg/lint/recipe.go

@@ -9,7 +9,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
@@ -334,11 +333,7 @@ func LintImagePresent(recipe recipe.Recipe) (bool, error) {
 }
 
 func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
-	// defaults since we can't take arguments here... this means this lint rule
-	// always access the network if e.g. the catalogue needs cloning / updating
-	conf := runtime.New()
-
-	catl, err := recipePkg.ReadRecipeCatalogue(conf)
+	catl, err := recipePkg.ReadRecipeCatalogue(false)
 	if err != nil {
 		logrus.Fatal(err)
 	}

pkg/recipe/recipe.go

@@ -17,7 +17,6 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/limit"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/abra/pkg/web"
@@ -25,7 +24,6 @@ import (
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
-	gitConfig "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
@@ -207,8 +205,8 @@ func (r Recipe) Tags() ([]string, error) {
 }
 
 // Get retrieves a recipe.
-func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
-	if err := EnsureExists(recipeName, conf); err != nil {
+func Get(recipeName string, offline bool) (Recipe, error) {
+	if err := EnsureExists(recipeName); err != nil {
 		return Recipe{}, err
 	}
 
@@ -234,7 +232,7 @@ func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
 		return Recipe{}, err
 	}
 
-	meta, err := GetRecipeMeta(recipeName, conf)
+	meta, err := GetRecipeMeta(recipeName, offline)
 	if err != nil {
 		switch err.(type) {
 		case RecipeMissingFromCatalogue:
@@ -252,19 +250,10 @@ func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
 }
 
 // EnsureExists ensures that a recipe is locally cloned
-func EnsureExists(recipeName string, conf *runtime.Config) error {
-	if !conf.RecipeExists {
-		logrus.Debug("skipping ensuring recipe locally exists")
-		return nil
-	}
-
+func EnsureExists(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
-		if conf.Offline {
-			return fmt.Errorf("no local copy of %s available, network access required", recipeName)
-		}
-
 		logrus.Debugf("%s does not exist, attemmpting to clone", recipeDir)
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipeName)
 		if err := gitPkg.Clone(recipeDir, url); err != nil {
@@ -283,15 +272,6 @@ func EnsureExists(recipeName string, conf *runtime.Config) error {
 func EnsureVersion(recipeName, version string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
-	isClean, err := gitPkg.IsClean(recipeDir)
-	if err != nil {
-		return err
-	}
-
-	if !isClean {
-		return fmt.Errorf("%s has locally unstaged changes", recipeName)
-	}
-
 	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
@@ -318,7 +298,10 @@ func EnsureVersion(recipeName, version string) error {
 		return err
 	}
 
-	logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
+	joinedTags := strings.Join(parsedTags, ", ")
+	if joinedTags != "" {
+		logrus.Debugf("read %s as tags for recipe %s", joinedTags, recipeName)
+	}
 
 	if tagRef.String() == "" {
 		return fmt.Errorf("the local copy of %s doesn't seem to have version %s available?", recipeName, version)
@@ -343,30 +326,31 @@ func EnsureVersion(recipeName, version string) error {
 	return nil
 }
 
-// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
-func EnsureLatest(recipeName string, conf *runtime.Config) error {
-	if !conf.RecipeLatest {
-		logrus.Debug("skipping ensuring recipe is synced with remote")
-		return nil
-	}
-
+// EnsureIsClean makes sure that the recipe repository has no unstaged changes.
+func EnsureIsClean(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	isClean, err := gitPkg.IsClean(recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to check git clean status in %s: %s", recipeDir, err)
 	}
 
 	if !isClean {
-		return fmt.Errorf("%s has locally unstaged changes", recipeName)
+		msg := "%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, recipeName, recipeDir)
 	}
 
+	return nil
+}
+
+// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
+func EnsureLatest(recipeName string) error {
+	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+
 	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
 
-	logrus.Debugf("attempting to open git repository in %s", recipeDir)
-
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return err
@@ -377,24 +361,9 @@ func EnsureLatest(recipeName string, conf *runtime.Config) error {
 		return err
 	}
 
-	meta, err := GetRecipeMeta(recipeName, conf)
+	branch, err := gitPkg.GetDefaultBranch(repo, recipeDir)
 	if err != nil {
-		switch err.(type) {
-		case RecipeMissingFromCatalogue:
-			meta = RecipeMeta{}
-		default:
-			return err
-		}
-	}
-
-	var branch plumbing.ReferenceName
-	if meta.DefaultBranch != "" {
-		branch = plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", meta.DefaultBranch))
-	} else {
-		branch, err = gitPkg.GetDefaultBranch(repo, recipeDir)
-		if err != nil {
-			return err
-		}
+		return err
 	}
 
 	checkOutOpts := &git.CheckoutOptions{
@@ -453,7 +422,7 @@ func GetVersionLabelLocal(recipe Recipe) (string, error) {
 
 	for _, service := range recipe.Config.Services {
 		for label, value := range service.Deploy.Labels {
-			if strings.HasPrefix(label, "coop-cloud") {
+			if strings.HasPrefix(label, "coop-cloud") && strings.Contains(label, "version") {
 				return value, nil
 			}
 		}
@@ -599,29 +568,9 @@ func GetStringInBetween(recipeName, str, start, end string) (result string, err
 }
 
 // EnsureUpToDate ensures that the local repo is synced to the remote
-func EnsureUpToDate(recipeName string, conf *runtime.Config) error {
-	if !conf.RecipeLatest {
-		logrus.Debug("skipping ensuring recipe is synced with remote")
-		return nil
-	}
-
+func EnsureUpToDate(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
-	isClean, err := gitPkg.IsClean(recipeDir)
-	if err != nil {
-		return fmt.Errorf("unable to check git clean status in %s: %s", recipeDir, err)
-	}
-
-	if !isClean {
-		msg := "%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding"
-		return fmt.Errorf(msg, recipeName, recipeDir)
-	}
-
-	if conf.Offline {
-		logrus.Debug("attempting to use local recipe without access network (\"--offline\")")
-		return nil
-	}
-
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return fmt.Errorf("unable to open %s: %s", recipeDir, err)
@@ -647,12 +596,7 @@ func EnsureUpToDate(recipeName string, conf *runtime.Config) error {
 		return fmt.Errorf("unable to check out default branch in %s: %s", recipeDir, err)
 	}
 
-	fetchOpts := &git.FetchOptions{
-		Tags: git.AllTags,
-		RefSpecs: []gitConfig.RefSpec{
-			gitConfig.RefSpec(fmt.Sprintf("%s:%s", branch, branch)),
-		},
-	}
+	fetchOpts := &git.FetchOptions{Tags: git.AllTags}
 	if err := repo.Fetch(fetchOpts); err != nil {
 		if !strings.Contains(err.Error(), "already up-to-date") {
 			return fmt.Errorf("unable to fetch tags in %s: %s", recipeDir, err)
@@ -677,15 +621,17 @@ func EnsureUpToDate(recipeName string, conf *runtime.Config) error {
 }
 
 // ReadRecipeCatalogue reads the recipe catalogue.
-func ReadRecipeCatalogue(conf *runtime.Config) (RecipeCatalogue, error) {
+func ReadRecipeCatalogue(offline bool) (RecipeCatalogue, error) {
 	recipes := make(RecipeCatalogue)
 
-	if err := catalogue.EnsureCatalogue(conf); err != nil {
+	if err := catalogue.EnsureCatalogue(); err != nil {
 		return nil, err
 	}
 
-	if err := catalogue.EnsureUpToDate(conf); err != nil {
-		return nil, err
+	if !offline {
+		if err := catalogue.EnsureUpToDate(); err != nil {
+			return nil, err
+		}
 	}
 
 	if err := readRecipeCatalogueFS(&recipes); err != nil {
@@ -712,10 +658,10 @@ func readRecipeCatalogueFS(target interface{}) error {
 }
 
 // VersionsOfService lists the version of a service.
-func VersionsOfService(recipe, serviceName string, conf *runtime.Config) ([]string, error) {
+func VersionsOfService(recipe, serviceName string, offline bool) ([]string, error) {
 	var versions []string
 
-	catalogue, err := ReadRecipeCatalogue(conf)
+	catalogue, err := ReadRecipeCatalogue(offline)
 	if err != nil {
 		return nil, err
 	}
@@ -749,8 +695,8 @@ func (r RecipeMissingFromCatalogue) Error() string {
 }
 
 // GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
-func GetRecipeMeta(recipeName string, conf *runtime.Config) (RecipeMeta, error) {
-	catl, err := ReadRecipeCatalogue(conf)
+func GetRecipeMeta(recipeName string, offline bool) (RecipeMeta, error) {
+	catl, err := ReadRecipeCatalogue(offline)
 	if err != nil {
 		return RecipeMeta{}, err
 	}
@@ -762,10 +708,6 @@ func GetRecipeMeta(recipeName string, conf *runtime.Config) (RecipeMeta, error)
 		}
 	}
 
-	if err := EnsureExists(recipeName, conf); err != nil {
-		return RecipeMeta{}, err
-	}
-
 	logrus.Debugf("recipe metadata retrieved for %s", recipeName)
 
 	return recipeMeta, nil
@@ -849,11 +791,7 @@ type InternalTracker struct {
 type RepoCatalogue map[string]RepoMeta
 
 // ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
-func ReadReposMetadata(conf *runtime.Config) (RepoCatalogue, error) {
-	if conf.Offline {
-		return nil, fmt.Errorf("network access required to query recipes metadata")
-	}
-
+func ReadReposMetadata() (RepoCatalogue, error) {
 	reposMeta := make(RepoCatalogue)
 
 	pageIdx := 1
@@ -888,7 +826,7 @@ func ReadReposMetadata(conf *runtime.Config) (RepoCatalogue, error) {
 }
 
 // GetRecipeVersions retrieves all recipe versions.
-func GetRecipeVersions(recipeName string, conf *runtime.Config) (RecipeVersions, error) {
+func GetRecipeVersions(recipeName string, offline bool) (RecipeVersions, error) {
 	versions := RecipeVersions{}
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
@@ -926,7 +864,7 @@ func GetRecipeVersions(recipeName string, conf *runtime.Config) (RecipeVersions,
 
 		logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
 
-		recipe, err := Get(recipeName, conf)
+		recipe, err := Get(recipeName, offline)
 		if err != nil {
 			return err
 		}
@@ -1033,11 +971,7 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri
 }
 
 // UpdateRepositories clones and updates all recipe repositories locally.
-func UpdateRepositories(repos RepoCatalogue, recipeName string, conf *runtime.Config) error {
-	if conf.Offline {
-		return fmt.Errorf("network access required to update recipes")
-	}
-
+func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 	var barLength int
 	if recipeName != "" {
 		barLength = 1
@@ -1071,10 +1005,6 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string, conf *runtime.Co
 				logrus.Fatal(err)
 			}
 
-			if err := EnsureUpToDate(rm.Name, conf); err != nil {
-				logrus.Fatal(err)
-			}
-
 			ch <- rm.Name
 			retrieveBar.Add(1)
 		}(repoMeta)

pkg/recipe/recipe_test.go

@@ -0,0 +1,31 @@
+package recipe
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetVersionLabelLocalDoesNotUseTimeoutLabel(t *testing.T) {
+	offline := true
+	recipe, err := Get("traefik", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 1; i < 1000; i++ {
+		label, err := GetVersionLabelLocal(recipe)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// NOTE(d1): this is potentially quite a brittle unit test as it needs to
+		// hardcode the default timeout label to ensure that the label parser never
+		// returns it. hopefully this won't fail too often! if you're here because
+		// of a failure, just update the `defaultTimeoutLabel` value & permalink
+		// below
+		// https://git.coopcloud.tech/coop-cloud/traefik/src/commit/ac3a47fe8ca3ef92db84f64cfedfbb348000faee/.env.sample#L2
+		defaultTimeoutLabel := "300"
+		assert.NotEqual(t, label, defaultTimeoutLabel)
+	}
+}

pkg/runtime/config.go

@@ -1,59 +0,0 @@
-package runtime
-
-import "github.com/sirupsen/logrus"
-
-// Config is a runtime behaviour modifier.
-type Config struct {
-	Offline      bool // Whether or not Abra should prefer local / offline access
-	RecipeExists bool // Whether or not Abra should ensure the recipe is locally cloned
-	RecipeLatest bool // Whether or not Abra should ensure the recipe has the latest commit
-}
-
-// Option is a runtime configuration option.
-type Option func(c *Config)
-
-// New creates a new runtime configuration.
-func New(opts ...Option) *Config {
-	conf := &Config{
-		Offline:      false,
-		RecipeExists: true,
-		RecipeLatest: true,
-	}
-
-	for _, optFunc := range opts {
-		optFunc(conf)
-	}
-
-	return conf
-}
-
-// WithOffline ensures Abra attempts to prefer local file system and offline
-// access.
-func WithOffline(offline bool) Option {
-	return func(c *Config) {
-		if offline {
-			logrus.Debugf("runtime config: attempting to run in offline mode")
-		}
-		c.Offline = offline
-	}
-}
-
-// WithEnsureRecipeExists ensures recipe exists locally.
-func WithEnsureRecipeExists(exists bool) Option {
-	return func(c *Config) {
-		if exists {
-			logrus.Debugf("runtime config: ensuring recipe exists")
-		}
-		c.RecipeExists = exists
-	}
-}
-
-// WithEnsureRecipeUpToDate ensures recipe is synced with the remote.
-func WithEnsureRecipeUpToDate(upToDate bool) Option {
-	return func(c *Config) {
-		if upToDate {
-			logrus.Debugf("runtime config: ensuring recipe is synced with remote")
-		}
-		c.RecipeLatest = upToDate
-	}
-}

pkg/upstream/stack/stack.go

@@ -5,7 +5,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"strings"
+	"os"
+	"os/signal"
 	"time"
 
 	"coopcloud.tech/abra/pkg/upstream/convert"
@@ -414,7 +415,7 @@ func deployServices(
 		return nil
 	}
 
-	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
+	logrus.Infof("Waiting for %s to deploy... please hold 🤚", appName)
 	ch := make(chan error, len(serviceIDs))
 	for serviceID, serviceName := range serviceIDs {
 		logrus.Debugf("waiting on %s to converge", serviceName)
@@ -431,7 +432,7 @@ func deployServices(
 		logrus.Debugf("assuming %s converged successfully", serviceID)
 	}
 
-	logrus.Info("services converged 👌")
+	logrus.Infof("Successfully deployed %s", appName)
 
 	return nil
 }
@@ -454,6 +455,10 @@ func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appN
 	errChan := make(chan error, 1)
 	pipeReader, pipeWriter := io.Pipe()
 
+	sigintChannel := make(chan os.Signal, 1)
+	signal.Notify(sigintChannel, os.Interrupt)
+	defer signal.Stop(sigintChannel)
+
 	go func() {
 		errChan <- progress.ServiceProgress(ctx, cl, serviceID, pipeWriter)
 	}()
@@ -465,6 +470,12 @@ func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appN
 	select {
 	case err := <-errChan:
 		return err
+	case <-sigintChannel:
+		return fmt.Errorf(fmt.Sprintf(`
+Not waiting for %s to deploy. The deployment is ongoing...
+
+If you want to stop the deployment, try:
+  abra app undeploy %s`, appName, appName))
 	case <-time.After(timeout):
 		return fmt.Errorf(fmt.Sprintf(`
 %s has not converged (%s second timeout reached).
@@ -481,7 +492,7 @@ And inspect the logs with:
 
     abra app logs %s
 
-If a service is failing to even start, try smoke out the error with:
+If a service is failing to even start, try to smoke out the error with:
 
     abra app errors --watch %s
 

tests/README.md

@@ -0,0 +1,10 @@
+# Test suite
+
+* Unit testing is done in the packages themselves, run `find . -name
+  "*_test.go"` to find those. Use `make test` to run the entire unit test
+  suite. Some unit tests require mocked files which are located in
+  `./tests/resources.`
+
+* Integration tests are in `./tests/integration`. Please see [these
+  docs](https://docs.coopcloud.tech/abra/hack/#integration-tests) for
+  instructions and tips on how to run them.

tests/integration/app_backup.bats

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+setup_file() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file() {
+  _rm_app
+  _rm_server
+}
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "retrieve recipe if missing" {
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app backup "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'no containers matching'
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --chaos --no-input
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_success
+  assert_output --partial 'running backup for the app service'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "ensure recipe not up to date if --offline" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  refute [ -z "$latestCommit" ];
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug --offline
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "detect backup labels" {
+  run $ABRA app backup "$TEST_APP_DOMAIN" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  assert_output --partial 'detected backup paths'
+  assert_output --partial 'detected pre-hook command'
+  assert_output --partial 'detected post-hook command'
+}
+
+@test "error if backups not enabled" {
+  run sed -i '/backupbot.backup=true/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_failure
+  assert_output --partial 'no backup config for app'
+
+  _checkout_recipe
+}
+
+@test "error if backup paths not configured" {
+  run sed -i '/backupbot.backup.path=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app --chaos
+  assert_failure
+  assert_output --partial 'backup paths are empty for app?'
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "backup single service" {
+  _deploy_app
+
+  run $ABRA app backup "$TEST_APP_DOMAIN" app
+  assert_success
+  assert_output --partial 'running backup for the app service'
+
+  sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
+  assert_output --partial "_$sanitisedDomainName_app"
+
+  assert_exists "$ABRA_DIR/backups"
+  assert bash -c "ls $ABRA_DIR/backups | grep -q $1_$sanitisedDomainName_app"
+
+  _undeploy_app
+}

tests/integration/app_check.bats

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app check
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app check DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'all necessary environment variables defined'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --chaos
+  assert_success
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "ensure recipe not up to date if --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --offline
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "error if missing .env.sample" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"
+  assert_success
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --chaos
+  assert_failure
+  assert_output --partial '.env.sample does not exist?'
+
+  _checkout_recipe
+}
+
+@test "error if missing env var" {
+  run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
+  assert_success
+
+  run $ABRA app check "$TEST_APP_DOMAIN" --chaos
+  assert_failure
+  assert_output --partial \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env is missing NEW_VAR"
+
+  _checkout_recipe
+}

tests/integration/app_cmd.bats

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app cmd
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app cmd DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_success
+  assert_output --partial 'baz'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
+  assert_success
+  assert_output --partial 'baz'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_success
+  assert_output --partial 'baz'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "ensure recipe not up to date if --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --offline
+  assert_success
+  assert_output --partial 'baz'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "error if missing arguments without passing --local" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing arguments'
+}
+
+@test "error if missing arguments when passing --local" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" --local
+  assert_failure
+  assert_output --partial 'missing arguments'
+}
+
+@test "cannot use --local and --user at same time" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --user root
+  assert_failure
+  assert_output --partial 'cannot use --local & --user together'
+}
+
+@test "error if missing abra.sh" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh"
+  assert_success
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
+  assert_failure
+  assert_output --partial "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh does not exist"
+
+  _checkout_recipe
+}
+
+@test "error if missing command" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist --local
+  assert_failure
+  assert_output --partial "doesn't have a doesnt_exist function"
+}
+
+@test "run --local command" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  assert_success
+  assert_output --partial 'baz'
+}
+
+@test "run command with single arg" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_arg --local -- bing
+  assert_success
+  assert_output --partial 'bing'
+}
+
+@test "run command with several args" {
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_args --local -- bong bang
+  assert_success
+  assert_output --partial 'bong bang'
+}
+
+# bats test_tags=slow
+@test "run command on service" {
+  _deploy_app
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" app test_cmd
+  assert_success
+  assert_output --partial 'baz'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "error if missing service" {
+  _deploy_app
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist test_cmd
+  assert_failure
+  assert_output --partial 'no service doesnt_exist'
+
+  _undeploy_app
+}

tests/integration/app_config.bats

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+}
+
+teardown_file(){
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app config
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app config DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}

tests/integration/app_cp.bats

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app cp
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app cp DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if missing src/dest arguments" {
+  run $ABRA app cp "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing <src> argument'
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt
+  assert_failure
+  assert_output --partial 'missing <dest> argument'
+}
+
+@test "either src/dest has correct syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app
+  assert_failure
+  assert_output --partial 'arguments must take $SERVICE:$PATH form'
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app .
+  assert_failure
+  assert_output --partial 'arguments must take $SERVICE:$PATH form'
+}
+
+@test "detect 'coming FROM' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug
+  assert_failure
+  assert_output --partial 'coming FROM the container'
+}
+
+@test "detect 'going TO' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug
+  assert_failure
+  assert_output --partial 'going TO the container'
+}
+
+@test "error if local file missing" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere
+  assert_failure
+  assert_output --partial 'myfile.txt does not exist locally?'
+}
+
+# bats test_tags=slow
+@test "error if service doesn't exist" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "copy to container" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
+  assert_success
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "copy from container" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
+  assert_success
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR"
+  assert_success
+  assert_exists "$BATS_TMPDIR/myfile.txt"
+  assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
+
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
+}

tests/integration/app_deploy.bats

@@ -0,0 +1,294 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app deploy
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app deploy DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+  refute_output --partial 'chaos'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --chaos --no-input --no-converge-checks
+  assert_success
+  assert_output --partial 'chaos'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+# bats test_tags=slow
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+
+  _reset_recipe "$TEST_RECIPE"
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "ensure recipe not up to date if --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  # NOTE(d1): need to use --chaos to force same commit
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos --offline
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  _undeploy_app
+  _reset_recipe "$TEST_RECIPE"
+}
+
+# bats test_tags=slow
+@test "deploy latest commit if no published versions and no --chaos" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  _remove_tags
+
+  # NOTE(d1): need to pass --offline to stop tags being pulled again
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --offline
+  assert_success
+  assert_output --partial "$latestCommit"
+  assert_output --partial 'using latest commit'
+  refute_output --partial 'chaos'
+
+  _undeploy_app
+  _reset_tags
+}
+
+# bats test_tags=slow
+@test "ensure same commit if --chaos" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  refute_output --partial "$latestCommit"
+  assert_output --partial "$threeCommitsBack"
+  assert_output --partial 'chaos'
+
+  _undeploy_app
+  _reset_recipe
+}
+
+# bats test_tags=slow
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+
+  _undeploy_app
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "no deploy if lint error" {
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_failure
+  assert_output --partial 'failed lint checks'
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "error if already deployed and no --force/--chaos" {
+  _deploy_app
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'already deployed'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "re-deploy deployed app if --force/--chaos" {
+  _deploy_app
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --force
+  assert_success
+  assert_output --partial 'already deployed but continuing'
+  assert_output --partial '--force'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial 'already deployed but continuing'
+  assert_output --partial '--chaos'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "deploy latest version from catalogue if no --chaos" {
+  latestVersion=$(jq -r '.gitea.versions[-1] | keys[0]' < "$ABRA_DIR/catalogue/recipes.json")
+  refute [ -z "$latestVersion" ];
+
+  run $ABRA app new gitea \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "gitea.$TEST_SERVER" \
+    --secrets
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/gitea.$TEST_SERVER.env"
+
+  run $ABRA app deploy "gitea.$TEST_SERVER" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestVersion"
+
+  run $ABRA app undeploy "gitea.$TEST_SERVER" --no-input
+  assert_success
+
+  run $ABRA app secret remove "gitea.$TEST_SERVER" --all --no-input
+  assert_success
+
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume remove "gitea.$TEST_SERVER" --no-input
+  assert_success
+
+  run $ABRA app remove "gitea.$TEST_SERVER" --no-input
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/gitea.$TEST_SERVER.env"
+}
+
+# bats test_tags=slow
+@test "skip domain check if missing DOMAIN=" {
+  run cp "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" "$BATS_TMPDIR/$TEST_APP_DOMAIN.env"
+  assert_success
+  assert_exists "$BATS_TMPDIR/$TEST_APP_DOMAIN.env"
+
+  run grep -q "DOMAIN=" "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i '/DOMAIN=.*/d' "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run grep -q "DOMAIN=" "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_failure
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial 'no DOMAIN=... configured for app'
+
+  run $ABRA app undeploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run mv "$BATS_TMPDIR/$TEST_APP_DOMAIN.env" "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+}
+
+# bats test_tags=slow
+@test "skip domain check when requested" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --no-domain-checks
+  assert_success
+  assert_output --partial 'skipping domain checks as requested'
+
+  _undeploy_app
+}
+
+@test "error if specific version does not exist" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" DOESNTEXIST --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial "doesn't seem to have version DOESNTEXIST available"
+}
+
+# bats test_tags=slow
+@test "deploy specific version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input
+  assert_success
+  assert_output --partial "0.2.0+1.21.0"
+
+  _undeploy_app
+}

tests/integration/app_errors.bats

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app errors
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app errors DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app errors "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "report errors" {
+  _deploy_app
+
+  run $ABRA app errors "$TEST_APP_DOMAIN"
+  assert_success
+
+  _undeploy_app
+}

tests/integration/app_list.bats

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "list without status" {
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "$TEST_APP_DOMAIN"
+}
+
+# bats test_tags=slow
+@test "list with status" {
+  run $ABRA app ls --status
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "$TEST_APP_DOMAIN"
+  assert_output --partial "unknown"
+
+  _deploy_app
+
+  run $ABRA app ls --status
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "$TEST_APP_DOMAIN"
+  assert_output --partial "deployed"
+
+  _undeploy_app
+}
+
+@test "filter by server" {
+  run mkdir -p "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/foo.com"
+
+  run cp \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" \
+    "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert_exists "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "foo.com"
+
+  run $ABRA app ls --server foo.com
+  assert_success
+  refute_output --partial "server: $TEST_SERVER |"
+  assert_output --partial "server: foo.com |"
+
+  run rm -rf "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/foo.com"
+}
+
+@test "filter by recipe" {
+  run mkdir -p "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/foo.com"
+
+  run cp \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" \
+    "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert_exists "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run sed -i "s/TYPE=$TEST_RECIPE/TYPE=foo-recipe/g" "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert grep -q "TYPE=foo-recipe" "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_RECIPE"
+  assert_output --partial "foo-recipe"
+
+  run $ABRA app ls --recipe foo-recipe
+  assert_success
+  refute_output --partial "$TEST_RECIPE"
+  assert_output --partial "foo-recipe"
+}
+
+@test "server stats are correct" {
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "server: $TEST_SERVER"
+  assert_output --partial "total apps: 1"
+
+  run mkdir -p "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/foo.com"
+
+  run cp \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" \
+    "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+  assert_exists "$ABRA_DIR/servers/foo.com/app.foo.com.env"
+
+  run $ABRA app ls
+  assert_success
+  assert_output --partial "$TEST_SERVER"
+  assert_output --partial "foo.com"
+  assert_output --partial "total servers: 2"
+  assert_output --partial "total apps: 2"
+
+  run rm -rf "$ABRA_DIR/servers/foo.com"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/foo.com"
+}
+
+@test "output is machine readable" {
+  run $ABRA app ls --machine
+
+  expectedOutput='{"'
+  expectedOutput+="$TEST_SERVER"
+  expectedOutput+='":{"apps":'
+
+  assert_output --partial "$expectedOutput"
+}

tests/integration/app_logs.bats

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "validate app argument" {
+  run $ABRA app logs
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app logs DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app logs "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}

tests/integration/app_new.bats

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "create new app" {
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  _rm_app
+}
+
+@test "does not overwrite existing env files" {
+  _new_app
+
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'already exists'
+
+  _rm_app
+}
+
+@test "generate secrets" {
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test_password'
+}

tests/integration/app_ps.bats

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app ps
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app ps DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app ps "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "show ps report" {
+  _deploy_app
+
+  run $ABRA app ps "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'app'
+  assert_output --partial 'healthy'
+
+  _undeploy_app
+}

tests/integration/app_remove.bats

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app deploy
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app deploy DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "do not show ALERTA warning if --force / --no-input" {
+  run $ABRA app rm "$TEST_APP_DOMAIN" --force
+  refute_output --partial 'ALERTA'
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  refute_output --partial 'ALERTA'
+
+  _new_app
+}
+
+# bats test_tags=slow
+@test "error if still deployed" {
+  _deploy_app
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  assert_failure
+  assert_output --partial 'is still deployed'
+
+  _undeploy_app
+}
+
+@test "detect no secrets to remove" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test_password'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all --no-input
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test_password'
+  assert_output --partial 'false'
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  assert_success
+  assert_output --partial 'no secrets to remove'
+
+  _new_app
+}
+
+@test "remove secrets" {
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_failure
+  assert_output --partial 'already exists'
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test_password'
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  assert_success
+  refute_output --partial 'no secrets to remove'
+
+  sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
+  assert_output --partial "$sanitisedDomainName_test_password_v1 removed"
+
+  _new_app
+}
+
+# bats test_tags=slow
+@test "detect no volumes to remove" {
+  _deploy_app
+
+  run $ABRA app volume ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test-volume'
+
+  _undeploy_app
+
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  assert_success
+
+  run $ABRA app volume ls "$TEST_APP_DOMAIN"
+  assert_success
+  refute_output --partial 'test-volume'
+  assert_output --partial 'no volumes created'
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  assert_success
+  assert_output --partial 'no volumes to remove'
+
+  _new_app
+}
+
+# bats test_tags=slow
+@test "remove volumes" {
+  _deploy_app
+
+  run $ABRA app volume ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test-volume'
+
+  _undeploy_app
+
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  assert_success
+  assert_output --partial 'test-volume'
+  assert_output --partial 'removed'
+
+  _new_app
+}
+
+@test "remove .env file" {
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  _new_app
+}

tests/integration/app_restart.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app restart
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app restart DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if service missing" {
+  run $ABRA app restart "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing service'
+}
+
+@test "error if not deployed" {
+  run $ABRA app restart "$TEST_APP_DOMAIN" app
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "app is restarted" {
+  _deploy_app
+
+  run $ABRA app restart "$TEST_APP_DOMAIN" app --debug
+  assert_success
+  assert_output --regexp 'attempting to scale .* to 0'
+  assert_output --regexp 'attempting to scale .* to 1'
+  assert_output --partial 'service successfully restarted'
+
+  _undeploy_app
+}

tests/integration/app_restore.bats

@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app restore
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app restore DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
+  assert_failure
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run touch "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.txt"
+
+  run tar -cvf "$BATS_TMPDIR/foo.tar.gz" "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.tar.gz"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --chaos --no-input
+  assert_success
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app "$BATS_TMPDIR/foo.tar.gz" \
+    --debug --chaos
+  assert_success
+  assert_output --partial 'restore config detected'
+  assert_output --partial 'detected pre-hook command'
+  assert_output --partial 'detected post-hook command'
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls "$BATS_TMPDIR"
+  assert_success
+  assert_output --partial 'foo.txt'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  _checkout_recipe
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
+  assert_failure
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+}
+
+@test "ensure recipe not up to date if --offline" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  refute [ -z "$latestCommit" ];
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST --offline
+  assert_failure
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+}
+
+@test "error if missing service" {
+  run $ABRA app restore "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing <service>'
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app
+  assert_failure
+  assert_output --partial 'missing <file>'
+}
+
+@test "error if file doesn't exist" {
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
+  assert_failure
+  assert_output --partial "doesn't exist"
+}
+
+# bats test_tags=slow
+@test "detect labels if restore enabled" {
+  run touch "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.txt"
+
+  run tar -cvf "$BATS_TMPDIR/foo.tar.gz" "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.tar.gz"
+
+  _deploy_app
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app "$BATS_TMPDIR/foo.tar.gz" --debug
+  assert_success
+  assert_output --partial 'restore config detected'
+  assert_output --partial 'detected pre-hook command'
+  assert_output --partial 'detected post-hook command'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "no error if restore not enabled" {
+  run sed -i '/backupbot.restore=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run touch "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.txt"
+
+  run tar -cvf "$BATS_TMPDIR/foo.tar.gz" "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.tar.gz"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --chaos
+  assert_success
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app "$BATS_TMPDIR/foo.tar.gz" \
+    --debug --chaos
+  assert_success
+  refute_output --partial 'restore config detected'
+  refute_output --partial 'detected pre-hook command'
+  refute_output --partial 'detected post-hook command'
+
+  _undeploy_app
+
+  _checkout_recipe
+}
+
+# bats test_tags=slow
+@test "error if service doesn't exist" {
+  run touch "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.txt"
+
+  run tar -cvf "$BATS_TMPDIR/foo.tar.gz" "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.tar.gz"
+
+  _deploy_app
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" DOESNTEXIST "$BATS_TMPDIR/foo.tar.gz" --debug
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "restore backup" {
+  run touch "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.txt"
+
+  run tar -cvf "$BATS_TMPDIR/foo.tar.gz" "$BATS_TMPDIR/foo.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/foo.tar.gz"
+
+  _deploy_app
+
+  run $ABRA app restore "$TEST_APP_DOMAIN" app "$BATS_TMPDIR/foo.tar.gz" --debug
+  assert_success
+  assert_output --partial 'restore config detected'
+  assert_output --partial 'detected pre-hook command'
+  assert_output --partial 'detected post-hook command'
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls "$BATS_TMPDIR"
+  assert_success
+  assert_output --partial 'foo.txt'
+
+  _undeploy_app
+}

tests/integration/app_rollback.bats

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app rollback
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app rollback DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}
+
+@test "ensure recipe up to date if no --offline" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
+}
+
+@test "ensure recipe not up to date if --offline" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  refute [ -z "$latestCommit" ];
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --offline
+  assert_failure
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
+  assert_success
+
+  _reset_recipe "$TEST_RECIPE"
+}
+
+@test "bail if unstaged changes and no --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+  refute_output --partial 'chaos'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+# bats test_tags=slow
+@test "do not bail if unstaged changes and --chaos" {
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_success
+  assert_output --partial 'foo'
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --chaos --no-input --no-converge-checks
+  assert_success
+  assert_output --partial 'chaos'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" \
+    --chaos --no-input --no-converge-checks
+  assert_success
+  assert_output --partial 'chaos'
+
+  _undeploy_app
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+}
+
+# bats test_tags=slow
+@test "ensure same commit if --chaos" {
+  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial "$latestCommit"
+  assert_output --partial 'chaos'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" \
+    --chaos --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestCommit"
+  assert_output --partial 'chaos'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "no rollback if lint error" {
+  _deploy_app
+
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" \
+    --no-input --chaos --chaos --no-converge-checks
+  assert_failure
+  assert_output --partial 'failed lint checks'
+
+  _undeploy_app
+
+  _checkout_recipe
+}
+
+@test "error if not already deployed" {
+  run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input
+  assert_failure
+  assert_output --partial 'not deployed'
+}
+
+# bats test_tags=slow
+@test "no rollback if on oldest version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "you're on oldest"
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "error if specific version is not downgrade" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'is not a downgrade'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "rollback to previous version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.1+1.20.2'
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "0.1.0+1.20.0"
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "rollback to a version 2 tags behind" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "0.2.0+1.21.0"
+
+  run $ABRA app rollback "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "0.1.0+1.20.0"
+
+  _undeploy_app
+}

tests/integration/app_run.bats

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app run
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app run DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if missing service" {
+  run $ABRA app run "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'no <service> provided'
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app
+  assert_failure
+  assert_output --partial 'no <args> provided'
+}
+
+# bats test_tags=slow
+@test "error if service doesn't exist" {
+  _deploy_app
+
+  run $ABRA app run "$TEST_APP_DOMAIN" DOESNTEXIST ls
+  assert_failure
+  assert_output --partial 'no containers matching'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "run command" {
+  _deploy_app
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /
+  assert_success
+  assert_output --partial 'root'
+
+  _undeploy_app
+}

tests/integration/app_secret.bats

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+}
+
+teardown_file(){
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  _rm_app
+  _rm_server
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "generate: validate arguments" {
+  run $ABRA app secret generate
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app secret generate DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing arguments'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" testSecret testVersion --all
+  assert_failure
+  assert_output --partial 'cannot use'
+  assert_output --partial "'--all' together"
+}
+
+@test "generate: single secret no match" {
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" DOESNTEXIST v1
+  assert_failure
+  assert_output --partial "doesn't exist in the env config"
+}
+
+@test "generate: create secrets" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'true'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+}
+
+@test "insert: validate arguments" {
+  run $ABRA app secret insert
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app secret insert "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'missing arguments'
+
+  run $ABRA app secret insert "$TEST_APP_DOMAIN" bar
+  assert_failure
+  assert_output --partial 'missing arguments'
+
+  run $ABRA app secret insert "$TEST_APP_DOMAIN" bar baz
+  assert_failure
+  assert_output --partial 'missing arguments'
+}
+
+@test "insert: create secret" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+
+  run $ABRA app secret insert "$TEST_APP_DOMAIN" test_password v1 foo
+  assert_success
+  assert_output --partial 'successfully stored on server'
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'true'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" test_password
+  assert_success
+}
+
+@test "rm: validate arguments" {
+  run $ABRA app secret rm
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app secret rm DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'no secret(s) specified'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" test_password --all
+  assert_failure
+  assert_output --partial 'cannot use'
+  assert_output --partial "'--all' together"
+}
+
+@test "rm: single secret no match" {
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" foo_password
+  assert_failure
+  assert_output --partial "doesn't exist on server"
+}
+
+@test "rm: no secret match" {
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_failure
+  assert_output --partial 'no secrets to remove'
+}
+
+@test "rm: remove secret" {
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'true'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+}
+
+@test "ls: validate arguments" {
+  run $ABRA app secret ls
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app secret ls DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "ls: show secrets" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'true'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+}

tests/integration/app_services.bats

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app services
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app services DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app services "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "list services" {
+  _deploy_app
+
+  run $ABRA app services "$TEST_APP_DOMAIN"
+  assert_success
+
+  sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
+  assert_output --partial "$sanitisedDomainName_app"
+  assert_output --partial "nginx"
+
+  _undeploy_app
+}

tests/integration/app_undeploy.bats

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app undeploy
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app undeploy DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app undeploy "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "undeploy app" {
+  _deploy_app
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "undeploy and prune" {
+  _deploy_app
+
+  run $ABRA app undeploy "$TEST_APP_DOMAIN" --no-input --prune
+  assert_success
+}

tests/integration/app_upgrade.bats

@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app upgrade
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app upgrade DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+# bats test_tags=slow
+@test "error if specific version is not an upgrade" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input
+  assert_failure
+  assert_output --partial 'is not an upgrade'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "no upgrade if lint error" {
+  _deploy_app
+
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_failure
+  assert_output --partial 'failed lint checks'
+
+  _checkout_recipe
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "no upgrade if on latest version" {
+  _deploy_app
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "you're on latest"
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "upgrade to latest catalogue published version" {
+  appDomain="custom-html.$TEST_SERVER"
+
+  run $ABRA app new custom-html \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$appDomain"
+
+  oneVersionBack=$(jq -r '."custom-html".versions[-2] | keys[0]' < "$ABRA_DIR/catalogue/recipes.json")
+  refute [ -z "$oneVersionBack" ];
+
+  run $ABRA app deploy "$appDomain" "$oneVersionBack" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$oneVersionsBack"
+
+  latestVersion=$(jq -r '."custom-html".versions[-1] | keys[0]' < "$ABRA_DIR/catalogue/recipes.json")
+  refute [ -z "$latestVersion" ];
+
+  run $ABRA app upgrade "$appDomain" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestVersion"
+
+  run $ABRA app undeploy "$appDomain" --no-input
+  assert_success
+
+ # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 3
+
+  run $ABRA app volume remove "$appDomain" --no-input
+  assert_success
+
+  run $ABRA app remove "$appDomain" --no-input
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$appDomain.env"
+}
+
+# bats test_tags=slow
+@test "upgrade specific version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.2.0+1.21.0'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "upgrade to latest" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  latestRelease=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | tail -n 1)
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestRelease"
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "show single release note" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.1+1.20.2'
+
+  latestRelease=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | tail -n 1)
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestRelease"
+  assert_output --partial 'release notes baz' # 0.2.0+1.21.0
+  refute_output --partial 'release notes bar' # 0.1.1+1.20.2
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "show single release note for specific version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.1+1.20.2'
+  assert_output --partial 'release notes bar' # 0.1.1+1.20.2
+  refute_output --partial 'release notes baz' # 0.2.0+1.21.0
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "show multiple release notes" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  latestRelease=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | tail -n 1)
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestRelease"
+  assert_output --partial 'release notes bar' # 0.1.1+1.20.2
+  assert_output --partial 'release notes baz' # 0.2.0+1.21.0
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "show multiple release notes for specific version" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.1.0+1.20.0'
+
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial '0.2.0+1.21.0'
+  assert_output --partial 'release notes bar' # 0.1.1+1.20.2
+  assert_output --partial 'release notes baz' # 0.2.0+1.21.0
+
+  _undeploy_app
+}

tests/integration/app_version.bats

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "validate app argument" {
+  run $ABRA app version
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app version DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "error if not deployed" {
+  run $ABRA app version "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+}
+
+# bats test_tags=slow
+@test "error if version unknown" {
+  run sed -i '/coop-cloud.${STACK_NAME}.version=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+
+  run $ABRA app version "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'failed to determine'
+
+  _checkout_recipe
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "error if not in catalogue" {
+  _deploy_app
+
+  run $ABRA app version "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'does not exist'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "list version" {
+  appDomain="custom-html.$TEST_SERVER"
+
+  run $ABRA app new custom-html \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$appDomain"
+
+  latestVersion=$(git -C "$ABRA_DIR/recipes/custom-html" tag | tail -n 1)
+  refute [ -z "$latestVersion" ];
+
+  run $ABRA app deploy "$appDomain" --no-input --no-converge-checks
+  assert_success
+  assert_output --partial "$latestVersion"
+
+  run $ABRA app version "$appDomain"
+  assert_success
+  assert_output --partial "$latestVersion"
+
+  run $ABRA app undeploy "$appDomain" --no-input
+  assert_success
+
+ # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 3
+
+  run $ABRA app volume remove "$appDomain" --no-input
+  assert_success
+
+  run $ABRA app remove "$appDomain" --no-input
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$appDomain.env"
+}

tests/integration/app_volume.bats

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+setup_file(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+  _add_server
+  _new_app
+}
+
+teardown_file(){
+  _rm_app
+  _rm_server
+}
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
+@test "ls validate app argument" {
+  run $ABRA app volume ls
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app volume ls DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+@test "list no volumes" {
+  run $ABRA app volume ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'no volumes created'
+}
+
+# bats test_tags=slow
+@test "list volumes" {
+  _deploy_app
+
+  run $ABRA app volume ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'test-volume'
+
+  _undeploy_app
+}
+
+@test "rm validate app argument" {
+  run $ABRA app volume rm
+  assert_failure
+  assert_output --partial 'no app provided'
+
+  run $ABRA app volume rm DOESNTEXIST
+  assert_failure
+  assert_output --partial 'cannot find app'
+}
+
+# bats test_tags=slow
+@test "rm error if deployed" {
+  _deploy_app
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  assert_failure
+  assert_output --partial 'is still deployed'
+
+  _undeploy_app
+}
+
+# bats test_tags=slow
+@test "remove volumes" {
+  _deploy_app
+
+  _undeploy_app
+
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  assert_success
+  assert_output --partial 'volumes removed successfully'
+}
+
+# bats test_tags=slow
+@test "remove no volumes" {
+  _deploy_app
+
+  _undeploy_app
+
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  assert_success
+  assert_output --partial 'volumes removed successfully'
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  assert_success
+  assert_output --partial 'no volumes removed'
+}

tests/integration/autocomplete.bats

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "bash autocompletion" {
+  run $ABRA autocomplete bash
+  assert_success
+  assert_exists "$ABRA_DIR/autocompletion/bash"
+}
+
+@test "zsh autocompletion" {
+  run $ABRA autocomplete zsh
+  assert_success
+  assert_exists "$ABRA_DIR/autocompletion/zsh"
+}
+
+@test "fish autocompletion" {
+  run $ABRA autocomplete fish
+  assert_success
+  assert_exists "$ABRA_DIR/autocompletion/fish"
+}
+
+@test "fizsh autocompletion" {
+  run $ABRA autocomplete fizsh
+  assert_success
+  assert_exists "$ABRA_DIR/autocompletion/zsh"
+}

tests/integration/catalogue.bats

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+# bats test_tags=slow
+@test "generate entire catalogue" {
+  run $ABRA catalogue generate
+  assert_success
+}
+
+@test "error if unstaged changes" {
+  run bash -c "echo foo >> $ABRA_DIR/catalogue/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/catalogue/foo"
+
+  run $ABRA catalogue generate
+  assert_failure
+  assert_output --partial 'locally unstaged changes'
+
+  run rm -rf "$ABRA_DIR/catalogue/foo"
+  assert_not_exists "$ABRA_DIR/catalogue/foo"
+}
+
+# bats test_tags=slow
+@test "no error if unstaged and --chaos" {
+  run bash -c "echo foo >> $ABRA_DIR/catalogue/foo"
+  assert_success
+  assert_exists "$ABRA_DIR/catalogue/foo"
+
+  run $ABRA catalogue generate gitea --chaos
+  assert_success
+
+  run rm -rf "$ABRA_DIR/catalogue/foo"
+  assert_not_exists "$ABRA_DIR/catalogue/foo"
+}
+
+# bats test_tags=slow
+@test "generate only specific recipe" {
+  run $ABRA catalogue generate gitea
+  assert_success
+}

tests/integration/dirs.bats

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+setup(){
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+
+  if [[ -d "$ABRA_DIR" ]]; then
+    rm -rf "$ABRA_DIR"
+  fi
+}
+
+@test "ABRA_DIR is overriden" {
+  ABRA_DIR="$HOME/.abra_foo"
+
+  run $ABRA app ls
+
+  # no servers yet, so will fail. however, it will run the required code which
+  # checks if it should create these base directories and that is what we want
+  assert_failure
+
+  assert_exists "$HOME/.abra_foo"
+
+  run rm -rf "$ABRA_DIR"
+  assert_success
+}
+
+@test "abra directory is created" {
+  run $ABRA app ls
+
+  # no servers yet, so will fail. however, it will run the required code which
+  # checks if it should create these base directories and that is what we want
+  assert_failure
+
+  assert_exists "$ABRA_DIR"
+  assert_exists "$ABRA_DIR/servers"
+  assert_exists "$ABRA_DIR/recipes"
+  assert_exists "$ABRA_DIR/backups"
+  assert_exists "$ABRA_DIR/vendor"
+  assert_exists "$ABRA_DIR/catalogue"
+}
+
+@test "catalogue recipe is a git repository" {
+  run $ABRA app ls
+
+  # no servers yet, so will fail. however, it will run the required code which
+  # checks if it should create these base directories and that is what we want
+  assert_failure
+
+  assert_output --partial 'local recipe catalogue is missing'
+
+  assert_exists "$ABRA_DIR/catalogue"
+  assert_exists "$ABRA_DIR/catalogue/.git"
+}

tests/integration/helpers/app.bash

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+_new_app() {
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+}
+
+_deploy_app() {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run $ABRA app ls --server "$TEST_SERVER" --status
+  assert_success
+  assert_output --partial "$TEST_APP_DOMAIN"
+  assert_output --partial 'deployed'
+}
+
+_undeploy_app() {
+  run $ABRA app undeploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run $ABRA app ls --server "$TEST_SERVER" --status
+  assert_success
+  assert_output --partial "$TEST_APP_DOMAIN"
+  assert_output --partial 'unknown'
+}
+
+_rm_app() {
+  # NOTE(d1): not asserting outcomes on teardown here since some might fail
+  # depending on what the test created. all commands run through anyway
+  if [[ -f "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" ]]; then
+    run $ABRA app undeploy "$TEST_APP_DOMAIN" --no-input
+    run $ABRA app secret remove "$TEST_APP_DOMAIN" --all --no-input
+    run $ABRA app volume remove "$TEST_APP_DOMAIN" --no-input
+    run $ABRA app remove "$TEST_APP_DOMAIN" --no-input
+  fi
+}

tests/integration/helpers/common.bash

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+_common_setup() {
+  load '/usr/lib/bats/bats-support/load'
+  load '/usr/lib/bats/bats-assert/load'
+  load '/usr/lib/bats/bats-file/load'
+
+  load "$PWD/tests/integration/helpers/app"
+  load "$PWD/tests/integration/helpers/git"
+  load "$PWD/tests/integration/helpers/recipe"
+  load "$PWD/tests/integration/helpers/server"
+
+  export ABRA="$PWD/abra"
+  export KADABRA="$PWD/kadabra"
+
+  export TEST_APP_NAME="$(basename "${BATS_TEST_FILENAME//./_}")"
+  export TEST_APP_DOMAIN="$TEST_APP_NAME.$TEST_SERVER"
+  export TEST_RECIPE="abra-integration-test-recipe"
+}

tests/integration/helpers/git.bash

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+_checkout_recipe() {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout .
+  assert_success
+}
+
+_remove_tags(){
+  run bash -c 'git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | wc -l'
+  assert_success
+  refute_output '0'
+
+  run bash -c 'git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | \
+    xargs -I{} git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -d {}'
+  assert_success
+  assert_output --partial 'Deleted tag'
+
+  run bash -c 'git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | wc -l'
+  assert_success
+  assert_output '0'
+}
+
+_reset_tags() {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" fetch --all
+  assert_success
+
+  run bash -c 'git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | wc -l'
+  assert_success
+  refute_output '0'
+}

tests/integration/helpers/recipe.bash

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+_fetch_recipe() {
+  if [[ ! -d "$ABRA_DIR/recipes/$TEST_RECIPE" ]]; then
+    run mkdir -p "$ABRA_DIR/recipes"
+    assert_success
+
+    run git clone "https://git.coopcloud.tech/coop-cloud/$TEST_RECIPE" "$ABRA_DIR/recipes/$TEST_RECIPE"
+    assert_success
+  fi
+}
+
+_reset_recipe(){
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  _fetch_recipe
+}

tests/integration/helpers/server.bash

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+_add_server() {
+  run $ABRA server add "$TEST_SERVER"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER"
+}
+
+_rm_server() {
+  run $ABRA server remove --no-input "$TEST_SERVER"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER"
+}
+
+_rm_default_server(){
+  run rm -rf "$ABRA_DIR/servers/default"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/default"
+}

tests/integration/install.bats

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+
+  if [[ -f "$HOME/.local/bin/abra" ]]; then
+    mv "$HOME/.local/bin/abra" "$HOME/.local/bin/abra_before_test"
+  fi
+}
+
+teardown(){
+  if [[ -f "$HOME/.local/bin/abra_before_test" ]]; then
+    rm -rf "$HOME/.local/bin/abra"
+    mv "$HOME/.local/bin/abra_before_test" "$HOME/.local/bin/abra"
+  fi
+}
+
+# bats test_tags=slow
+@test "install from script" {
+  run bash -c 'curl https://install.abra.coopcloud.tech | bash'
+  assert_success
+
+  assert_exists "$HOME/.local/bin/abra"
+  run "$HOME/.local/bin/abra" -v
+  assert_output --partial 'beta'
+}
+
+# bats test_tags=slow
+@test "install release candidate from script" {
+  run bash -c 'curl https://install.abra.coopcloud.tech | bash -s -- --rc'
+  assert_success
+
+  assert_exists "$HOME/.local/bin/abra"
+  run "$HOME/.local/bin/abra" -v
+  assert_output --partial '-rc'
+}