chore: go mod tidy

Medium-sized options refactor in here too!

See coop-cloud/organising#292.

.dockerignore

@@ -0,0 +1,4 @@
+Dockerfile
+.dockerignore
+*.swp
+*.swo

.drone.yml

@@ -3,35 +3,23 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.19
+    image: golang:1.20
     commands:
       - make check
 
   - name: make build
-    image: golang:1.19
+    image: golang:1.20
     commands:
       - make build
-
-  - name: make test
-    image: golang:1.19
-    commands:
-      - make test
-
-  - name: notify on failure
-    image: plugins/matrix
-    settings:
-      homeserver: https://matrix.autonomic.zone
-      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
-      userid: "@autono-bot:autonomic.zone"
-      accesstoken:
-        from_secret: autono_bot_access_token
     depends_on:
       - make check
-      - make build
+
+  - name: make test
+    image: golang:1.20
+    commands:
       - make test
-    when:
-      status:
-        - failure
+    depends_on:
+      - make check
 
   - name: fetch
     image: docker:git
@@ -45,7 +33,7 @@ steps:
       event: tag
 
   - name: release
-    image: golang:1.19
+    image: golang:1.20
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token
@@ -59,6 +47,23 @@ steps:
     when:
       event: tag
 
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: 3wordchant
+      password:
+        from_secret: git_coopcloud_tech_token_3wc
+      repo: git.coopcloud.tech/coop-cloud/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      event:
+        exclude:
+          - pull_request
+    depends_on:
+      - make check
+
 volumes:
   - name: deps
     temp: {}

.gitignore

@@ -2,6 +2,7 @@
 .e2e.env
 .envrc
 .vscode/
+/kadabra
 abra
 dist/
 tests/integration/.abra/catalogue

.goreleaser.yml

@@ -1,16 +1,19 @@
 ---
-project_name: abra
 gitea_urls:
   api: https://git.coopcloud.tech/api/v1
   download: https://git.coopcloud.tech/
   skip_tls_verify: false
+
 before:
   hooks:
     - go mod tidy
+
 builds:
-  - env:
-      - CGO_ENABLED=0
+  - id: abra
+    binary: abra
     dir: cmd/abra
+    env:
+      - CGO_ENABLED=0
     goos:
       - linux
       - darwin
@@ -26,15 +29,40 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
+
+  - id: kadabra
+    binary: kadabra
+    dir: cmd/kadabra
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+
 archives:
   - replacements:
       386: i386
       amd64: x86_64
     format: binary
+
 checksum:
   name_template: "checksums.txt"
+
 snapshot:
   name_template: "{{ incpatch .Version }}-next"
+
 changelog:
   sort: desc
   filters:

Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.20-alpine AS build
+
+ENV GOPRIVATE coopcloud.tech
+
+RUN apk add --no-cache make git gcc musl-dev
+
+COPY . /app
+
+WORKDIR /app
+
+RUN CGO_ENABLED=0 make build
+
+FROM scratch
+
+COPY --from=build /app/abra /abra
+
+ENTRYPOINT ["/abra"]

LICENSE

@@ -1,5 +1,5 @@
 Abra: The Co-op Cloud utility belt
-Copyright (C) 2022  Co-op Cloud <helo@coopcloud.tech>
+Copyright (C) 2022 Co-op Cloud <helo@coopcloud.tech>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Makefile

@@ -1,8 +1,10 @@
-ABRA    := ./cmd/abra
-COMMIT  := $(shell git rev-list -1 HEAD)
-GOPATH  := $(shell go env GOPATH)
-LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
+ABRA         := ./cmd/abra
+KADABRA      := ./cmd/kadabra
+COMMIT       := $(shell git rev-list -1 HEAD)
+GOPATH       := $(shell go env GOPATH)
+LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+
 export GOPRIVATE=coopcloud.tech
 
 all: format check build test
@@ -14,19 +16,22 @@ install:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
 build-dev:
-	@go build -ldflags=$(LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
 
 build:
-	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
 clean:
 	@rm '$(GOPATH)/bin/abra'
+	@rm '$(GOPATH)/bin/kadabra'
 
 format:
 	@gofmt -s -w .
 
 check:
-	@test -z $$(gofmt -l .) || (echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+	@test -z $$(gofmt -l .) || \
+	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
 
 test:
 	@go test ./... -cover -v

README.md

@@ -7,6 +7,6 @@ The Co-op Cloud utility belt 🎩🐇
 
 <a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-`abra` is our flagship client & command-line tool which has been developed specifically in the context of the Co-op Cloud project for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
 
 Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

cli/app/app.go

@@ -11,26 +11,27 @@ var AppCommand = cli.Command{
 	ArgsUsage:   "<domain>",
 	Description: "Functionality for managing the life cycle of your apps",
 	Subcommands: []cli.Command{
-		appNewCommand,
-		appConfigCommand,
-		appRestartCommand,
-		appDeployCommand,
-		appUpgradeCommand,
-		appUndeployCommand,
-		appRemoveCommand,
-		appCheckCommand,
-		appListCommand,
-		appPsCommand,
-		appLogsCommand,
-		appCpCommand,
-		appRunCommand,
-		appRollbackCommand,
-		appSecretCommand,
-		appVolumeCommand,
-		appVersionCommand,
-		appErrorsCommand,
-		appCmdCommand,
 		appBackupCommand,
+		appCheckCommand,
+		appCmdCommand,
+		appConfigCommand,
+		appCpCommand,
+		appDeployCommand,
+		appErrorsCommand,
+		appListCommand,
+		appLogsCommand,
+		appNewCommand,
+		appPsCommand,
+		appRemoveCommand,
+		appRestartCommand,
 		appRestoreCommand,
+		appRollbackCommand,
+		appRunCommand,
+		appSecretCommand,
+		appServicesCommand,
+		appUndeployCommand,
+		appUpgradeCommand,
+		appVersionCommand,
+		appVolumeCommand,
 	},
 }

cli/app/backup.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 
@@ -17,10 +16,12 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/system"
 	"github.com/klauspost/pgzip"
@@ -72,8 +73,14 @@ This single file can be used to restore your app. See "abra app restore" for mor
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
+		conf := runtime.New()
 
-		recipe, err := recipe.Get(app.Recipe)
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		recipe, err := recipe.Get(app.Recipe, conf)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -116,14 +123,18 @@ This single file can be used to restore your app. See "abra app restore" for mor
 
 			logrus.Infof("running backup for the %s service", serviceName)
 
-			if err := runBackup(app, serviceName, backupConfig); err != nil {
+			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
 				logrus.Fatal(err)
 			}
 		} else {
+			if len(backupConfigs) == 0 {
+				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
+			}
+
 			for serviceName, backupConfig := range backupConfigs {
 				logrus.Infof("running backup for the %s service", serviceName)
 
-				if err := runBackup(app, serviceName, backupConfig); err != nil {
+				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
 					logrus.Fatal(err)
 				}
 			}
@@ -133,17 +144,18 @@ This single file can be used to restore your app. See "abra app restore" for mor
 	},
 }
 
+// TimeStamp generates a file name friendly timestamp.
+func TimeStamp() string {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	return strings.Replace(ts, ":", "-", -1)
+}
+
 // runBackup does the actual backup logic.
-func runBackup(app config.App, serviceName string, bkConfig backupConfig) error {
+func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
 	if len(bkConfig.backupPaths) == 0 {
 		return fmt.Errorf("backup paths are empty for %s?", serviceName)
 	}
 
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
-
 	// FIXME: avoid instantiating a new CLI
 	dcli, err := command.NewDockerCli()
 	if err != nil {
@@ -182,9 +194,8 @@ func runBackup(app config.App, serviceName string, bkConfig backupConfig) error
 
 	var tempBackupPaths []string
 	for _, remoteBackupPath := range bkConfig.backupPaths {
-		timestamp := strconv.Itoa(time.Now().Nanosecond())
 		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp))
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
 		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
 
 		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
@@ -290,8 +301,7 @@ func mergeArchives(tarPaths []string, serviceName string) error {
 	var out io.Writer
 	var cout *pgzip.Writer
 
-	timestamp := strconv.Itoa(time.Now().Nanosecond())
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, timestamp))
+	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
 
 	fout, err := os.Create(localBackupPath)
 	if err != nil {

cli/app/cmd.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -52,6 +53,11 @@ Example:
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if internal.LocalCmd && internal.RemoteUser != "" {
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
 		}
@@ -129,7 +135,7 @@ Example:
 				logrus.Debug("did not detect any command arguments")
 			}
 
-			if err := runCmdRemote(app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			if err := runCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -170,12 +176,7 @@ func ensureCommand(abraSh, recipeName, execCmd string) error {
 	return nil
 }
 
-func runCmdRemote(app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
-
+func runCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
 	filters := filters.NewArgs()
 	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
 

cli/app/cp.go

@@ -14,6 +14,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -43,6 +44,11 @@ And if you want to copy that file back to your current working directory locally
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		src := c.Args().Get(1)
 		dst := c.Args().Get(2)
 		if src == "" {
@@ -88,28 +94,24 @@ And if you want to copy that file back to your current working directory locally
 				logrus.Fatalf("%s does not exist locally?", dstPath)
 			}
 		}
-		err := configureAndCp(c, app, srcPath, dstPath, service, isToContainer)
-		if err != nil {
+
+		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
 			logrus.Fatal(err)
 		}
-		return nil
 
+		return nil
 	},
 	BashComplete: autocomplete.AppNameComplete,
 }
 
 func configureAndCp(
 	c *cli.Context,
+	cl *dockerClient.Client,
 	app config.App,
 	srcPath string,
 	dstPath string,
 	service string,
 	isToContainer bool) error {
-	cl, err := client.New(app.Server)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
 	filters := filters.NewArgs()
 	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
 

cli/app/errors.go

@@ -12,6 +12,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -55,6 +56,7 @@ the logs.
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
+		conf := runtime.New()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -71,14 +73,14 @@ the logs.
 		}
 
 		if !internal.Watch {
-			if err := checkErrors(c, cl, app); err != nil {
+			if err := checkErrors(c, cl, app, conf); err != nil {
 				logrus.Fatal(err)
 			}
 			return nil
 		}
 
 		for {
-			if err := checkErrors(c, cl, app); err != nil {
+			if err := checkErrors(c, cl, app, conf); err != nil {
 				logrus.Fatal(err)
 			}
 			time.Sleep(2 * time.Second)
@@ -88,8 +90,8 @@ the logs.
 	},
 }
 
-func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
-	recipe, err := recipe.Get(app.Recipe)
+func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App, conf *runtime.Config) error {
+	recipe, err := recipe.Get(app.Recipe, conf)
 	if err != nil {
 		return err
 	}

cli/app/list.go

@@ -10,7 +10,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/ssh"
 	"coopcloud.tech/tagcmp"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -23,12 +22,12 @@ var statusFlag = &cli.BoolFlag{
 	Destination: &status,
 }
 
-var appRecipe string
+var recipeFilter string
 var recipeFlag = &cli.StringFlag{
 	Name:        "recipe, r",
 	Value:       "",
 	Usage:       "Show apps of a specific recipe",
-	Destination: &appRecipe,
+	Destination: &recipeFilter,
 }
 
 var listAppServer string
@@ -84,7 +83,7 @@ can take some time.
 			logrus.Fatal(err)
 		}
 
-		apps, err := config.GetApps(appFiles)
+		apps, err := config.GetApps(appFiles, recipeFilter)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -97,19 +96,15 @@ can take some time.
 			alreadySeen := make(map[string]bool)
 			for _, app := range apps {
 				if _, ok := alreadySeen[app.Server]; !ok {
-					if err := ssh.EnsureHostKey(app.Server); err != nil {
-						logrus.Fatal(fmt.Sprintf(internal.SSHFailMsg, app.Server))
-					}
 					alreadySeen[app.Server] = true
 				}
 			}
 
-			statuses, err = config.GetAppStatuses(appFiles)
+			statuses, err = config.GetAppStatuses(apps, internal.MachineReadable)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-			var err error
 			catl, err = recipe.ReadRecipeCatalogue()
 			if err != nil {
 				logrus.Fatal(err)
@@ -124,14 +119,14 @@ can take some time.
 			var ok bool
 			if stats, ok = allStats[app.Server]; !ok {
 				stats = serverStatus{}
-				if appRecipe == "" {
+				if recipeFilter == "" {
 					// count server, no filtering
 					totalServersCount++
 				}
 			}
 
-			if app.Recipe == appRecipe || appRecipe == "" {
-				if appRecipe != "" {
+			if app.Recipe == recipeFilter || recipeFilter == "" {
+				if recipeFilter != "" {
 					// only count server if matches filter
 					totalServersCount++
 				}
@@ -207,6 +202,7 @@ can take some time.
 			}
 			allStats[app.Server] = stats
 		}
+
 		if internal.MachineReadable {
 			jsonstring, err := json.Marshal(allStats)
 			if err != nil {
@@ -216,6 +212,7 @@ can take some time.
 			}
 			return nil
 		}
+
 		alreadySeen := make(map[string]bool)
 		for _, app := range apps {
 			if _, ok := alreadySeen[app.Server]; ok {

cli/app/logs.go

@@ -11,6 +11,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/service"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -20,12 +21,14 @@ import (
 )
 
 var logOpts = types.ContainerLogsOptions{
-	Details:    false,
-	Follow:     true,
 	ShowStderr: true,
 	ShowStdout: true,
-	Tail:       "20",
+	Since:      "",
+	Until:      "",
 	Timestamps: true,
+	Follow:     true,
+	Tail:       "20",
+	Details:    false,
 }
 
 // stackLogs lists logs for all stack services
@@ -74,18 +77,21 @@ var appLogsCommand = cli.Command{
 	Usage:     "Tail app logs",
 	Flags: []cli.Flag{
 		internal.StdErrOnlyFlag,
+		internal.SinceLogsFlag,
 		internal.DebugFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+		app := internal.ValidateApp(c, runtime.WithEnsureRecipeExists(false))
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		logOpts.Since = internal.SinceLogs
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			logrus.Debugf("tailing logs for all %s services", app.Recipe)

cli/app/remove.go

@@ -15,35 +15,43 @@ import (
 	"github.com/urfave/cli"
 )
 
-// Volumes stores the variable from VolumesFlag
-var Volumes bool
-
-// VolumesFlag is used to specify if volumes should be deleted when deleting an app
-var VolumesFlag = &cli.BoolFlag{
-	Name:        "volumes, V",
-	Destination: &Volumes,
-}
-
 var appRemoveCommand = cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
 	ArgsUsage: "<domain>",
-	Usage:     "Remove an already undeployed app",
+	Usage:     "Remove all app data, locally and remotely",
+	Description: `
+This command removes everything related to an app which is already undeployed.
+
+By default, it will prompt for confirmation before proceeding. All secrets,
+volumes and the local app env file will be deleted.
+
+Only run this command when you are sure you want to completely remove the app
+and all associated app data. This is a destructive action, Be Careful!
+
+If you would like to delete specific volumes or secrets, please use removal
+sub-commands under "app volume" and "app secret" instead.
+
+Please note, if you delete the local app env file without removing volumes and
+secrets first, Abra will *not* be able to help you remove them afterwards.
+
+To delete everything without prompt, use the "--force/-f" or the "--no-input/n"
+flag.
+`,
 	Flags: []cli.Flag{
-		VolumesFlag,
 		internal.ForceFlag,
 		internal.DebugFlag,
 		internal.NoInputFlag,
 	},
-	Before: internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
 		if !internal.Force && !internal.NoInput {
 			response := false
-			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("about to remove %s, are you sure?", app.Name),
-			}
+			msg := "ALERTA ALERTA: this will completely remove %s data and configurations locally and remotely, are you sure?"
+			prompt := &survey.Confirm{Message: fmt.Sprintf(msg, app.Name)}
 			if err := survey.AskOne(prompt, &response); err != nil {
 				logrus.Fatal(err)
 			}
@@ -84,26 +92,7 @@ var appRemoveCommand = cli.Command{
 		}
 
 		if len(secrets) > 0 {
-			var secretNamesToRemove []string
-
-			if !internal.Force && !internal.NoInput {
-				secretsPrompt := &survey.MultiSelect{
-					Message: "which secrets do you want to remove?",
-					Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-					VimMode: true,
-					Options: secretNames,
-					Default: secretNames,
-				}
-				if err := survey.AskOne(secretsPrompt, &secretNamesToRemove); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if internal.Force || internal.NoInput {
-				secretNamesToRemove = secretNames
-			}
-
-			for _, name := range secretNamesToRemove {
+			for _, name := range secretNames {
 				err := cl.SecretRemove(context.Background(), secrets[name])
 				if err != nil {
 					logrus.Fatal(err)
@@ -131,44 +120,24 @@ var appRemoveCommand = cli.Command{
 		}
 
 		if len(vols) > 0 {
-			if Volumes {
-				var removeVols []string
-				if !internal.Force && !internal.NoInput {
-					volumesPrompt := &survey.MultiSelect{
-						Message: "which volumes do you want to remove?",
-						Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-						VimMode: true,
-						Options: vols,
-						Default: vols,
-					}
-					if err := survey.AskOne(volumesPrompt, &removeVols); err != nil {
-						logrus.Fatal(err)
-					}
+			var removeVols []string
+			for _, vol := range removeVols {
+				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				if err != nil {
+					logrus.Fatal(err)
 				}
-
-				for _, vol := range removeVols {
-					err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
-					if err != nil {
-						logrus.Fatal(err)
-					}
-					logrus.Info(fmt.Sprintf("volume %s removed", vol))
-				}
-			} else {
-				logrus.Info("no volumes were removed")
+				logrus.Info(fmt.Sprintf("volume %s removed", vol))
 			}
 		} else {
-			if Volumes {
-				logrus.Info("no volumes to remove")
-			}
+			logrus.Info("no volumes to remove")
 		}
 
-		err = os.Remove(app.Path)
-		if err != nil {
+		if err = os.Remove(app.Path); err != nil {
 			logrus.Fatal(err)
 		}
+
 		logrus.Info(fmt.Sprintf("file: %s removed", app.Path))
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/restore.go

@@ -12,10 +12,12 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -54,6 +56,12 @@ Example:
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
+		conf := runtime.New()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
@@ -71,7 +79,7 @@ Example:
 			}
 		}
 
-		recipe, err := recipe.Get(app.Recipe)
+		recipe, err := recipe.Get(app.Recipe, conf)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -104,7 +112,8 @@ Example:
 		if !ok {
 			rsConfig = restoreConfig{}
 		}
-		if err := runRestore(app, backupPath, serviceName, rsConfig); err != nil {
+
+		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -113,12 +122,7 @@ Example:
 }
 
 // runRestore does the actual restore logic.
-func runRestore(app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
-
+func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
 	// FIXME: avoid instantiating a new CLI
 	dcli, err := command.NewDockerCli()
 	if err != nil {

cli/app/rollback.go

@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 
@@ -49,6 +50,7 @@ recipes.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
+		conf := runtime.New()
 
 		if !internal.Chaos {
 			if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
@@ -56,7 +58,7 @@ recipes.
 			}
 		}
 
-		r, err := recipe.Get(app.Recipe)
+		r, err := recipe.Get(app.Recipe, conf)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -178,6 +180,10 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetUpdateLabel(compose, stackName, app.Env)
 
 		if !internal.Force {
 			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {

cli/app/secret.go

@@ -48,6 +48,11 @@ var appSecretGenerateCommand = cli.Command{
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if len(c.Args()) == 1 && !allSecrets {
 			err := errors.New("missing arguments <secret>/<version> or '--all'")
 			internal.ShowSubcommandHelpAndError(c, err)
@@ -79,7 +84,7 @@ var appSecretGenerateCommand = cli.Command{
 			}
 		}
 
-		secretVals, err := secret.GenerateSecrets(secretsToCreate, app.StackName(), app.Server)
+		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -135,6 +140,11 @@ Example:
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		if len(c.Args()) != 4 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
 		}
@@ -144,7 +154,7 @@ Example:
 		data := c.Args().Get(3)
 
 		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
-		if err := client.StoreSecret(secretName, data, app.Server); err != nil {
+		if err := client.StoreSecret(cl, secretName, data, app.Server); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/services.go

@@ -0,0 +1,80 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appServicesCommand = cli.Command{
+	Name:      "services",
+	Aliases:   []string{"sr"},
+	Usage:     "Display all services of an app",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		filters, err := app.Filters(true, true)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableCol := []string{"service name", "image"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, container := range containers {
+			var containerNames []string
+			for _, containerName := range container.Names {
+				trimmed := strings.TrimPrefix(containerName, "/")
+				containerNames = append(containerNames, trimmed)
+			}
+
+			serviceShortName := service.ContainerToServiceName(container.Names, app.StackName())
+			serviceLongName := fmt.Sprintf("%s_%s", app.StackName(), serviceShortName)
+
+			tableRow := []string{
+				serviceLongName,
+				formatter.RemoveSha(container.Image),
+			}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+
+		return nil
+	},
+}

cli/app/upgrade.go

@@ -10,6 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -52,6 +53,12 @@ recipes.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
+		conf := runtime.New()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
 		if !internal.Chaos {
 			if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
@@ -59,7 +66,7 @@ recipes.
 			}
 		}
 
-		r, err := recipe.Get(app.Recipe)
+		r, err := recipe.Get(app.Recipe, conf)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -68,11 +75,6 @@ recipes.
 			logrus.Fatal(err)
 		}
 
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		logrus.Debugf("checking whether %s is already deployed", stackName)
 
 		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
@@ -189,6 +191,10 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetUpdateLabel(compose, stackName, app.Env)
 
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)

cli/app/version.go

@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
@@ -48,6 +49,7 @@ version.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
+		conf := runtime.New()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -69,7 +71,7 @@ version.
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe)
+		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, conf)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -85,12 +87,12 @@ version.
 			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"version", "service", "image", "digest"}
+		tableCol := []string{"version", "service", "image"}
 		table := formatter.CreateTable(tableCol)
 		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Digest})
+			table.Append([]string{deployedVersion, serviceName, versionMeta.Image})
 		}
 
 		table.Render()

cli/app/volume.go

@@ -26,12 +26,17 @@ var appVolumeListCommand = cli.Command{
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		filters, err := app.Filters(false, true)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		volumeList, err := client.GetVolumes(context.Background(), app.Server, filters)
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -80,12 +85,17 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		filters, err := app.Filters(false, true)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		volumeList, err := client.GetVolumes(context.Background(), app.Server, filters)
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -109,7 +119,7 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 			volumesToRemove = volumeNames
 		}
 
-		err = client.RemoveVolumes(context.Background(), app.Server, volumesToRemove, internal.Force)
+		err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/catalogue/catalogue.go

@@ -8,56 +8,16 @@ import (
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/limit"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                   true,
-	"abra-apps":              true,
-	"abra-aur":               true,
-	"abra-bash":              true,
-	"abra-capsul":            true,
-	"abra-gandi":             true,
-	"abra-hetzner":           true,
-	"apps":                   true,
-	"aur-abra-git":           true,
-	"auto-apps-json":         true,
-	"auto-mirror":            true,
-	"backup-bot":             true,
-	"backup-bot-two":         true,
-	"beta.coopcloud.tech":    true,
-	"comrade-renovate-bot":   true,
-	"coopcloud.tech":         true,
-	"coturn":                 true,
-	"docker-cp-deploy":       true,
-	"docker-dind-bats-kcov":  true,
-	"docs.coopcloud.tech":    true,
-	"drone-abra":             true,
-	"example":                true,
-	"gardening":              true,
-	"go-abra":                true,
-	"organising":             true,
-	"outline-with-patch":     true,
-	"pyabra":                 true,
-	"radicle-seed-node":      true,
-	"recipes-catalogue-json": true,
-	"recipes-wishlist":       true,
-	"recipes.coopcloud.tech": true,
-	"stack-ssh-deploy":       true,
-	"swarm-cronjob":          true,
-	"tagcmp":                 true,
-	"traefik-cert-dumper":    true,
-	"tyop":                   true,
-}
-
 var catalogueGenerateCommand = cli.Command{
 	Name:    "generate",
 	Aliases: []string{"g"},
@@ -87,7 +47,7 @@ It is quite easy to get rate limited by Docker Hub when running this command.
 If you have a Hub account you can have Abra log you in to avoid this. Pass
 "--user" and "--pass".
 
-Push your new release git.coopcloud.tech with "-p/--publish". This requires
+Push your new release to git.coopcloud.tech with "-p/--publish". This requires
 that you have permission to git push to these repositories and have your SSH
 keys configured on your account.
 `,
@@ -95,7 +55,11 @@ keys configured on your account.
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
 		if recipeName != "" {
-			internal.ValidateRecipe(c, true)
+			internal.ValidateRecipe(c)
+		}
+
+		if err := catalogue.EnsureUpToDate(); err != nil {
+			logrus.Fatal(err)
 		}
 
 		repos, err := recipe.ReadReposMetadata()
@@ -115,7 +79,7 @@ keys configured on your account.
 
 		if !internal.SkipUpdates {
 			logrus.Warn(logMsg)
-			if err := updateRepositories(repos, recipeName); err != nil {
+			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -128,7 +92,7 @@ keys configured on your account.
 				continue
 			}
 
-			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
+			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
 				catlBar.Add(1)
 				continue
 			}
@@ -257,62 +221,3 @@ var CatalogueCommand = cli.Command{
 		catalogueGenerateCommand,
 	},
 }
-
-func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error {
-	var barLength int
-	if recipeName != "" {
-		barLength = 1
-	} else {
-		barLength = len(repos)
-	}
-
-	cloneLimiter := limit.New(10)
-
-	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
-	ch := make(chan string, barLength)
-	for _, repoMeta := range repos {
-		go func(rm recipe.RepoMeta) {
-			cloneLimiter.Begin()
-			defer cloneLimiter.End()
-
-			if recipeName != "" && recipeName != rm.Name {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
-			if _, exists := CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
-
-			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
-			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
-				logrus.Fatal(err)
-			}
-
-			isClean, err := gitPkg.IsClean(recipeDir)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !isClean {
-				logrus.Fatalf("%s has locally unstaged changes", rm.Name)
-			}
-
-			if err := recipe.EnsureUpToDate(rm.Name); err != nil {
-				logrus.Fatal(err)
-			}
-
-			ch <- rm.Name
-			retrieveBar.Add(1)
-		}(repoMeta)
-	}
-
-	for range repos {
-		<-ch // wait for everything
-	}
-
-	return nil
-}

cli/cli.go

@@ -85,7 +85,7 @@ Example:
 sudo mkdir /etc/bash_completion.d/
 sudo cp %s /etc/bash_completion.d/abra
 echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
 `, autocompletionFile))
 		case "zsh":
 			fmt.Println(fmt.Sprintf(`
@@ -93,7 +93,7 @@ echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
 sudo mkdir /etc/zsh/completion.d/
 sudo cp %s /etc/zsh/completion.d/abra
 echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
 `, autocompletionFile))
 		case "fish":
 			fmt.Println(fmt.Sprintf(`
@@ -101,7 +101,7 @@ echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra
 sudo mkdir -p /etc/fish/completions
 sudo cp %s /etc/fish/completions/abra
 echo "source /etc/fish/completions/abra" >> ~/.config/fish/config.fish
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
 `, autocompletionFile))
 		}
 
@@ -189,6 +189,7 @@ func newAbraApp(version, commit string) *cli.App {
 
 		return nil
 	}
+
 	return app
 }
 

cli/internal/cli.go

@@ -328,6 +328,14 @@ var StdErrOnlyFlag = &cli.BoolFlag{
 	Destination: &StdErrOnly,
 }
 
+var SinceLogs string
+var SinceLogsFlag = &cli.StringFlag{
+	Name:        "since, S",
+	Value:       "",
+	Usage:       "tail logs since YYYY-MM-DDTHH:MM:SSZ",
+	Destination: &SinceLogs,
+}
+
 var DontWaitConverge bool
 var DontWaitConvergeFlag = &cli.BoolFlag{
 	Name:        "no-converge-checks, c",
@@ -378,73 +386,6 @@ var RemoteUserFlag = &cli.StringFlag{
 	Destination: &RemoteUser,
 }
 
-// SSHFailMsg is a hopefully helpful SSH failure message
-var SSHFailMsg = `
-Woops, Abra is unable to connect to connect to %s.
-
-Here are a few tips for debugging your local SSH config. Abra uses plain 'ol
-SSH to make connections to servers, so if your SSH config is working, Abra is
-working.
-
-In the first place, Abra will always try to read your Docker context connection
-string for SSH connection details. You can view your server context configs
-with the following command. Are they correct?
-
-    abra server ls
-
-Is your ssh-agent running? You can start it by running the following command:
-
-    eval "$(ssh-agent)"
-
-If your SSH private key loaded? You can check by running the following command:
-
-    ssh-add -L
-
-If, you can add it with:
-
-    ssh-add ~/.ssh/<private-key-part>
-
-If you are using a non-default public/private key, you can configure this in
-your ~/.ssh/config file which Abra will read in order to figure out connection
-details:
-
-Host foo.coopcloud.tech
-  Hostname foo.coopcloud.tech
-  User bar
-  Port 12345
-  IdentityFile ~/.ssh/bar@foo.coopcloud.tech
-
-If you're only using password authentication, you can use the following config:
-
-Host foo.coopcloud.tech
-  Hostname foo.coopcloud.tech
-  User bar
-  Port 12345
-  PreferredAuthentications=password
-  PubkeyAuthentication=no
-
-Good luck!
-
-`
-
-var ServerAddFailMsg = `
-Failed to add server %s.
-
-This could be caused by two things.
-
-Abra isn't picking up your SSH configuration or you need to specify it on the
-command-line (e.g you use a non-standard port or username to connect). Run
-"server add" with "-d/--debug" to learn more about what Abra is doing under the
-hood.
-
-Docker is not installed on your server. You can pass "-p/--provision" to
-install Docker and initialise Docker Swarm mode. See help output for "server
-add"
-
-See "abra server add -h" for more.
-
-`
-
 // SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
 func SubCommandBefore(c *cli.Context) error {
 	if Debug {

cli/internal/deploy.go

@@ -15,6 +15,7 @@ import (
 	"coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
@@ -24,6 +25,12 @@ import (
 // DeployAction is the main command-line action for this package
 func DeployAction(c *cli.Context) error {
 	app := ValidateApp(c)
+	conf := runtime.New()
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		logrus.Fatal(err)
+	}
 
 	if !Chaos {
 		if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
@@ -31,7 +38,7 @@ func DeployAction(c *cli.Context) error {
 		}
 	}
 
-	r, err := recipe.Get(app.Recipe)
+	r, err := recipe.Get(app.Recipe, conf)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -40,11 +47,6 @@ func DeployAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	cl, err := client.New(app.Server)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
 	logrus.Debugf("checking whether %s is already deployed", app.StackName())
 
 	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
@@ -83,7 +85,7 @@ func DeployAction(c *cli.Context) error {
 			}
 			version = formatter.SmallSHA(head.String())
 			logrus.Warn("no versions detected, using latest commit")
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe, conf); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -134,15 +136,23 @@ func DeployAction(c *cli.Context) error {
 	if err != nil {
 		logrus.Fatal(err)
 	}
+	config.ExposeAllEnv(app.StackName(), compose, app.Env)
+	config.SetRecipeLabel(compose, app.StackName(), app.Recipe)
+	config.SetChaosLabel(compose, app.StackName(), Chaos)
+	config.SetUpdateLabel(compose, app.StackName(), app.Env)
 
 	if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
 		logrus.Fatal(err)
 	}
 
 	if !NoDomainChecks {
-		domainName := app.Env["DOMAIN"]
-		if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
-			logrus.Fatal(err)
+		domainName, ok := app.Env["DOMAIN"]
+		if ok {
+			if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
 		}
 	} else {
 		logrus.Warn("skipping domain checks as requested")

cli/internal/new.go

@@ -5,14 +5,16 @@ import (
 	"path"
 
 	"coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/jsontable"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/secret"
-	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/olekukonko/tablewriter"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -24,7 +26,7 @@ type AppSecrets map[string]string
 var RecipeName string
 
 // createSecrets creates all secrets for a new app.
-func createSecrets(sanitisedAppName string) (AppSecrets, error) {
+func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets, error) {
 	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", Domain))
 	appEnv, err := config.ReadEnv(appEnvPath)
 	if err != nil {
@@ -32,7 +34,7 @@ func createSecrets(sanitisedAppName string) (AppSecrets, error) {
 	}
 
 	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
-	secrets, err := secret.GenerateSecrets(secretEnvVars, sanitisedAppName, NewAppServer)
+	secrets, err := secret.GenerateSecrets(cl, secretEnvVars, sanitisedAppName, NewAppServer)
 	if err != nil {
 		return nil, err
 	}
@@ -118,7 +120,7 @@ func ensureServerFlag() error {
 
 // NewAction is the new app creation logic
 func NewAction(c *cli.Context) error {
-	recipe := ValidateRecipeWithPrompt(c, false)
+	recipe := ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
 
 	if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
 		logrus.Fatal(err)
@@ -143,15 +145,15 @@ func NewAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	var secrets AppSecrets
-	var secretTable *tablewriter.Table
-	if Secrets {
-		if err := ssh.EnsureHostKey(NewAppServer); err != nil {
-			logrus.Fatal(err)
-		}
+	cl, err := client.New(NewAppServer)
+	if err != nil {
+		logrus.Fatal(err)
+	}
 
-		var err error
-		secrets, err = createSecrets(sanitisedAppName)
+	var secrets AppSecrets
+	var secretTable *jsontable.JSONTable
+	if Secrets {
+		secrets, err := createSecrets(cl, sanitisedAppName)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/internal/validate.go

@@ -9,7 +9,7 @@ import (
 	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/ssh"
+	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -19,14 +19,15 @@ import (
 var AppName string
 
 // ValidateRecipe ensures the recipe arg is valid.
-func ValidateRecipe(c *cli.Context, ensureLatest bool) recipe.Recipe {
+func ValidateRecipe(c *cli.Context, opts ...runtime.Option) recipe.Recipe {
 	recipeName := c.Args().First()
+	conf := runtime.New(opts...)
 
 	if recipeName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName)
+	chosenRecipe, err := recipe.Get(recipeName, conf)
 	if err != nil {
 		if c.Command.Name == "generate" {
 			if strings.Contains(err.Error(), "missing a compose") {
@@ -34,12 +35,15 @@ func ValidateRecipe(c *cli.Context, ensureLatest bool) recipe.Recipe {
 			}
 			logrus.Warn(err)
 		} else {
-			logrus.Fatal(err)
+			if strings.Contains(err.Error(), "template_driver is not allowed") {
+				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
+			}
+			logrus.Fatalf("unable to validate recipe: %s", err)
 		}
 	}
 
-	if ensureLatest {
-		if err := recipe.EnsureLatest(recipeName); err != nil {
+	if conf.EnsureRecipeLatest {
+		if err := recipe.EnsureLatest(recipeName, conf); err != nil {
 			logrus.Fatal(err)
 		}
 	}
@@ -51,8 +55,9 @@ func ValidateRecipe(c *cli.Context, ensureLatest bool) recipe.Recipe {
 
 // ValidateRecipeWithPrompt ensures a recipe argument is present before
 // validating, asking for input if required.
-func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
+func ValidateRecipeWithPrompt(c *cli.Context, opts ...runtime.Option) recipe.Recipe {
 	recipeName := c.Args().First()
+	conf := runtime.New(opts...)
 
 	if recipeName == "" && !NoInput {
 		var recipes []string
@@ -100,13 +105,13 @@ func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
 		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName)
+	chosenRecipe, err := recipe.Get(recipeName, conf)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	if ensureLatest {
-		if err := recipe.EnsureLatest(recipeName); err != nil {
+	if conf.EnsureRecipeLatest {
+		if err := recipe.EnsureLatest(recipeName, conf); err != nil {
 			logrus.Fatal(err)
 		}
 	}
@@ -117,8 +122,9 @@ func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
 }
 
 // ValidateApp ensures the app name arg is valid.
-func ValidateApp(c *cli.Context) config.App {
+func ValidateApp(c *cli.Context, opts ...runtime.Option) config.App {
 	appName := c.Args().First()
+	conf := runtime.New(opts...)
 
 	if AppName != "" {
 		appName = AppName
@@ -134,11 +140,7 @@ func ValidateApp(c *cli.Context) config.App {
 		logrus.Fatal(err)
 	}
 
-	if err := recipe.EnsureExists(app.Recipe); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ssh.EnsureHostKey(app.Server); err != nil {
+	if err := recipe.EnsureExists(app.Recipe, conf); err != nil {
 		logrus.Fatal(err)
 	}
 

cli/recipe/fetch.go

@@ -0,0 +1,40 @@
+package recipe
+
+import (
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeFetchCommand = cli.Command{
+	Name:        "fetch",
+	Usage:       "Fetch recipe local copies",
+	Aliases:     []string{"f"},
+	ArgsUsage:   "[<recipe>]",
+	Description: "Fetchs all recipes without arguments.",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+			return nil // ValidateRecipe ensures latest checkout
+		}
+
+		repos, err := recipe.ReadReposMetadata()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/lint.go

@@ -24,44 +24,68 @@ var recipeLintCommand = cli.Command{
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, true)
+		recipe := internal.ValidateRecipe(c)
 
 		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
-		tableCol := []string{"ref", "rule", "satisfied", "severity", "resolve"}
+		tableCol := []string{"ref", "rule", "severity", "satisfied", "skipped", "resolve"}
 		table := formatter.CreateTable(tableCol)
 
 		hasError := false
 		bar := formatter.CreateProgressbar(-1, "running recipe lint rules...")
 		for level := range lint.LintRules {
 			for _, rule := range lint.LintRules[level] {
-				ok, err := rule.Function(recipe)
-				if err != nil {
-					logrus.Warn(err)
+				if internal.OnlyErrors && rule.Level != "error" {
+					logrus.Debugf("skipping %s, does not have level \"error\"", rule.Ref)
+					continue
 				}
 
-				if !ok && rule.Level == "error" {
-					hasError = true
+				skipped := false
+				if rule.Skip(recipe) {
+					skipped = true
 				}
 
-				var result string
-				if ok {
-					result = "yes"
-				} else {
-					result = "NO"
+				skippedOutput := "-"
+				if skipped {
+					skippedOutput = "yes"
 				}
 
-				if internal.OnlyErrors {
-					if !ok && rule.Level == "error" {
-						table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
-						bar.Add(1)
+				satisfied := false
+				if !skipped {
+					ok, err := rule.Function(recipe)
+					if err != nil {
+						logrus.Warn(err)
+					}
+
+					if !ok && rule.Level == "error" {
+						hasError = true
+					}
+
+					if ok {
+						satisfied = true
 					}
-				} else {
-					table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
-					bar.Add(1)
 				}
+
+				satisfiedOutput := "yes"
+				if !satisfied {
+					satisfiedOutput = "NO"
+					if skipped {
+						satisfiedOutput = "-"
+					}
+				}
+
+				table.Append([]string{
+					rule.Ref,
+					rule.Description,
+					rule.Level,
+					satisfiedOutput,
+					skippedOutput,
+					rule.HowToResolve,
+				})
+
+				bar.Add(1)
 			}
 		}
 

cli/recipe/list.go

@@ -27,6 +27,7 @@ var recipeListCommand = cli.Command{
 	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
+		internal.MachineReadableFlag,
 		patternFlag,
 	},
 	Before: internal.SubCommandBefore,
@@ -66,10 +67,14 @@ var recipeListCommand = cli.Command{
 			}
 		}
 
-		table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
-
 		if table.NumLines() > 0 {
-			table.Render()
+			if internal.MachineReadable {
+				table.SetCaption(false, "")
+				table.JSONRender()
+			} else {
+				table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
+				table.Render()
+			}
 		}
 
 		return nil

cli/recipe/recipe.go

@@ -22,12 +22,13 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 "abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands.
 `,
 	Subcommands: []cli.Command{
-		recipeListCommand,
-		recipeVersionCommand,
-		recipeReleaseCommand,
-		recipeNewCommand,
-		recipeUpgradeCommand,
-		recipeSyncCommand,
+		recipeFetchCommand,
 		recipeLintCommand,
+		recipeListCommand,
+		recipeNewCommand,
+		recipeReleaseCommand,
+		recipeSyncCommand,
+		recipeUpgradeCommand,
+		recipeVersionCommand,
 	},
 }

cli/recipe/release.go

@@ -13,6 +13,7 @@ import (
 	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
@@ -58,7 +59,7 @@ your SSH keys configured on your account.
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, false)
+		recipe := internal.ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
@@ -373,7 +374,7 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip
 	}
 
 	if lastGitTag.String() == tagString {
-		logrus.Fatalf("latest git tag (%s) and synced lable (%s) are the same?", lastGitTag, tagString)
+		logrus.Fatalf("latest git tag (%s) and synced label (%s) are the same?", lastGitTag, tagString)
 	}
 
 	if !internal.NoInput {

cli/recipe/sync.go

@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -41,7 +42,7 @@ auto-generate it for you. The <recipe> configuration will be updated on the
 local file system.
 `,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, false)
+		recipe := internal.ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
 
 		mainApp, err := internal.GetMainAppImage(recipe)
 		if err != nil {

cli/recipe/upgrade.go

@@ -59,7 +59,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, true)
+		recipe := internal.ValidateRecipeWithPrompt(c)
 
 		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
 			logrus.Fatal(err)

cli/recipe/version.go

@@ -5,6 +5,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/formatter"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -20,7 +21,7 @@ var recipeVersionCommand = cli.Command{
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, false)
+		recipe := internal.ValidateRecipe(c, runtime.WithEnsureRecipeLatest(false))
 
 		catalogue, err := recipePkg.ReadRecipeCatalogue()
 		if err != nil {
@@ -32,13 +33,13 @@ var recipeVersionCommand = cli.Command{
 			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
 		}
 
-		tableCol := []string{"Version", "Service", "Image", "Tag", "Digest"}
+		tableCol := []string{"Version", "Service", "Image", "Tag"}
 		table := formatter.CreateTable(tableCol)
 
 		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
 			for tag, meta := range recipeMeta.Versions[i] {
 				for service, serviceMeta := range meta {
-					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag, serviceMeta.Digest})
+					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag})
 				}
 			}
 		}

cli/server/add.go

@@ -1,14 +1,9 @@
 package server
 
 import (
-	"context"
 	"errors"
-	"fmt"
 	"os"
-	"os/exec"
-	"os/user"
 	"path/filepath"
-	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
@@ -16,34 +11,11 @@ import (
 	contextPkg "coopcloud.tech/abra/pkg/context"
 	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/server"
-	"coopcloud.tech/abra/pkg/ssh"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/swarm"
-	dockerClient "github.com/docker/docker/client"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var (
-	dockerInstallMsg = `
-A docker installation cannot be found on %s. This is a required system
-dependency for running Co-op Cloud apps on your server. If you would like, Abra
-can attempt to install Docker for you using the upstream non-interactive
-installation script.
-
-See the following documentation for more:
-
-    https://docs.docker.com/engine/install/debian/#install-using-the-convenience-script
-
-N.B Docker doesn't recommend it for production environments but many use it for
-such purposes. Docker stable is now installed by default by this script. The
-source for this script can be seen here:
-
-    https://github.com/docker/docker-install
-`
-)
-
 var local bool
 var localFlag = &cli.BoolFlag{
 	Name:        "local, l",
@@ -51,106 +23,36 @@ var localFlag = &cli.BoolFlag{
 	Destination: &local,
 }
 
-var provision bool
-var provisionFlag = &cli.BoolFlag{
-	Name:        "provision, p",
-	Usage:       "Provision server so it can deploy apps",
-	Destination: &provision,
-}
-
-var sshAuth string
-var sshAuthFlag = &cli.StringFlag{
-	Name:        "ssh-auth, s",
-	Value:       "identity-file",
-	Usage:       "Select SSH authentication method (identity-file, password)",
-	Destination: &sshAuth,
-}
-
-var askSudoPass bool
-var askSudoPassFlag = &cli.BoolFlag{
-	Name:        "ask-sudo-pass, a",
-	Usage:       "Ask for sudo password",
-	Destination: &askSudoPass,
-}
-
 func cleanUp(domainName string) {
-	logrus.Warnf("cleaning up context for %s", domainName)
-	if err := client.DeleteContext(domainName); err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Warnf("cleaning up server directory for %s", domainName)
-	if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, domainName)); err != nil {
-		logrus.Fatal(err)
-	}
-}
-
-func installDockerLocal(c *cli.Context) error {
-	fmt.Println(fmt.Sprintf(dockerInstallMsg, "this local server"))
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: fmt.Sprintf("attempt install docker on local server?"),
-	}
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	for _, exe := range []string{"wget", "bash"} {
-		exists, err := ensureLocalExecutable(exe)
-		if err != nil {
-			return err
-		}
-		if !exists {
-			return fmt.Errorf("%s missing, please install it", exe)
+	if domainName != "default" {
+		logrus.Infof("cleaning up context for %s", domainName)
+		if err := client.DeleteContext(domainName); err != nil {
+			logrus.Fatal(err)
 		}
 	}
 
-	cmd := exec.Command("bash", "-c", "wget -O- https://get.docker.com | bash")
-	if err := internal.RunCmd(cmd); err != nil {
-		return err
-	}
+	logrus.Infof("attempting to clean up server directory for %s", domainName)
 
-	return nil
-}
-
-func newLocalServer(c *cli.Context, domainName string) error {
-	if err := createServerDir(domainName); err != nil {
-		return err
-	}
-
-	cl, err := newClient(c, domainName)
+	serverDir := filepath.Join(config.SERVERS_DIR, domainName)
+	files, err := config.GetAllFilesInDirectory(serverDir)
 	if err != nil {
-		return err
+		logrus.Fatalf("unable to list files in %s: %s", serverDir, err)
 	}
 
-	if provision {
-		exists, err := ensureLocalExecutable("docker")
-		if err != nil {
-			return err
-		}
-
-		if !exists {
-			if err := installDockerLocal(c); err != nil {
-				return err
-			}
-		}
-
-		if err := initSwarmLocal(c, cl, domainName); err != nil {
-			if !strings.Contains(err.Error(), "proxy already exists") {
-				logrus.Fatal(err)
-			}
-		}
+	if len(files) > 0 {
+		logrus.Warnf("aborting clean up of %s because it is not empty", serverDir)
+		return
 	}
 
-	logrus.Info("local server has been added")
-
-	return nil
+	if err := os.RemoveAll(serverDir); err != nil {
+		logrus.Fatal(err)
+	}
 }
 
+// newContext creates a new internal Docker context for a server. This is how
+// Docker manages SSH connection details. These are stored to disk in
+// ~/.docker. Abra can manage this completely for the user, so it's an
+// implementation detail.
 func newContext(c *cli.Context, domainName, username, port string) error {
 	store := contextPkg.NewDefaultDockerContextStore()
 	contexts, err := store.Store.List()
@@ -174,187 +76,7 @@ func newContext(c *cli.Context, domainName, username, port string) error {
 	return nil
 }
 
-func newClient(c *cli.Context, domainName string) (*dockerClient.Client, error) {
-	cl, err := client.New(domainName)
-	if err != nil {
-		return &dockerClient.Client{}, err
-	}
-	return cl, nil
-}
-
-func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, domainName string) error {
-	exists, err := ensureRemoteExecutable("docker", sshCl)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		fmt.Println(fmt.Sprintf(dockerInstallMsg, domainName))
-
-		response := false
-		prompt := &survey.Confirm{
-			Message: fmt.Sprintf("attempt install docker on %s?", domainName),
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return err
-		}
-
-		if !response {
-			logrus.Fatal("exiting as requested")
-		}
-
-		exes := []string{"wget", "bash"}
-		if askSudoPass {
-			exes = append(exes, "ssh-askpass")
-		}
-
-		for _, exe := range exes {
-			exists, err := ensureRemoteExecutable(exe, sshCl)
-			if err != nil {
-				return err
-			}
-			if !exists {
-				return fmt.Errorf("%s missing on remote, please install it", exe)
-			}
-		}
-
-		var sudoPass string
-		if askSudoPass {
-			cmd := "wget -O- https://get.docker.com | bash"
-
-			prompt := &survey.Password{
-				Message: "sudo password?",
-			}
-
-			if err := survey.AskOne(prompt, &sudoPass); err != nil {
-				return err
-			}
-
-			logrus.Debugf("running %s on %s now with sudo password", cmd, domainName)
-
-			if sudoPass == "" {
-				return fmt.Errorf("missing sudo password but requested --ask-sudo-pass?")
-			}
-
-			logrus.Warn("installing docker, this could take some time...")
-
-			if err := ssh.RunSudoCmd(cmd, sudoPass, sshCl); err != nil {
-				fmt.Print(fmt.Sprintf(`
-Abra was unable to bootstrap Docker, see below for logs:
-
-
-%s
-
-If nothing works, you can try running the Docker install script manually on your server:
-
-    wget -O- https://get.docker.com | bash
-
-`, string(err.Error())))
-				logrus.Fatal("Process exited with status 1")
-			}
-
-			logrus.Infof("docker is installed on %s", domainName)
-
-			remoteUser := sshCl.SSHClient.Conn.User()
-			logrus.Infof("adding %s to docker group", remoteUser)
-			permsCmd := fmt.Sprintf("sudo usermod -aG docker %s", remoteUser)
-			if err := ssh.RunSudoCmd(permsCmd, sudoPass, sshCl); err != nil {
-				return err
-			}
-		} else {
-			cmd := "wget -O- https://get.docker.com | bash"
-
-			logrus.Debugf("running %s on %s now without sudo password", cmd, domainName)
-
-			logrus.Warn("installing docker, this could take some time...")
-
-			if out, err := sshCl.Exec(cmd); err != nil {
-				fmt.Print(fmt.Sprintf(`
-Abra was unable to bootstrap Docker, see below for logs:
-
-
-%s
-
-This could be due to several reasons. One of the most common is that your
-server user account does not have sudo access, and if it does, you need to pass
-"--ask-sudo-pass" in order to supply Abra with your password.
-
-If nothing works, you try running the Docker install script manually on your server:
-
-    wget -O- https://get.docker.com | bash
-
-`, string(out)))
-				logrus.Fatal(err)
-			}
-
-			logrus.Infof("docker is installed on %s", domainName)
-		}
-	}
-
-	return nil
-}
-
-func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string) error {
-	initReq := swarm.InitRequest{ListenAddr: "0.0.0.0:2377"}
-	if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
-		if strings.Contains(err.Error(), "is already part of a swarm") ||
-			strings.Contains(err.Error(), "must specify a listening address") {
-			logrus.Infof("swarm mode already initialised on %s", domainName)
-		} else {
-			return err
-		}
-	} else {
-		logrus.Infof("initialised swarm mode on local server")
-	}
-
-	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
-	if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
-		if !strings.Contains(err.Error(), "proxy already exists") {
-			return err
-		}
-		logrus.Info("swarm overlay network already created on local server")
-	} else {
-		logrus.Infof("swarm overlay network created on local server")
-	}
-
-	return nil
-}
-
-func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error {
-	ipv4, err := dns.EnsureIPv4(domainName)
-	if err != nil {
-		return err
-	}
-
-	initReq := swarm.InitRequest{
-		ListenAddr:    "0.0.0.0:2377",
-		AdvertiseAddr: ipv4,
-	}
-	if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
-		if strings.Contains(err.Error(), "is already part of a swarm") ||
-			strings.Contains(err.Error(), "must specify a listening address") {
-			logrus.Infof("swarm mode already initialised on %s", domainName)
-		} else {
-			return err
-		}
-	} else {
-		logrus.Infof("initialised swarm mode on %s", domainName)
-	}
-
-	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
-	if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
-		if !strings.Contains(err.Error(), "proxy already exists") {
-			return err
-		}
-		logrus.Infof("swarm overlay network already created on %s", domainName)
-	} else {
-		logrus.Infof("swarm overlay network created on %s", domainName)
-	}
-
-	return nil
-}
-
+// createServerDir creates the ~/.abra/servers/... directory for a new server.
 func createServerDir(domainName string) error {
 	if err := server.CreateServerDir(domainName); err != nil {
 		if !os.IsExist(err) {
@@ -362,6 +84,7 @@ func createServerDir(domainName string) error {
 		}
 		logrus.Debugf("server dir for %s already created", domainName)
 	}
+
 	return nil
 }
 
@@ -370,34 +93,21 @@ var serverAddCommand = cli.Command{
 	Aliases: []string{"a"},
 	Usage:   "Add a server to your configuration",
 	Description: `
-Add a new server to your configuration so that it can be managed by Abra. This
-command can also provision your server ("--provision/-p") with a Docker
-installation so that it is capable of hosting Co-op Cloud apps.
+Add a new server to your configuration so that it can be managed by Abra.
 
-Abra will default to expecting that you have a running ssh-agent and are using
-SSH keys to connect to your new server. Abra will also read your SSH config
-(matching "Host" as <domain>). SSH connection details precedence follows as
-such: command-line > SSH config > guessed defaults.
+Abra uses the SSH command-line to discover connection details for your server.
+It is advised to configure an entry per-host in your ~/.ssh/config for each
+server. For example:
 
-If you have no SSH key configured for this host and are instead using password
-authentication, you may pass "--ssh-auth password" to have Abra ask you for the
-password. "--ask-sudo-pass" may be passed if you run your provisioning commands
-via sudo privilege escalation.
+Host example.com
+  Hostname example.com
+  User exampleUser
+  Port 12345
+  IdentityFile ~/.ssh/example@somewhere
 
-The <domain> argument must be a publicy accessible domain name which points to
-your server. You should have working SSH access to this server already, Abra
-will assume port 22 and will use your current system username to make an
-initial connection. You can use the <user> and <port> arguments to adjust this.
+Abra can then load SSH connection details from this configuratiion with:
 
-Example:
-
-    abra server add varia.zone glodemodem 12345 -p
-
-Abra will construct the following SSH connection and Docker context:
-
-    ssh://globemodem@varia.zone:12345
-
-All communication between Abra and the server will use this SSH connection.
+    abra server add example.com
 
 If "--local" is passed, then Abra assumes that the current local server is
 intended as the target server. This is useful when you want to have your entire
@@ -408,104 +118,64 @@ developer machine.
 		internal.DebugFlag,
 		internal.NoInputFlag,
 		localFlag,
-		provisionFlag,
-		sshAuthFlag,
-		askSudoPassFlag,
 	},
 	Before:    internal.SubCommandBefore,
-	ArgsUsage: "<domain> [<user>] [<port>]",
+	ArgsUsage: "<domain>",
 	Action: func(c *cli.Context) error {
 		if len(c.Args()) > 0 && local || !internal.ValidateSubCmdFlags(c) {
 			err := errors.New("cannot use <domain> and --local together")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
-		if sshAuth != "password" && sshAuth != "identity-file" {
-			err := errors.New("--ssh-auth only accepts identity-file or password")
-			internal.ShowSubcommandHelpAndError(c, err)
+		var domainName string
+		if local {
+			domainName = "default"
+		} else {
+			domainName = internal.ValidateDomain(c)
 		}
 
-		domainName := internal.ValidateDomain(c)
-
 		if local {
-			if err := newLocalServer(c, "default"); err != nil {
+			if err := createServerDir(domainName); err != nil {
 				logrus.Fatal(err)
 			}
+
+			logrus.Infof("attempting to create client for %s", domainName)
+			if _, err := client.New(domainName); err != nil {
+				cleanUp(domainName)
+				logrus.Fatal(err)
+			}
+
+			logrus.Info("local server added")
+
 			return nil
 		}
 
-		username := c.Args().Get(1)
-		if username == "" {
-			systemUser, err := user.Current()
-			if err != nil {
-				return err
-			}
-			username = systemUser.Username
-		}
-
-		port := c.Args().Get(2)
-		if port == "" {
-			port = "22"
+		if _, err := dns.EnsureIPv4(domainName); err != nil {
+			logrus.Fatal(err)
 		}
 
 		if err := createServerDir(domainName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := newContext(c, domainName, username, port); err != nil {
+		hostConfig, err := sshPkg.GetHostConfig(domainName)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		cl, err := newClient(c, domainName)
-		if err != nil {
+		if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("attempting to create client for %s", domainName)
+		if _, err := client.New(domainName); err != nil {
 			cleanUp(domainName)
 			logrus.Debugf("failed to construct client for %s, saw %s", domainName, err.Error())
-			logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
+			logrus.Fatal(sshPkg.Fatal(domainName, err))
 		}
 
-		if provision {
-			logrus.Debugf("attempting to construct SSH client for %s", domainName)
-			sshCl, err := ssh.New(domainName, sshAuth, username, port)
-			if err != nil {
-				cleanUp(domainName)
-				logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
-			}
-			defer sshCl.Close()
-			logrus.Debugf("successfully created SSH client for %s", domainName)
-
-			if err := installDocker(c, cl, sshCl, domainName); err != nil {
-				logrus.Fatal(err)
-			}
-			if err := initSwarm(c, cl, domainName); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		if _, err := cl.Info(context.Background()); err != nil {
-			cleanUp(domainName)
-			logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
-		}
+		logrus.Infof("%s added", domainName)
 
 		return nil
 	},
 }
-
-// ensureLocalExecutable ensures that an executable is present on the local machine
-func ensureLocalExecutable(exe string) (bool, error) {
-	out, err := exec.Command("which", exe).Output()
-	if err != nil {
-		return false, err
-	}
-
-	return string(out) != "", nil
-}
-
-// ensureRemoteExecutable ensures that an executable is present on a remote machine
-func ensureRemoteExecutable(exe string, sshCl *ssh.Client) (bool, error) {
-	out, err := sshCl.Exec(fmt.Sprintf("which %s", exe))
-	if err != nil && string(out) != "" {
-		return false, err
-	}
-
-	return string(out) != "", nil
-}

cli/server/list.go

@@ -12,12 +12,22 @@ import (
 	"github.com/urfave/cli"
 )
 
+var problemsFilter bool
+
+var problemsFilterFlag = &cli.BoolFlag{
+	Name:        "problems, p",
+	Usage:       "Show only servers with potential connection problems",
+	Destination: &problemsFilter,
+}
+
 var serverListCommand = cli.Command{
 	Name:    "list",
 	Aliases: []string{"ls"},
 	Usage:   "List managed servers",
 	Flags: []cli.Flag{
+		problemsFilterFlag,
 		internal.DebugFlag,
+		internal.MachineReadableFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
@@ -29,8 +39,11 @@ var serverListCommand = cli.Command{
 
 		tableColumns := []string{"name", "host", "user", "port"}
 		table := formatter.CreateTable(tableColumns)
-		defer table.Render()
-
+		if internal.MachineReadable {
+			defer table.JSONRender()
+		} else {
+			defer table.Render()
+		}
 		serverNames, err := config.ReadServerNames()
 		if err != nil {
 			logrus.Fatal(err)
@@ -44,6 +57,7 @@ var serverListCommand = cli.Command{
 					// No local context found, we can continue safely
 					continue
 				}
+
 				if ctx.Name == serverName {
 					sp, err := ssh.ParseURL(endpoint)
 					if err != nil {
@@ -52,6 +66,7 @@ var serverListCommand = cli.Command{
 					row = []string{serverName, sp.Host, sp.User, sp.Port}
 				}
 			}
+
 			if len(row) == 0 {
 				if serverName == "default" {
 					row = []string{serverName, "local", "n/a", "n/a"}
@@ -59,7 +74,14 @@ var serverListCommand = cli.Command{
 					row = []string{serverName, "unknown", "unknown", "unknown"}
 				}
 			}
-			table.Append(row)
+
+			if problemsFilter {
+				if row[1] == "unknown" {
+					table.Append(row)
+				}
+			} else {
+				table.Append(row)
+			}
 		}
 
 		return nil

cli/updater/updater.go

@@ -0,0 +1,497 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/runtime"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerclient "github.com/docker/docker/client"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+const SERVER = "localhost"
+
+var majorUpdate bool
+var majorFlag = &cli.BoolFlag{
+	Name:        "major, m",
+	Usage:       "Also check for major updates",
+	Destination: &majorUpdate,
+}
+
+var updateAll bool
+var allFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Usage:       "Update all deployed apps",
+	Destination: &updateAll,
+}
+
+// Notify checks for available upgrades
+var Notify = cli.Command{
+	Name:    "notify",
+	Aliases: []string{"n"},
+	Usage:   "Check for available upgrades",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		majorFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+It reads the deployed app versions and looks for new versions in the recipe
+catalogue. If a new patch/minor version is available, a notification is
+printed. To include major versions use the --major flag.
+`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if recipeName != "" {
+				_, err = getLatestUpgrade(cl, stackName, recipeName)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+// UpgradeApp upgrades apps.
+var UpgradeApp = cli.Command{
+	Name:      "upgrade",
+	Aliases:   []string{"u"},
+	Usage:     "Upgrade apps",
+	ArgsUsage: "<stack-name> <recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.ChaosFlag,
+		majorFlag,
+		allFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Upgrade an app by specifying its stack name and recipe. By passing "--all"
+instead, every deployed app is upgraded. For each apps with enabled auto
+updates the deployed version is compared with the current recipe catalogue
+version. If a new patch/minor version is available, the app is upgraded. To
+include major versions use the "--major" flag. Don't do that, it will probably
+break things. Only apps that are not deployed with "--chaos" are upgraded, to
+update chaos deployments use the "--chaos" flag. Use it with care.
+`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		conf := runtime.New()
+
+		if !updateAll {
+			stackName := c.Args().Get(0)
+			recipeName := c.Args().Get(1)
+			err = tryUpgrade(cl, stackName, recipeName, conf)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			return nil
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			err = tryUpgrade(cl, stackName, recipeName, conf)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+// getLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
+func getLabel(cl *dockerclient.Client, stackName string, label string) (string, error) {
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return "", err
+	}
+
+	for _, service := range services {
+		labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+		if labelValue, ok := service.Spec.Labels[labelKey]; ok {
+			return labelValue, nil
+		}
+	}
+
+	logrus.Debugf("no %s label found for %s", label, stackName)
+
+	return "", nil
+}
+
+// getBoolLabel reads a boolean docker label.
+func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool, error) {
+	lableValue, err := getLabel(cl, stackName, label)
+	if err != nil {
+		return false, err
+	}
+
+	if lableValue != "" {
+		value, err := strconv.ParseBool(lableValue)
+		if err != nil {
+			return false, err
+		}
+
+		return value, nil
+	}
+
+	logrus.Debugf("Boolean label %s could not be found for %s, set default to false.", label, stackName)
+
+	return false, nil
+}
+
+// getEnv reads env variables from docker services.
+func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) {
+	envMap := make(map[string]string)
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, service := range services {
+		envList := service.Spec.TaskTemplate.ContainerSpec.Env
+		for _, envString := range envList {
+			splitString := strings.SplitN(envString, "=", 2)
+			if len(splitString) != 2 {
+				logrus.Debugf("can't separate key from value: %s (this variable is probably unset)", envString)
+				continue
+			}
+			k := splitString[0]
+			v := splitString[1]
+			logrus.Debugf("For %s read env %s with value: %s from docker service", stackName, k, v)
+			envMap[k] = v
+		}
+	}
+
+	return envMap, nil
+}
+
+// getLatestUpgrade returns the latest available version for an app respecting
+// the "--major" flag if it is newer than the currently deployed version.
+func getLatestUpgrade(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	deployedVersion, err := getDeployedVersion(cl, stackName, recipeName)
+	if err != nil {
+		return "", err
+	}
+
+	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion)
+	if err != nil {
+		return "", err
+	}
+
+	if len(availableUpgrades) == 0 {
+		logrus.Debugf("no available upgrades for %s", stackName)
+		return "", nil
+	}
+
+	var chosenUpgrade string
+	if len(availableUpgrades) > 0 {
+		chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+		logrus.Infof("%s (%s) can be upgraded from version %s to %s", stackName, recipeName, deployedVersion, chosenUpgrade)
+	}
+
+	return chosenUpgrade, nil
+}
+
+// getDeployedVersion returns the currently deployed version of an app.
+func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	logrus.Debugf("Retrieve deployed version whether %s is already deployed", stackName)
+
+	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+	if err != nil {
+		return "", err
+	}
+
+	if !isDeployed {
+		return "", fmt.Errorf("%s is not deployed?", stackName)
+	}
+
+	if deployedVersion == "unknown" {
+		return "", fmt.Errorf("failed to determine deployed version of %s", stackName)
+	}
+
+	return deployedVersion, nil
+}
+
+// getAvailableUpgrades returns all available versions of an app that are newer
+// than the deployed version. It only includes major upgrades if the "--major"
+// flag is set.
+func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
+	deployedVersion string) ([]string, error) {
+	catl, err := recipe.ReadRecipeCatalogue()
+	if err != nil {
+		return nil, err
+	}
+
+	versions, err := recipe.GetRecipeCatalogueVersions(recipeName, catl)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(versions) == 0 {
+		return nil, fmt.Errorf("no published releases for %s in the recipe catalogue?", recipeName)
+	}
+
+	var availableUpgrades []string
+	for _, version := range versions {
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return nil, err
+		}
+
+		versionDelta, err := parsedDeployedVersion.UpgradeDelta(parsedVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		if 0 < versionDelta.UpgradeType() && (versionDelta.UpgradeType() < 4 || majorUpdate) {
+			availableUpgrades = append(availableUpgrades, version)
+		}
+	}
+
+	logrus.Debugf("Available updates for %s: %s", stackName, availableUpgrades)
+
+	return availableUpgrades, nil
+}
+
+// processRecipeRepoVersion clones, pulls, checks out the version and lints the
+// recipe repository.
+func processRecipeRepoVersion(recipeName, version string, conf *runtime.Config) error {
+	if err := recipe.EnsureExists(recipeName, conf); err != nil {
+		return err
+	}
+
+	if err := recipe.EnsureUpToDate(recipeName); err != nil {
+		return err
+	}
+
+	if err := recipe.EnsureVersion(recipeName, version); err != nil {
+		return err
+	}
+
+	if r, err := recipe.Get(recipeName, conf); err != nil {
+		return err
+	} else if err := lint.LintForErrors(r); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// mergeAbraShEnv merges abra.sh env vars into the app env vars.
+func mergeAbraShEnv(recipeName string, env config.AppEnv) error {
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		return err
+	}
+
+	for k, v := range abraShEnv {
+		logrus.Debugf("read v:%s k: %s", v, k)
+		env[k] = v
+	}
+
+	return nil
+}
+
+// createDeployConfig merges and enriches the compose config for the deployment.
+func createDeployConfig(recipeName string, stackName string, env config.AppEnv) (*composetypes.Config, stack.Deploy, error) {
+	env["STACK_NAME"] = stackName
+
+	deployOpts := stack.Deploy{
+		Namespace:    stackName,
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+
+	composeFiles, err := config.GetAppComposeFiles(recipeName, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+
+	deployOpts.Composefiles = composeFiles
+	compose, err := config.GetAppComposeConfig(stackName, deployOpts, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+
+	config.ExposeAllEnv(stackName, compose, env)
+
+	// after the upgrade the deployment won't be in chaos state anymore
+	config.SetChaosLabel(compose, stackName, false)
+	config.SetRecipeLabel(compose, stackName, recipeName)
+	config.SetUpdateLabel(compose, stackName, env)
+
+	return compose, deployOpts, nil
+}
+
+// tryUpgrade performs the upgrade if all the requirements are fulfilled.
+func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *runtime.Config) error {
+	if recipeName == "" {
+		logrus.Debugf("Don't update %s due to missing recipe name", stackName)
+		return nil
+	}
+
+	chaos, err := getBoolLabel(cl, stackName, "chaos")
+	if err != nil {
+		return err
+	}
+
+	if chaos && !internal.Chaos {
+		logrus.Debugf("Don't update %s due to chaos deployment.", stackName)
+		return nil
+	}
+
+	updatesEnabled, err := getBoolLabel(cl, stackName, "autoupdate")
+	if err != nil {
+		return err
+	}
+
+	if !updatesEnabled {
+		logrus.Debugf("Don't update %s due to disabling auto updates or missing ENABLE_AUTOUPDATE env.", stackName)
+		return nil
+	}
+
+	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName)
+	if err != nil {
+		return err
+	}
+
+	if upgradeVersion == "" {
+		logrus.Debugf("Don't update %s due to no new version.", stackName)
+		return nil
+	}
+
+	err = upgrade(cl, stackName, recipeName, upgradeVersion, conf)
+
+	return err
+}
+
+// upgrade performs all necessary steps to upgrade an app.
+func upgrade(cl *dockerclient.Client, stackName, recipeName, upgradeVersion string, conf *runtime.Config) error {
+	env, err := getEnv(cl, stackName)
+	if err != nil {
+		return err
+	}
+
+	app := config.App{
+		Name:   stackName,
+		Recipe: recipeName,
+		Server: SERVER,
+		Env:    env,
+	}
+
+	if err = processRecipeRepoVersion(recipeName, upgradeVersion, conf); err != nil {
+		return err
+	}
+
+	if err = mergeAbraShEnv(recipeName, app.Env); err != nil {
+		return err
+	}
+
+	compose, deployOpts, err := createDeployConfig(recipeName, stackName, app.Env)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("Upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
+
+	err = stack.RunDeploy(cl, deployOpts, compose, stackName, true)
+
+	return err
+}
+
+func newAbraApp(version, commit string) *cli.App {
+	app := &cli.App{
+		Name: "kadabra",
+		Usage: `The Co-op Cloud auto-updater
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []cli.Command{
+			Notify,
+			UpgradeApp,
+		},
+	}
+
+	app.Before = func(c *cli.Context) error {
+		logrus.Debugf("kadabra version %s, commit %s", version, commit)
+		return nil
+	}
+
+	return app
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := newAbraApp(version, commit)
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cmd/abra/main.go

@@ -5,10 +5,10 @@ import (
 	"coopcloud.tech/abra/cli"
 )
 
-// Version is the current version of Abra
+// Version is the current version of Abra.
 var Version string
 
-// Commit is the current git commit of Abra
+// Commit is the current git commit of Abra.
 var Commit string
 
 func main() {

cmd/kadabra/main.go

@@ -0,0 +1,23 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli/updater"
+)
+
+// Version is the current version of Kadabra.
+var Version string
+
+// Commit is the current git commit of Kadabra.
+var Commit string
+
+func main() {
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	updater.RunApp(Version, Commit)
+}

go.mod

@@ -7,17 +7,17 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.6
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.21+incompatible
+	github.com/docker/cli v20.10.23+incompatible
 	github.com/docker/distribution v2.8.1+incompatible
-	github.com/docker/docker v20.10.21+incompatible
+	github.com/docker/docker v20.10.23+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.5.1
-	github.com/hetznercloud/hcloud-go v1.38.0
+	github.com/go-git/go-git/v5 v5.5.2
+	github.com/hetznercloud/hcloud-go v1.40.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.12.2
+	github.com/schollz/progressbar/v3 v3.13.0
 	github.com/sirupsen/logrus v1.9.0
 	gotest.tools/v3 v3.4.0
 )
@@ -32,11 +32,9 @@ require (
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/gliderlabs/ssh v0.3.5
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.1
-	github.com/kevinburke/ssh_config v1.2.0
+	github.com/hashicorp/go-retryablehttp v0.7.2
 	github.com/klauspost/pgzip v1.2.5
 	github.com/libdns/gandi v1.0.2
 	github.com/libdns/libdns v0.2.1
@@ -47,6 +45,6 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/crypto v0.4.0
-	golang.org/x/sys v0.3.0
+	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/sys v0.5.0
 )

go.sum

@@ -335,16 +335,16 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v20.10.21+incompatible h1:qVkgyYUnOLQ98LtXBrwd/duVqPT2X4SHndOuGsfwyhU=
-github.com/docker/cli v20.10.21+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.23+incompatible h1:qwyha/T3rXk9lfuVcn533cKFc7n/6IzL5GXVAgMVPBg=
+github.com/docker/cli v20.10.23+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.21+incompatible h1:UTLdBmHk3bEY+w8qeO5KttOhy6OmXWsl/FEet9Uswog=
-github.com/docker/docker v20.10.21+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.23+incompatible h1:1ZQUUYAdh+oylOT85aA2ZcfRp22jmLhoaEcVEfK8dyA=
+github.com/docker/docker v20.10.23+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -409,12 +409,13 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
-github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34=
 github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE=
+github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
 github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.5.1 h1:5vtv2TB5PM/gPM+EvsHJ16hJh4uAkdGcKilcwY7FYwo=
-github.com/go-git/go-git/v5 v5.5.1/go.mod h1:uz5PQ3d0gz7mSgzZhSJToM6ALPaKCdSnl58/Xb5hzr8=
+github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw=
+github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -585,8 +586,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
-github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
+github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -604,8 +605,8 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn
 github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hetznercloud/hcloud-go v1.38.0 h1:K6Pd/mMdcLfBhvwG39qyAaacp4pCS3dKa8gChmLKxLg=
-github.com/hetznercloud/hcloud-go v1.38.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA=
+github.com/hetznercloud/hcloud-go v1.40.0 h1:rchS+LVd80GuAOQUM5pLs0Fn7tb4PflPCuxskLYc1JQ=
+github.com/hetznercloud/hcloud-go v1.40.0/go.mod h1:jcKomw7kDIjrJ9FI72S/RdRPK8X8arYzEnXjRM/uXT8=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -709,8 +710,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -898,8 +899,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/schollz/progressbar/v3 v3.12.2 h1:yLqqqpQNMxGxHY8uEshRihaHWwa0rf0yb7/Zrpgq2C0=
-github.com/schollz/progressbar/v3 v3.12.2/go.mod h1:HFJYIYQQJX32UJdyoigUl19xoV6aMwZt6iX/C30RWfg=
+github.com/schollz/progressbar/v3 v3.13.0 h1:9TeeWRcjW2qd05I8Kf9knPkW4vLM/hYoa6z9ABvxje8=
+github.com/schollz/progressbar/v3 v3.13.0/go.mod h1:ZBYnSuLAX2LU8P8UiKN/KgF2DY58AJC8yfVYLPC8Ly4=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
@@ -1069,8 +1070,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
-golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1164,8 +1165,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk=
-golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1310,16 +1311,18 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1330,8 +1333,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/app/app.go

@@ -1,13 +1,9 @@
 package app
 
 import (
-	"context"
-	"fmt"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/upstream/stack"
-	apiclient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
@@ -37,45 +33,6 @@ type deployedServiceSpec struct {
 // VersionSpec represents a deployed app and associated metadata.
 type VersionSpec map[string]deployedServiceSpec
 
-// DeployedVersions lists metadata (e.g. versions) for deployed
-func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App) (VersionSpec, bool, error) {
-	services, err := stack.GetStackServices(ctx, cl, app.StackName())
-	if err != nil {
-		return VersionSpec{}, false, err
-	}
-
-	appSpec := make(VersionSpec)
-	for _, service := range services {
-		serviceName := ParseServiceName(service.Spec.Name)
-		label := fmt.Sprintf("coop-cloud.%s.%s.version", app.StackName(), serviceName)
-		if deployLabel, ok := service.Spec.Labels[label]; ok {
-			version, _ := ParseVersionLabel(deployLabel)
-			appSpec[serviceName] = deployedServiceSpec{Name: serviceName, Version: version}
-		}
-	}
-
-	deployed := len(services) > 0
-
-	if deployed {
-		logrus.Debugf("detected %s as deployed versions of %s", appSpec, app.Name)
-	} else {
-		logrus.Debugf("detected %s as not deployed", app.Name)
-	}
-
-	return appSpec, len(services) > 0, nil
-}
-
-// ParseVersionLabel parses a $VERSION-$DIGEST app service label.
-func ParseVersionLabel(label string) (string, string) {
-	// versions may look like v4.2-abcd or v4.2-alpine-abcd
-	idx := strings.LastIndex(label, "-")
-	version := label[:idx]
-	digest := label[idx+1:]
-	logrus.Debugf("parsed %s as version from %s", version, label)
-	logrus.Debugf("parsed %s as digest from %s", digest, label)
-	return version, digest
-}
-
 // ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label.
 func ParseServiceName(label string) string {
 	idx := strings.LastIndex(label, "_")

pkg/catalogue/catalogue.go

@@ -0,0 +1,122 @@
+package catalogue
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                        true,
+	"abra-apps":                   true,
+	"abra-aur":                    true,
+	"abra-bash":                   true,
+	"abra-capsul":                 true,
+	"abra-gandi":                  true,
+	"abra-hetzner":                true,
+	"apps":                        true,
+	"aur-abra-git":                true,
+	"auto-recipes-catalogue-json": true,
+	"auto-mirror":                 true,
+	"backup-bot":                  true,
+	"backup-bot-two":              true,
+	"beta.coopcloud.tech":         true,
+	"comrade-renovate-bot":        true,
+	"coopcloud.tech":              true,
+	"coturn":                      true,
+	"docker-cp-deploy":            true,
+	"docker-dind-bats-kcov":       true,
+	"docs.coopcloud.tech":         true,
+	"drone-abra":                  true,
+	"example":                     true,
+	"gardening":                   true,
+	"go-abra":                     true,
+	"organising":                  true,
+	"pyabra":                      true,
+	"radicle-seed-node":           true,
+	"recipes-catalogue-json":      true,
+	"recipes-wishlist":            true,
+	"recipes.coopcloud.tech":      true,
+	"stack-ssh-deploy":            true,
+	"swarm-cronjob":               true,
+	"tagcmp":                      true,
+	"traefik-cert-dumper":         true,
+	"tyop":                        true,
+}
+
+// EnsureCatalogue ensures that the catalogue is cloned locally & present.
+func EnsureCatalogue() error {
+	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		logrus.Debugf("cloned catalogue repository to %s", catalogueDir)
+	}
+
+	return nil
+}
+
+// EnsureUpToDate ensures that the local catalogue has no unstaged changes as
+// is up to date. This is useful to run before doing catalogue generation.
+func EnsureUpToDate() error {
+	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		msg := "%s has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, config.CATALOGUE_DIR)
+	}
+
+	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	remotes, err := repo.Remotes()
+	if err != nil {
+		return err
+	}
+
+	if len(remotes) == 0 {
+		msg := "cannot ensure %s is up-to-date, no git remotes configured"
+		logrus.Debugf(msg, config.CATALOGUE_DIR)
+		return nil
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	branch, err := gitPkg.CheckoutDefaultBranch(repo, config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	opts := &git.PullOptions{
+		Force:         true,
+		ReferenceName: branch,
+	}
+
+	if err := worktree.Pull(opts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return err
+		}
+	}
+
+	logrus.Debugf("fetched latest git changes for %s", config.CATALOGUE_DIR)
+
+	return nil
+}

pkg/client/client.go

@@ -2,24 +2,31 @@
 package client
 
 import (
+	"context"
+	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
 
 	contextPkg "coopcloud.tech/abra/pkg/context"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
 	"github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
-// New initiates a new Docker client.
-func New(contextName string) (*client.Client, error) {
+// New initiates a new Docker client. New client connections are validated so
+// that we ensure connections via SSH to the daemon can succeed. It takes into
+// account that you may only want the local client and not communicate via SSH.
+// For this use-case, please pass "default" as the contextName.
+func New(serverName string) (*client.Client, error) {
 	var clientOpts []client.Opt
 
-	if contextName != "default" {
-		context, err := GetContext(contextName)
+	if serverName != "default" {
+		context, err := GetContext(serverName)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("unknown server, run \"abra server add %s\"?", serverName)
 		}
 
 		ctxEndpoint, err := contextPkg.GetContextEndpoint(context)
@@ -33,7 +40,6 @@ func New(contextName string) (*client.Client, error) {
 		}
 
 		httpClient := &http.Client{
-			// No tls, no proxy
 			Transport: &http.Transport{
 				DialContext:     helper.Dialer,
 				IdleConnTimeout: 30 * time.Second,
@@ -59,7 +65,20 @@ func New(contextName string) (*client.Client, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("created client for %s", contextName)
+	logrus.Debugf("created client for %s", serverName)
+
+	info, err := cl.Info(context.Background())
+	if err != nil {
+		return cl, sshPkg.Fatal(serverName, err)
+	}
+
+	if info.Swarm.LocalNodeState == "inactive" {
+		if serverName != "default" {
+			return cl, fmt.Errorf("swarm mode not enabled on %s?", serverName)
+		} else {
+			return cl, errors.New("swarm mode not enabled on local server?")
+		}
+	}
 
 	return cl, nil
 }

pkg/client/registry.go

@@ -3,13 +3,10 @@ package client
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/containers/image/docker"
 	"github.com/containers/image/types"
 	"github.com/docker/distribution/reference"
-	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
 )
 
 // GetRegistryTags retrieves all tags of an image from a container registry.
@@ -29,29 +26,3 @@ func GetRegistryTags(img reference.Named) ([]string, error) {
 
 	return tags, nil
 }
-
-// GetTagDigest retrieves an image digest from a container registry.
-func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
-	target := fmt.Sprintf("//%s", reference.Path(image))
-
-	ref, err := docker.ParseReference(target)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
-	}
-
-	ctx := context.Background()
-	img, err := ref.NewImage(ctx, nil)
-	if err != nil {
-		logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
-		return "", fmt.Errorf("unable to read digest for %s", image)
-	}
-	defer img.Close()
-
-	digest := img.ConfigInfo().Digest.String()
-
-	if digest == "" {
-		return digest, fmt.Errorf("unable to read digest for %s", image)
-	}
-
-	return strings.Split(digest, ":")[1][:7], nil
-}

pkg/client/secret.go

@@ -4,20 +4,14 @@ import (
 	"context"
 
 	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
 )
 
-func StoreSecret(secretName, secretValue, server string) error {
-	cl, err := New(server)
-	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
+func StoreSecret(cl *client.Client, secretName, secretValue, server string) error {
 	ann := swarm.Annotations{Name: secretName}
 	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
 
-	// We don't bother with the secret IDs for now
-	if _, err := cl.SecretCreate(ctx, spec); err != nil {
+	if _, err := cl.SecretCreate(context.Background(), spec); err != nil {
 		return err
 	}
 

pkg/client/volumes.go

@@ -5,14 +5,10 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
 )
 
-func GetVolumes(ctx context.Context, server string, fs filters.Args) ([]*types.Volume, error) {
-	cl, err := New(server)
-	if err != nil {
-		return nil, err
-	}
-
+func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*types.Volume, error) {
 	volumeListOKBody, err := cl.VolumeList(ctx, fs)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
@@ -32,12 +28,7 @@ func GetVolumeNames(volumes []*types.Volume) []string {
 	return volumeNames
 }
 
-func RemoveVolumes(ctx context.Context, server string, volumeNames []string, force bool) error {
-	cl, err := New(server)
-	if err != nil {
-		return err
-	}
-
+func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
 	for _, volName := range volumeNames {
 		err := cl.VolumeRemove(ctx, volName, force)
 		if err != nil {

pkg/config/app.go

@@ -5,8 +5,12 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 
+	"github.com/schollz/progressbar/v3"
+
+	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
@@ -43,9 +47,9 @@ type App struct {
 	Path   string
 }
 
-// StackName gets what the docker safe stack name is for the app. This should
-// not not shown to the user, use a.Name for that. Give the output of this
-// command to Docker only.
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
@@ -169,7 +173,7 @@ func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	if !exists {
 		recipe, exists = env["TYPE"]
 		if !exists {
-			return App{}, fmt.Errorf("%s is missing the RECIPE env var", name)
+			return App{}, fmt.Errorf("%s is missing the TYPE env var?", name)
 		}
 	}
 
@@ -183,7 +187,7 @@ func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	}, nil
 }
 
-// LoadAppFiles gets all app files for a given set of servers or all servers
+// LoadAppFiles gets all app files for a given set of servers or all servers.
 func LoadAppFiles(servers ...string) (AppFiles, error) {
 	appFiles := make(AppFiles)
 	if len(servers) == 1 {
@@ -192,7 +196,7 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 			var err error
 			servers, err = GetAllFoldersInDirectory(SERVERS_DIR)
 			if err != nil {
-				return nil, err
+				return appFiles, err
 			}
 		}
 	}
@@ -201,10 +205,11 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 
 	for _, server := range servers {
 		serverDir := path.Join(SERVERS_DIR, server)
-		files, err := getAllFilesInDirectory(serverDir)
+		files, err := GetAllFilesInDirectory(serverDir)
 		if err != nil {
-			return nil, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server)
+			return appFiles, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server)
 		}
+
 		for _, file := range files {
 			appName := strings.TrimSuffix(file.Name(), ".env")
 			appFilePath := path.Join(SERVERS_DIR, server, file.Name())
@@ -214,12 +219,13 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 			}
 		}
 	}
+
 	return appFiles, nil
 }
 
-// GetApp loads an apps settings, reading it from file, in preparation to use it
-//
-// ONLY use when ready to use the env file to keep IO down
+// GetApp loads an apps settings, reading it from file, in preparation to use
+// it. It should only be used when ready to use the env file to keep IO
+// operations down.
 func GetApp(apps AppFiles, name AppName) (App, error) {
 	appFile, exists := apps[name]
 	if !exists {
@@ -234,8 +240,9 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
 	return app, nil
 }
 
-// GetApps returns a slice of Apps with their env files read from a given slice of AppFiles
-func GetApps(appFiles AppFiles) ([]App, error) {
+// GetApps returns a slice of Apps with their env files read from a given
+// slice of AppFiles.
+func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) {
 	var apps []App
 
 	for name := range appFiles {
@@ -243,7 +250,14 @@ func GetApps(appFiles AppFiles) ([]App, error) {
 		if err != nil {
 			return nil, err
 		}
-		apps = append(apps, app)
+
+		if recipeFilter != "" {
+			if app.Recipe == recipeFilter {
+				apps = append(apps, app)
+			}
+		} else {
+			apps = append(apps, app)
+		}
 	}
 
 	return apps, nil
@@ -290,7 +304,7 @@ func GetAppNames() ([]string, error) {
 		return appNames, err
 	}
 
-	apps, err := GetApps(appFiles)
+	apps, err := GetApps(appFiles, "")
 	if err != nil {
 		return appNames, err
 	}
@@ -302,7 +316,8 @@ func GetAppNames() ([]string, error) {
 	return appNames, nil
 }
 
-// TemplateAppEnvSample copies the example env file for the app into the users env files
+// TemplateAppEnvSample copies the example env file for the app into the users
+// env files.
 func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample")
 	envSample, err := ioutil.ReadFile(envSamplePath)
@@ -337,35 +352,49 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 	return nil
 }
 
-// SanitiseAppName makes a app name usable with Docker by replacing illegal characters
+// SanitiseAppName makes a app name usable with Docker by replacing illegal
+// characters.
 func SanitiseAppName(name string) string {
 	return strings.ReplaceAll(name, ".", "_")
 }
 
-// GetAppStatuses queries servers to check the deployment status of given apps
-func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
+// GetAppStatuses queries servers to check the deployment status of given apps.
+func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) {
 	statuses := make(map[string]map[string]string)
 
-	var unique []string
 	servers := make(map[string]struct{})
-	for _, appFile := range appFiles {
-		if _, ok := servers[appFile.Server]; !ok {
-			servers[appFile.Server] = struct{}{}
-			unique = append(unique, appFile.Server)
+	for _, app := range apps {
+		if _, ok := servers[app.Server]; !ok {
+			servers[app.Server] = struct{}{}
 		}
 	}
 
-	bar := formatter.CreateProgressbar(len(servers), "querying remote servers...")
+	var bar *progressbar.ProgressBar
+	if !MachineReadable {
+		bar = formatter.CreateProgressbar(len(servers), "querying remote servers...")
+	}
+
 	ch := make(chan stack.StackStatus, len(servers))
 	for server := range servers {
+		cl, err := client.New(server)
+		if err != nil {
+			return statuses, err
+		}
+
 		go func(s string) {
-			ch <- stack.GetAllDeployedServices(s)
-			bar.Add(1)
+			ch <- stack.GetAllDeployedServices(cl, s)
+			if !MachineReadable {
+				bar.Add(1)
+			}
 		}(server)
 	}
 
 	for range servers {
 		status := <-ch
+		if status.Err != nil {
+			return statuses, status.Err
+		}
+
 		for _, service := range status.Services {
 			result := make(map[string]string)
 			name := service.Spec.Labels[convert.LabelNamespace]
@@ -428,3 +457,61 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp
 
 	return compose, nil
 }
+
+// ExposeAllEnv exposes all env variables to the app container
+func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("Add the following environment to the app service config of %s:", stackName)
+			for k, v := range appEnv {
+				_, exists := service.Environment[k]
+				if !exists {
+					value := v
+					service.Environment[k] = &value
+					logrus.Debugf("Add Key: %s Value: %s to %s", k, value, stackName)
+				}
+			}
+		}
+	}
+}
+
+// SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container
+// to signal which recipe is connected to the deployed app
+func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName)
+			service.Deploy.Labels[labelKey] = recipe
+		}
+	}
+}
+
+// SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container
+// to signal if the app is deployed in chaos mode
+func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName)
+			service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos)
+		}
+	}
+}
+
+// SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the
+// auto update process for this app. The default if this variable is not set is to disable
+// the auto update process.
+func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"]
+			if !exists {
+				enable_auto_update = "false"
+			}
+			logrus.Debugf("set label 'coop-cloud.%s.autoupdate' to %s for %s", stackName, enable_auto_update, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.autoupdate", stackName)
+			service.Deploy.Labels[labelKey] = enable_auto_update
+		}
+	}
+}

pkg/config/env.go

@@ -19,6 +19,7 @@ var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
 var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
 var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
 var BACKUP_DIR = path.Join(ABRA_DIR, "backups")
+var CATALOGUE_DIR = path.Join(ABRA_DIR, "catalogue")
 var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json")
 var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
@@ -65,8 +66,8 @@ func ReadServerNames() ([]string, error) {
 	return serverNames, nil
 }
 
-// getAllFilesInDirectory returns filenames of all files in directory
-func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
+// GetAllFilesInDirectory returns filenames of all files in directory
+func GetAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
 	var realFiles []fs.FileInfo
 
 	files, err := ioutil.ReadDir(directory)

pkg/config/env_test.go

@@ -54,7 +54,7 @@ func TestGetAllFoldersInDirectory(t *testing.T) {
 }
 
 func TestGetAllFilesInDirectory(t *testing.T) {
-	files, err := getAllFilesInDirectory(testFolder)
+	files, err := GetAllFilesInDirectory(testFolder)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/dns/common.go

@@ -1,7 +1,6 @@
 package dns
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"os"
@@ -32,35 +31,12 @@ func NewToken(provider, providerTokenEnvVar string) (string, error) {
 
 // EnsureIPv4 ensures that an ipv4 address is set for a domain name
 func EnsureIPv4(domainName string) (string, error) {
-	var ipv4 string
-
-	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
-	freifunkDNS := "5.1.66.255:53"
-
-	resolver := &net.Resolver{
-		PreferGo: false,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Millisecond * time.Duration(10000),
-			}
-			return d.DialContext(ctx, "udp", freifunkDNS)
-		},
-	}
-
-	ctx := context.Background()
-	ips, err := resolver.LookupIPAddr(ctx, domainName)
+	ipv4, err := net.ResolveIPAddr("ip", domainName)
 	if err != nil {
-		return ipv4, err
+		return "", err
 	}
 
-	if len(ips) == 0 {
-		return ipv4, fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
-	}
-
-	ipv4 = ips[0].IP.To4().String()
-	logrus.Debugf("%s points to %s (resolver: %s)", domainName, ipv4, freifunkDNS)
-
-	return ipv4, nil
+	return ipv4.String(), nil
 }
 
 // EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address

pkg/formatter/formatter.go

@@ -6,7 +6,8 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
-	"github.com/olekukonko/tablewriter"
+	// "github.com/olekukonko/tablewriter"
+	"coopcloud.tech/abra/pkg/jsontable"
 	"github.com/schollz/progressbar/v3"
 	"github.com/sirupsen/logrus"
 )
@@ -32,8 +33,8 @@ func HumanDuration(timestamp int64) string {
 }
 
 // CreateTable prepares a table layout for output.
-func CreateTable(columns []string) *tablewriter.Table {
-	table := tablewriter.NewWriter(os.Stdout)
+func CreateTable(columns []string) *jsontable.JSONTable {
+	table := jsontable.NewJSONTable(os.Stdout)
 	table.SetAutoWrapText(false)
 	table.SetHeader(columns)
 	return table

pkg/git/branch.go

@@ -1,16 +1,19 @@
 package git
 
 import (
+	"fmt"
+
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
 )
 
-// Check if a branch exists in a repo.
-// Use this and not repository.Branch(), because the latter does not
-// actually check for existing branches.
-// See https://github.com/go-git/go-git/issues/518
+// Check if a branch exists in a repo. Use this and not repository.Branch(),
+// because the latter does not actually check for existing branches. See
+// https://github.com/gogit/gogit/issues/518 for more.
 func HasBranch(repository *git.Repository, name string) bool {
 	var exist bool
+
 	if iter, err := repository.Branches(); err == nil {
 		iterFunc := func(reference *plumbing.Reference) error {
 			if name == reference.Name().Short() {
@@ -21,10 +24,11 @@ func HasBranch(repository *git.Repository, name string) bool {
 		}
 		_ = iter.ForEach(iterFunc)
 	}
+
 	return exist
 }
 
-// GetCurrentBranch retrieves the current branch of a repository
+// GetCurrentBranch retrieves the current branch of a repository.
 func GetCurrentBranch(repository *git.Repository) (string, error) {
 	branchRefs, err := repository.Branches()
 	if err != nil {
@@ -52,3 +56,45 @@ func GetCurrentBranch(repository *git.Repository) (string, error) {
 
 	return currentBranchName, nil
 }
+
+// GetDefaultBranch retrieves the default branch of a repository.
+func GetDefaultBranch(repo *git.Repository, repoPath string) (plumbing.ReferenceName, error) {
+	branch := "master"
+
+	if !HasBranch(repo, "master") {
+		if !HasBranch(repo, "main") {
+			return "", fmt.Errorf("failed to select default branch in %s", repoPath)
+		}
+		branch = "main"
+	}
+
+	return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
+}
+
+// CheckoutDefaultBranch checks out the default branch of a repository.
+func CheckoutDefaultBranch(repo *git.Repository, repoPath string) (plumbing.ReferenceName, error) {
+	branch, err := GetDefaultBranch(repo, repoPath)
+	if err != nil {
+		return plumbing.ReferenceName(""), err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return plumbing.ReferenceName(""), err
+	}
+
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: branch,
+	}
+
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out %s in %s", branch, repoPath)
+		return branch, err
+	}
+
+	logrus.Debugf("successfully checked out %v in %s", branch, repoPath)
+
+	return branch, nil
+}

pkg/git/clone.go

@@ -15,22 +15,32 @@ import (
 func Clone(dir, url string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
 		logrus.Debugf("%s does not exist, attempting to git clone from %s", dir, url)
-		_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
+
+		_, err := git.PlainClone(dir, false, &git.CloneOptions{
+			URL:           url,
+			Tags:          git.AllTags,
+			ReferenceName: plumbing.ReferenceName("refs/heads/master"),
+			SingleBranch:  true,
+		})
 		if err != nil {
 			logrus.Debugf("cloning %s default branch failed, attempting from main branch", url)
+
 			_, err := git.PlainClone(dir, false, &git.CloneOptions{
 				URL:           url,
 				Tags:          git.AllTags,
 				ReferenceName: plumbing.ReferenceName("refs/heads/main"),
+				SingleBranch:  true,
 			})
 			if err != nil {
 				if strings.Contains(err.Error(), "authentication required") {
 					name := filepath.Base(dir)
 					return fmt.Errorf("unable to clone %s, does %s exist?", name, url)
 				}
+
 				return err
 			}
 		}
+
 		logrus.Debugf("%s has been git cloned successfully", dir)
 	} else {
 		logrus.Debugf("%s already exists", dir)

pkg/jsontable/jsontable.go

@@ -0,0 +1,203 @@
+package jsontable
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+// A quick-and-dirty proxy/emulator of tablewriter to enable more easy machine readable output
+// - Does not strictly support types, just quoted or unquoted values
+// - Does not support nested values.
+// If a datalabel is set with SetDataLabel(true, "..."), that will be used as the key for teh data of the table,
+// otherwise if the caption is set with SetCaption(true, "..."), the data label will be set to the default of
+// "rows", otherwise the table will output as a JSON list.
+//
+// Proxys all actions through to the tablewriter except addrow and addbatch, which it does at render time
+//
+
+type JSONTable struct {
+	out             io.Writer
+	colsize         int
+	rows            [][]string
+	keys            []string
+	quoted          []bool // hack to do output typing, quoted vs. unquoted
+	hasDataLabel    bool
+	dataLabel       string
+	hasCaption      bool
+	caption         string // the actual caption
+	hasCaptionLabel bool
+	captionLabel    string // the key in the dictionary for the caption
+	tbl             *tablewriter.Table
+}
+
+func writeChar(w io.Writer, c byte) {
+	w.Write([]byte{c})
+
+}
+
+func NewJSONTable(writer io.Writer) *JSONTable {
+	t := &JSONTable{
+		out:             writer,
+		colsize:         0,
+		rows:            [][]string{},
+		keys:            []string{},
+		quoted:          []bool{},
+		hasDataLabel:    false,
+		dataLabel:       "rows",
+		hasCaption:      false,
+		caption:         "",
+		hasCaptionLabel: false,
+		captionLabel:    "caption",
+		tbl:             tablewriter.NewWriter(writer),
+	}
+	return t
+}
+
+func (t *JSONTable) NumLines() int {
+	// JSON only but reflects a shared state.
+	return len(t.rows)
+}
+
+func (t *JSONTable) SetHeader(keys []string) {
+	// Set the keys value which will assign each column to the keys.
+	// Note that we'll ignore values that are beyond the length of the keys list
+	t.colsize = len(keys)
+	t.keys = []string{}
+	for _, k := range keys {
+		t.keys = append(t.keys, k)
+		t.quoted = append(t.quoted, true)
+	}
+	t.tbl.SetHeader(keys)
+}
+
+func (t *JSONTable) SetColumnQuoting(quoting []bool) {
+	// Specify which columns are quoted or unquoted in output
+	// JSON only
+	for i := 0; i < t.colsize; i++ {
+		t.quoted[i] = quoting[i]
+	}
+}
+
+func (t *JSONTable) Append(row []string) {
+	// We'll just append whatever to the rows list. If they fix the keys after appending rows, it'll work as
+	// expected.
+	// We should detect if the row is narrower than the key list tho.
+	// JSON only (but we use the rows later when rendering a regular table)
+	t.rows = append(t.rows, row)
+}
+
+func (t *JSONTable) Render() {
+	// Load the table with rows and render.
+	// Proxy only
+	for _, row := range t.rows {
+		t.tbl.Append(row)
+	}
+
+	t.tbl.Render()
+}
+
+func (t *JSONTable) _JSONRenderInner() {
+	// JSON only
+	// Render the list of dictionaries to the writer.
+	//// inner render loop
+	writeChar(t.out, '[')
+	for rowidx, row := range t.rows {
+		if rowidx != 0 {
+			writeChar(t.out, ',')
+		}
+		writeChar(t.out, '{')
+		for keyidx, key := range t.keys {
+			value := "nil"
+			if keyidx < len(row) {
+				value = row[keyidx]
+			}
+			if keyidx != 0 {
+				writeChar(t.out, ',')
+			}
+			if t.quoted[keyidx] {
+				fmt.Fprintf(t.out, "\"%s\":\"%s\"", key, value)
+			} else {
+				fmt.Fprintf(t.out, "\"%s\":%s", key, value)
+			}
+		}
+		writeChar(t.out, '}')
+	}
+	writeChar(t.out, ']')
+
+}
+
+func (t *JSONTable) JSONRender() {
+	// write JSON table to output
+	// JSON only
+
+	if t.hasDataLabel || t.hasCaption {
+		// dict mode
+		writeChar(t.out, '{')
+
+		if t.hasCaption {
+			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
+
+		}
+		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
+
+	}
+
+	// write list
+	t._JSONRenderInner()
+
+	if t.hasDataLabel || t.hasCaption {
+		// dict mode
+		writeChar(t.out, '}')
+	}
+
+}
+
+func (t *JSONTable) SetCaption(caption bool, captionText ...string) {
+	t.hasCaption = caption
+	if len(captionText) == 1 {
+		t.caption = captionText[0]
+	}
+	t.tbl.SetCaption(caption, captionText...)
+}
+
+func (t *JSONTable) SetCaptionLabel(captionLabel bool, captionLabelText ...string) {
+	// JSON only
+	t.hasCaptionLabel = captionLabel
+	if len(captionLabelText) == 1 {
+		t.captionLabel = captionLabelText[0]
+	}
+}
+
+func (t *JSONTable) SetDataLabel(dataLabel bool, dataLabelText ...string) {
+	// JSON only
+	t.hasDataLabel = dataLabel
+	if len(dataLabelText) == 1 {
+		t.dataLabel = dataLabelText[0]
+	}
+}
+
+func (t *JSONTable) AppendBulk(rows [][]string) {
+	// JSON only but reflects shared state
+	for _, row := range rows {
+		t.Append(row)
+	}
+}
+
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAutoMergeCellsByColumnIndex(cols []int) {
+	// FIXME
+	t.tbl.SetAutoMergeCellsByColumnIndex(cols)
+}
+
+func (t *JSONTable) SetAutoMergeCells(auto bool) {
+	// FIXME
+	t.tbl.SetAutoMergeCells(auto)
+}
+
+// Stub functions
+func (t *JSONTable) SetAutoWrapText(auto bool) {
+	t.tbl.SetAutoWrapText(auto)
+	return
+}

pkg/jsontable/jsontable_test.go

@@ -0,0 +1,83 @@
+package jsontable
+
+import (
+	"testing"
+
+	"bytes"
+	"encoding/json"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+var TestLine = []string{"1", "2"}
+var TestGroup = [][]string{{"1", "2", "3"}, {"a", "teohunteohu", "c", "d"}, {"☺", "☹"}}
+var TestKeys = []string{"key0", "key1", "key2"}
+
+// test creation
+func TestNewTable(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+	if tbl.NumLines() != 0 {
+		t.Fatalf("Something went weird when making table (should have 0 lines)")
+	}
+}
+
+// test adding things
+func TestTableAdd(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+
+	tbl.Append(TestLine)
+	if tbl.NumLines() != 1 {
+		t.Fatalf("Appending a line does not result in a length of 1.")
+	}
+
+	tbl.AppendBulk(TestGroup)
+	numlines := tbl.NumLines()
+	if numlines != (len(TestGroup) + 1) {
+		t.Fatalf("Appending two lines does not result in a length of 4 (length is %d).", numlines)
+	}
+}
+
+// test JSON output is parsable
+func TestJsonParsable(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+
+	tbl.AppendBulk(TestGroup)
+	tbl.SetHeader(TestKeys)
+
+	tbl.JSONRender()
+
+	var son []map[string]interface{}
+
+	err := json.Unmarshal(b.Bytes(), &son)
+
+	if err != nil {
+		t.Fatalf("Did not produce parsable JSON: %s", err.Error())
+	}
+}
+
+// test identical commands to a tablewriter and jsontable produce the same rendered output
+func TestTableWriter(t *testing.T) {
+	var bjson bytes.Buffer
+	var btable bytes.Buffer
+
+	tbl := NewJSONTable(&bjson)
+
+	tbl.AppendBulk(TestGroup)
+	tbl.SetHeader(TestKeys)
+	tbl.Render()
+
+	wtbl := tablewriter.NewWriter(&btable)
+
+	wtbl.AppendBulk(TestGroup)
+	wtbl.SetHeader(TestKeys)
+	wtbl.Render()
+
+	if bytes.Compare(bjson.Bytes(), btable.Bytes()) != 0 {
+		t.Fatalf("JSON table and TableWriter produce non-identical outputs.\n%s\n%s", bjson.Bytes(), btable.Bytes())
+	}
+}
+
+/// FIXME test different output formats when captions etc. are added

pkg/lint/recipe.go

@@ -19,12 +19,40 @@ var Critical = "critical"
 
 type LintFunction func(recipe.Recipe) (bool, error)
 
+// SkipFunction determines whether the LintFunction is run or not. It should
+// not take the lint rule level into account because some rules are always an
+// error but may depend on some additional context of the recipe configuration.
+// This function aims to cover those additional cases.
+type SkipFunction func(recipe.Recipe) (bool, error)
+
+// LintRule is a linting rule which helps a recipe maintainer avoid common
+// problems in their recipe configurations. We aim to highlight things that
+// might result in critical errors or hours lost in debugging obscure
+// Docker-isms. Humans make the final call on these rules, please raise an
+// issue if you disagree.
 type LintRule struct {
-	Ref          string
-	Level        string
-	Description  string
-	HowToResolve string
-	Function     LintFunction
+	Ref           string       // Reference of the linting rule
+	Level         string       // Level of the warning
+	Description   string       // Description of the issue
+	HowToResolve  string       // Documentation for recipe maintainer
+	Function      LintFunction // Rule implementation
+	SkipCondition SkipFunction // Whether or not to execute the lint rule
+}
+
+// Skip implements the SkipFunction for the lint rule.
+func (l LintRule) Skip(recipe recipe.Recipe) bool {
+	if l.SkipCondition != nil {
+		ok, err := l.SkipCondition(recipe)
+		if err != nil {
+			logrus.Debugf("%s: skip condition: %s", l.Ref, err)
+		}
+		if ok {
+			logrus.Debugf("skipping %s based on skip condition", l.Ref)
+			return true
+		}
+	}
+
+	return false
 }
 
 var LintRules = map[string][]LintRule{
@@ -102,11 +130,12 @@ var LintRules = map[string][]LintRule{
 			Function:     LintAppService,
 		},
 		{
-			Ref:          "R010",
-			Level:        "error",
-			Description:  "traefik routing enabled",
-			HowToResolve: "include \"traefik.enable=true\" deploy label",
-			Function:     LintTraefikEnabled,
+			Ref:           "R010",
+			Level:         "error",
+			Description:   "traefik routing enabled",
+			HowToResolve:  "include \"traefik.enable=true\" deploy label",
+			Function:      LintTraefikEnabled,
+			SkipCondition: LintTraefikEnabledSkipCondition,
 		},
 		{
 			Ref:          "R011",
@@ -125,6 +154,9 @@ var LintRules = map[string][]LintRule{
 	},
 }
 
+// LintForErrors lints specifically for errors and not other levels. This is
+// used in code paths such as "app deploy" to avoid nasty surprises but not for
+// the typical linting commands, which do handle other levels.
 func LintForErrors(recipe recipe.Recipe) error {
 	logrus.Debugf("linting for critical errors in %s configs", recipe.Name)
 
@@ -132,7 +164,12 @@ func LintForErrors(recipe recipe.Recipe) error {
 		if level != "error" {
 			continue
 		}
+
 		for _, rule := range LintRules[level] {
+			if rule.Skip(recipe) {
+				continue
+			}
+
 			ok, err := rule.Function(recipe)
 			if err != nil {
 				return err
@@ -175,6 +212,24 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 	return false, nil
 }
 
+// LintTraefikEnabledSkipCondition signals a skip for this linting rule if it
+// confirms that there is no "DOMAIN=..." in the .env.sample configuration of
+// the recipe. This typically means that no domain is required to deploy and
+// therefore no matching traefik deploy label will be present.
+func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	sampleEnv, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
+	}
+
+	if _, ok := sampleEnv["DOMAIN"]; !ok {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func LintTraefikEnabled(recipe recipe.Recipe) (bool, error) {
 	for _, service := range recipe.Config.Services {
 		for label := range service.Deploy.Labels {

pkg/recipe/recipe.go

@@ -8,20 +8,24 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
-	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/compose"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/limit"
+	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/abra/pkg/web"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
+	gitConfig "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
@@ -40,9 +44,8 @@ type service = string
 
 // ServiceMeta represents meta info associated with a service.
 type ServiceMeta struct {
-	Digest string `json:"digest"`
-	Image  string `json:"image"`
-	Tag    string `json:"tag"`
+	Image string `json:"image"`
+	Tag   string `json:"tag"`
 }
 
 // RecipeVersions are the versions associated with a recipe.
@@ -204,8 +207,8 @@ func (r Recipe) Tags() ([]string, error) {
 }
 
 // Get retrieves a recipe.
-func Get(recipeName string) (Recipe, error) {
-	if err := EnsureExists(recipeName); err != nil {
+func Get(recipeName string, conf *runtime.Config) (Recipe, error) {
+	if err := EnsureExists(recipeName, conf); err != nil {
 		return Recipe{}, err
 	}
 
@@ -231,7 +234,7 @@ func Get(recipeName string) (Recipe, error) {
 		return Recipe{}, err
 	}
 
-	meta, err := GetRecipeMeta(recipeName)
+	meta, err := GetRecipeMeta(recipeName, conf)
 	if err != nil {
 		if strings.Contains(err.Error(), "does not exist") {
 			meta = RecipeMeta{}
@@ -248,7 +251,11 @@ func Get(recipeName string) (Recipe, error) {
 }
 
 // EnsureExists ensures that a recipe is locally cloned
-func EnsureExists(recipeName string) error {
+func EnsureExists(recipeName string, conf *runtime.Config) error {
+	if !conf.EnsureRecipeExists {
+		return nil
+	}
+
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
@@ -308,8 +315,7 @@ func EnsureVersion(recipeName, version string) error {
 	logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
 
 	if tagRef.String() == "" {
-		logrus.Warnf("no published release discovered for %s", recipeName)
-		return nil
+		return fmt.Errorf("no published release discovered for %s", recipeName)
 	}
 
 	worktree, err := repo.Worktree()
@@ -332,7 +338,7 @@ func EnsureVersion(recipeName, version string) error {
 }
 
 // EnsureLatest makes sure the latest commit is checked out for a local recipe repository
-func EnsureLatest(recipeName string) error {
+func EnsureLatest(recipeName string, conf *runtime.Config) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	isClean, err := gitPkg.IsClean(recipeDir)
@@ -360,11 +366,21 @@ func EnsureLatest(recipeName string) error {
 		return err
 	}
 
-	branch, err := GetDefaultBranch(repo, recipeName)
+	meta, err := GetRecipeMeta(recipeName, conf)
 	if err != nil {
 		return err
 	}
 
+	var branch plumbing.ReferenceName
+	if meta.DefaultBranch != "" {
+		branch = plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", meta.DefaultBranch))
+	} else {
+		branch, err = gitPkg.GetDefaultBranch(repo, recipeDir)
+		if err != nil {
+			return err
+		}
+	}
+
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
 		Force:  true,
@@ -464,6 +480,9 @@ func GetRecipeFeaturesAndCategory(recipeName string) (Features, string, error) {
 		"\n")
 
 	for _, val := range readmeLines {
+		if strings.Contains(val, "**Status**") {
+			feat.Status, _ = strconv.Atoi(strings.TrimSpace(strings.Split(strings.TrimPrefix(val, "* **Status**:"), ",")[0]))
+		}
 		if strings.Contains(val, "**Category**") {
 			category = strings.TrimSpace(
 				strings.TrimPrefix(val, "* **Category**:"),
@@ -569,21 +588,22 @@ func EnsureUpToDate(recipeName string) error {
 
 	isClean, err := gitPkg.IsClean(recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to check git clean status in %s: %s", recipeDir, err)
 	}
 
 	if !isClean {
-		return fmt.Errorf("%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding", recipeName, recipeDir)
+		msg := "%s (%s) has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, recipeName, recipeDir)
 	}
 
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to open %s: %s", recipeDir, err)
 	}
 
 	remotes, err := repo.Remotes()
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to read remotes in %s: %s", recipeDir, err)
 	}
 
 	if len(remotes) == 0 {
@@ -593,22 +613,35 @@ func EnsureUpToDate(recipeName string) error {
 
 	worktree, err := repo.Worktree()
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to open git work tree in %s: %s", recipeDir, err)
 	}
 
-	branch, err := CheckoutDefaultBranch(repo, recipeName)
+	branch, err := gitPkg.CheckoutDefaultBranch(repo, recipeDir)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to check out default branch in %s: %s", recipeDir, err)
+	}
+
+	fetchOpts := &git.FetchOptions{
+		Tags: git.AllTags,
+		RefSpecs: []gitConfig.RefSpec{
+			gitConfig.RefSpec(fmt.Sprintf("%s:%s", branch, branch)),
+		},
+	}
+	if err := repo.Fetch(fetchOpts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return fmt.Errorf("unable to fetch tags in %s: %s", recipeDir, err)
+		}
 	}
 
 	opts := &git.PullOptions{
 		Force:         true,
 		ReferenceName: branch,
+		SingleBranch:  true,
 	}
 
 	if err := worktree.Pull(opts); err != nil {
 		if !strings.Contains(err.Error(), "already up-to-date") {
-			return err
+			return fmt.Errorf("unable to git pull in %s: %s", recipeDir, err)
 		}
 	}
 
@@ -617,55 +650,6 @@ func EnsureUpToDate(recipeName string) error {
 	return nil
 }
 
-func GetDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
-	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-
-	meta, _ := GetRecipeMeta(recipeName)
-	if meta.DefaultBranch != "" {
-		return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", meta.DefaultBranch)), nil
-	}
-
-	branch := "master"
-	if !gitPkg.HasBranch(repo, "master") {
-		if !gitPkg.HasBranch(repo, "main") {
-			return "", fmt.Errorf("failed to select default branch in %s", recipeDir)
-		}
-		branch = "main"
-	}
-
-	return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
-}
-
-func CheckoutDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
-	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-
-	branch, err := GetDefaultBranch(repo, recipeName)
-	if err != nil {
-		return plumbing.ReferenceName(""), err
-	}
-
-	worktree, err := repo.Worktree()
-	if err != nil {
-		return plumbing.ReferenceName(""), err
-	}
-
-	checkOutOpts := &git.CheckoutOptions{
-		Create: false,
-		Force:  true,
-		Branch: branch,
-	}
-
-	if err := worktree.Checkout(checkOutOpts); err != nil {
-		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-		logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
-		return branch, err
-	}
-
-	logrus.Debugf("successfully checked out %v in %s", branch, recipeDir)
-
-	return branch, nil
-}
-
 type CatalogueOfflineError struct {
 	msg string
 }
@@ -715,7 +699,7 @@ func recipeCatalogueFSIsLatest() (bool, error) {
 func ReadRecipeCatalogue() (RecipeCatalogue, error) {
 	recipes := make(RecipeCatalogue)
 
-	if err := EnsureCatalogue(); err != nil {
+	if err := catalogue.EnsureCatalogue(); err != nil {
 		return nil, err
 	}
 
@@ -811,7 +795,7 @@ func VersionsOfService(recipe, serviceName string) ([]string, error) {
 }
 
 // GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
-func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
+func GetRecipeMeta(recipeName string, conf *runtime.Config) (RecipeMeta, error) {
 	catl, err := ReadRecipeCatalogue()
 	if err != nil {
 		return RecipeMeta{}, err
@@ -822,7 +806,7 @@ func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
 		return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
 	}
 
-	if err := EnsureExists(recipeName); err != nil {
+	if err := EnsureExists(recipeName, conf); err != nil {
 		return RecipeMeta{}, err
 	}
 
@@ -944,8 +928,9 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 }
 
 // GetRecipeVersions retrieves all recipe versions.
-func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
+func GetRecipeVersions(recipeName string, opts ...runtime.Option) (RecipeVersions, error) {
 	versions := RecipeVersions{}
+	conf := runtime.New(opts...)
 
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
@@ -983,17 +968,11 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 
 		logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
 
-		recipe, err := Get(recipeName)
+		recipe, err := Get(recipeName, conf)
 		if err != nil {
 			return err
 		}
 
-		cl, err := client.New("default") // only required for container registry calls
-		if err != nil {
-			return err
-		}
-
-		queryCache := make(map[reference.Named]string)
 		versionMeta := make(map[string]ServiceMeta)
 		for _, service := range recipe.Config.Services {
 
@@ -1015,28 +994,9 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 				continue
 			}
 
-			var exists bool
-			var digest string
-			if digest, exists = queryCache[img]; !exists {
-				logrus.Debugf("cache miss: querying for image: %s, tag: %s", path, tag)
-
-				var err error
-				digest, err = client.GetTagDigest(cl, img)
-				if err != nil {
-					logrus.Warn(err)
-					digest = "unknown"
-				}
-
-				queryCache[img] = digest
-				logrus.Debugf("cached insert: %s, tag: %s, digest: %s", path, tag, digest)
-			} else {
-				logrus.Debugf("cache hit: image: %s, tag: %s, digest: %s", path, tag, digest)
-			}
-
 			versionMeta[service.Name] = ServiceMeta{
-				Digest: digest,
-				Image:  path,
-				Tag:    img.(reference.NamedTagged).Tag(),
+				Image: path,
+				Tag:   tag,
 			}
 		}
 
@@ -1047,7 +1007,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 		return versions, err
 	}
 
-	_, err = CheckoutDefaultBranch(repo, recipeName)
+	_, err = gitPkg.CheckoutDefaultBranch(repo, recipeDir)
 	if err != nil {
 		return versions, err
 	}
@@ -1072,16 +1032,52 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri
 	return versions, nil
 }
 
-// EnsureCatalogue ensures that the catalogue is cloned locally & present.
-func EnsureCatalogue() error {
-	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
-	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
-		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
-		if err := gitPkg.Clone(catalogueDir, url); err != nil {
-			return err
-		}
+// UpdateRepositories clones and updates all recipe repositories locally.
+func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
+	var barLength int
+	if recipeName != "" {
+		barLength = 1
+	} else {
+		barLength = len(repos)
+	}
 
-		logrus.Debugf("cloned catalogue repository to %s", catalogueDir)
+	cloneLimiter := limit.New(10)
+
+	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
+	ch := make(chan string, barLength)
+	for _, repoMeta := range repos {
+		go func(rm RepoMeta) {
+			cloneLimiter.Begin()
+			defer cloneLimiter.End()
+
+			if recipeName != "" && recipeName != rm.Name {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
+			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
+
+			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
+
+			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := EnsureUpToDate(rm.Name); err != nil {
+				logrus.Fatal(err)
+			}
+
+			ch <- rm.Name
+			retrieveBar.Add(1)
+		}(repoMeta)
+	}
+
+	for range repos {
+		<-ch // wait for everything
 	}
 
 	return nil

pkg/runtime/runtime.go

@@ -0,0 +1,58 @@
+package runtime
+
+import "github.com/sirupsen/logrus"
+
+// Config is an internal configuration modifier. It can be instantiated on a
+// command call and can be changed on the fly to help make decisions further
+// down in the internals, e.g. whether or not to clone the recipe locally or
+// not.
+type Config struct {
+	EnsureRecipeExists bool // ensure that the recipe is cloned locally
+	EnsureRecipeLatest bool // ensure the local recipe has latest changes
+}
+
+// Option modified a Config.
+type Option func(c *Config)
+
+// New instantiates a new Config.
+func New(opts ...Option) *Config {
+	conf := &Config{
+		EnsureRecipeExists: true,
+	}
+
+	for _, optFunc := range opts {
+		optFunc(conf)
+	}
+
+	return conf
+}
+
+// WithEnsureRecipeExists determines whether or not we should be cloning the
+// local recipe or not. This can be useful for being more adaptable to offline
+// scenarios.
+func WithEnsureRecipeExists(ensureRecipeExists bool) Option {
+	return func(c *Config) {
+		if ensureRecipeExists {
+			logrus.Debugf("runtime config: EnsureRecipeExists = %v, ensuring recipes are cloned", ensureRecipeExists)
+		} else {
+			logrus.Debugf("runtime config: EnsureRecipeExists = %v, not cloning recipes", ensureRecipeExists)
+		}
+
+		c.EnsureRecipeExists = ensureRecipeExists
+	}
+}
+
+// WithEnsureRecipeLatest determines whether we should update the local recipes
+// remotely via Git. This can be useful when e.g. ensuring we have the latest
+// changes before making new ones.
+func WithEnsureRecipeLatest(ensureRecipeLatest bool) Option {
+	return func(c *Config) {
+		if ensureRecipeLatest {
+			logrus.Debugf("runtime config: EnsureRecipeLatest = %v, ensuring recipes have latest changes", ensureRecipeLatest)
+		} else {
+			logrus.Debugf("runtime config: EnsureRecipeLatest = %v, leaving recipes alone", ensureRecipeLatest)
+		}
+
+		c.EnsureRecipeLatest = ensureRecipeLatest
+	}
+}

pkg/secret/secret.go

@@ -13,6 +13,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/decentral1se/passgen"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
@@ -117,7 +118,7 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
+func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
 	secrets := make(map[string]string)
 
 	var mutex sync.Mutex
@@ -146,7 +147,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					return
 				}
 
-				if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
 						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil
@@ -166,7 +167,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					return
 				}
 
-				if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
 						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil

pkg/ssh/ssh.go

@@ -1,37 +1,13 @@
 package ssh
 
 import (
-	"bufio"
-	"bytes"
-	"crypto/sha256"
-	"encoding/base64"
 	"fmt"
-	"io"
-	"net"
-	"os"
-	"os/user"
-	"path/filepath"
+	"os/exec"
 	"strings"
-	"sync"
-	"time"
 
-	"coopcloud.tech/abra/pkg/context"
-	"github.com/AlecAivazis/survey/v2"
-	dockerSSHPkg "github.com/docker/cli/cli/connhelper/ssh"
-	sshPkg "github.com/gliderlabs/ssh"
-	"github.com/kevinburke/ssh_config"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/agent"
-	"golang.org/x/crypto/ssh/knownhosts"
 )
 
-var KnownHostsPath = filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts")
-
-type Client struct {
-	SSHClient *ssh.Client
-}
-
 // HostConfig is a SSH host config.
 type HostConfig struct {
 	Host         string
@@ -40,509 +16,70 @@ type HostConfig struct {
 	User         string
 }
 
-// Exec cmd on the remote host and return stderr and stdout
-func (c *Client) Exec(cmd string) ([]byte, error) {
-	session, err := c.SSHClient.NewSession()
-	if err != nil {
-		return nil, err
-	}
-	defer session.Close()
-
-	return session.CombinedOutput(cmd)
-}
-
-// Close the underlying SSH connection
-func (c *Client) Close() error {
-	return c.SSHClient.Close()
-}
-
-// New creates a new SSH client connection.
-func New(domainName, sshAuth, username, port string) (*Client, error) {
-	var client *Client
-
-	ctxConnDetails, err := GetContextConnDetails(domainName)
-	if err != nil {
-		return client, nil
-	}
-
-	if sshAuth == "identity-file" {
-		var err error
-		client, err = connectWithAgentTimeout(
-			ctxConnDetails.Host,
-			ctxConnDetails.User,
-			ctxConnDetails.Port,
-			5*time.Second,
-		)
-		if err != nil {
-			return client, err
-		}
-	} else {
-		password := ""
-		prompt := &survey.Password{
-			Message: "SSH password?",
-		}
-		if err := survey.AskOne(prompt, &password); err != nil {
-			return client, err
-		}
-
-		var err error
-		client, err = connectWithPasswordTimeout(
-			ctxConnDetails.Host,
-			ctxConnDetails.User,
-			ctxConnDetails.Port,
-			password,
-			5*time.Second,
-		)
-		if err != nil {
-			return client, err
-		}
-	}
-
-	return client, nil
-}
-
-// sudoWriter supports sudo command handling
-type sudoWriter struct {
-	b     bytes.Buffer
-	pw    string
-	stdin io.Writer
-	m     sync.Mutex
-}
-
-// Write satisfies the write interface for sudoWriter
-func (w *sudoWriter) Write(p []byte) (int, error) {
-	if strings.Contains(string(p), "sudo_password") {
-		w.stdin.Write([]byte(w.pw + "\n"))
-		w.pw = ""
-		return len(p), nil
-	}
-
-	w.m.Lock()
-	defer w.m.Unlock()
-
-	return w.b.Write(p)
-}
-
-// RunSudoCmd runs SSH commands and streams output
-func RunSudoCmd(cmd, passwd string, cl *Client) error {
-	session, err := cl.SSHClient.NewSession()
-	if err != nil {
-		return err
-	}
-	defer session.Close()
-
-	sudoCmd := fmt.Sprintf("SSH_ASKPASS=/usr/bin/ssh-askpass; sudo -p sudo_password -S %s", cmd)
-
-	w := &sudoWriter{pw: passwd}
-	w.stdin, err = session.StdinPipe()
-	if err != nil {
-		return err
-	}
-
-	session.Stdout = w
-	session.Stderr = w
-
-	modes := ssh.TerminalModes{
-		ssh.ECHO:          0,
-		ssh.TTY_OP_ISPEED: 14400,
-		ssh.TTY_OP_OSPEED: 14400,
-	}
-
-	err = session.RequestPty("xterm", 80, 40, modes)
-	if err != nil {
-		return err
-	}
-
-	if err := session.Run(sudoCmd); err != nil {
-		return fmt.Errorf("%s", string(w.b.Bytes()))
-	}
-
-	return nil
-}
-
-// EnsureKnowHostsFiles ensures that ~/.ssh/known_hosts is created
-func EnsureKnowHostsFiles() error {
-	if _, err := os.Stat(KnownHostsPath); os.IsNotExist(err) {
-		logrus.Debugf("missing %s, creating now", KnownHostsPath)
-		file, err := os.OpenFile(KnownHostsPath, os.O_CREATE, 0600)
-		if err != nil {
-			return err
-		}
-		file.Close()
-	}
-
-	return nil
-}
-
-// GetHostKey checks if a host key is registered in the ~/.ssh/known_hosts file
-func GetHostKey(hostname string) (bool, sshPkg.PublicKey, error) {
-	var hostKey sshPkg.PublicKey
-
-	ctxConnDetails, err := GetContextConnDetails(hostname)
-	if err != nil {
-		return false, hostKey, err
-	}
-
-	if err := EnsureKnowHostsFiles(); err != nil {
-		return false, hostKey, err
-	}
-
-	file, err := os.Open(KnownHostsPath)
-	if err != nil {
-		return false, hostKey, err
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		fields := strings.Split(scanner.Text(), " ")
-		if len(fields) != 3 {
-			continue
-		}
-
-		hostnameAndPort := fmt.Sprintf("%s:%s", ctxConnDetails.Host, ctxConnDetails.Port)
-		hashed := knownhosts.Normalize(hostnameAndPort)
-
-		if strings.Contains(fields[0], hashed) {
-			var err error
-			hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes())
-			if err != nil {
-				return false, hostKey, fmt.Errorf("error parsing server SSH host key %q: %v", fields[2], err)
-			}
-			break
-		}
-	}
-
-	if hostKey != nil {
-		logrus.Debugf("server SSH host key present in ~/.ssh/known_hosts for %s", hostname)
-		return true, hostKey, nil
-	}
-
-	return false, hostKey, nil
-}
-
-// InsertHostKey adds a new host key to the ~/.ssh/known_hosts file
-func InsertHostKey(hostname string, remote net.Addr, pubKey ssh.PublicKey) error {
-	file, err := os.OpenFile(KnownHostsPath, os.O_APPEND|os.O_WRONLY, 0600)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	hashedHostname := knownhosts.Normalize(hostname)
-	lineHostname := knownhosts.Line([]string{hashedHostname}, pubKey)
-	_, err = file.WriteString(fmt.Sprintf("%s\n", lineHostname))
-	if err != nil {
-		return err
-	}
-
-	hashedRemote := knownhosts.Normalize(remote.String())
-	lineRemote := knownhosts.Line([]string{hashedRemote}, pubKey)
-	_, err = file.WriteString(fmt.Sprintf("%s\n", lineRemote))
-	if err != nil {
-		return err
-	}
-
-	logrus.Debugf("SSH host key generated: %s", lineHostname)
-	logrus.Debugf("SSH host key generated: %s", lineRemote)
-
-	return nil
-}
-
-// HostKeyAddCallback ensures server ssh host keys are handled
-func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.PublicKey) error {
-	exists, _, err := GetHostKey(hostnameAndPort)
-	if err != nil {
-		return err
-	}
-
-	if exists {
-		hostname := strings.Split(hostnameAndPort, ":")[0]
-		logrus.Debugf("server SSH host key found for %s", hostname)
-		return nil
-	}
-
-	if !exists {
-		hostname := strings.Split(hostnameAndPort, ":")[0]
-		parsedPubKey := FingerprintSHA256(pubKey)
-
-		fmt.Printf(fmt.Sprintf(`
-You are attempting to make an SSH connection to a server but there is no entry
-in your ~/.ssh/known_hosts file which confirms that you have already validated
-that this is indeed the server you want to connect to. Please take a moment to
-validate the following SSH host key, it is important.
-
-    Host:        %s
-    Fingerprint: %s
-
-If this is confusing to you, you can read the article below and learn how to
-validate this fingerprint safely. Thanks to the comrades at cyberia.club for
-writing this extensive guide <3
-
-    https://sequentialread.com/understanding-the-secure-shell-protocol-ssh/
-
-`, hostname, parsedPubKey))
-
-		response := false
-		prompt := &survey.Confirm{
-			Message: "are you sure you trust this host key?",
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return err
-		}
-
-		if !response {
-			logrus.Fatal("exiting as requested")
-		}
-
-		logrus.Debugf("attempting to insert server SSH host key for %s, %s", hostnameAndPort, remote)
-
-		if err := InsertHostKey(hostnameAndPort, remote, pubKey); err != nil {
-			return err
-		}
-
-		logrus.Infof("successfully added server SSH host key for %s", hostname)
-	}
-
-	return nil
-}
-
-// connect makes the SSH connection
-func connect(username, host, port string, authMethod ssh.AuthMethod, timeout time.Duration) (*Client, error) {
-	config := &ssh.ClientConfig{
-		User:            username,
-		Auth:            []ssh.AuthMethod{authMethod},
-		HostKeyCallback: HostKeyAddCallback, // the main reason why we fork
-	}
-
-	hostnameAndPort := fmt.Sprintf("%s:%s", host, port)
-
-	logrus.Debugf("tcp dialing %s", hostnameAndPort)
-
-	var conn net.Conn
-	var err error
-	conn, err = net.DialTimeout("tcp", hostnameAndPort, timeout)
-	if err != nil {
-		logrus.Debugf("tcp dialing %s failed, trying via ~/.ssh/config", hostnameAndPort)
-		hostConfig, err := GetHostConfig(host, username, port, true)
-		if err != nil {
-			return nil, err
-		}
-		conn, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostConfig.Host, hostConfig.Port), timeout)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	sshConn, chans, reqs, err := ssh.NewClientConn(conn, hostnameAndPort, config)
-	if err != nil {
-		return nil, err
-	}
-
-	client := ssh.NewClient(sshConn, chans, reqs)
-	c := &Client{SSHClient: client}
-
-	return c, nil
-}
-
-func connectWithAgentTimeout(host, username, port string, timeout time.Duration) (*Client, error) {
-	logrus.Debugf("using ssh-agent to make an SSH connection for %s", host)
-
-	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
-	if err != nil {
-		return nil, err
-	}
-
-	agentCl := agent.NewClient(sshAgent)
-	authMethod := ssh.PublicKeysCallback(agentCl.Signers)
-
-	loadedKeys, err := agentCl.List()
-	if err != nil {
-		return nil, err
-	}
-
-	var convertedKeys []string
-	for _, key := range loadedKeys {
-		convertedKeys = append(convertedKeys, key.String())
-	}
-
-	if len(convertedKeys) > 0 {
-		logrus.Debugf("ssh-agent has these keys loaded: %s", strings.Join(convertedKeys, ","))
-	} else {
-		logrus.Debug("ssh-agent has no keys loaded")
-	}
-
-	return connect(username, host, port, authMethod, timeout)
-}
-
-func connectWithPasswordTimeout(host, username, port, pass string, timeout time.Duration) (*Client, error) {
-	authMethod := ssh.Password(pass)
-
-	return connect(username, host, port, authMethod, timeout)
-}
-
-// EnsureHostKey ensures that a host key trusted and added to the ~/.ssh/known_hosts file
-func EnsureHostKey(hostname string) error {
-	if hostname == "default" || hostname == "local" {
-		logrus.Debugf("not checking server SSH host key against local/default target")
-		return nil
-	}
-
-	exists, _, err := GetHostKey(hostname)
-	if err != nil {
-		return err
-	}
-	if exists {
-		return nil
-	}
-
-	ctxConnDetails, err := GetContextConnDetails(hostname)
-	if err != nil {
-		return err
-	}
-
-	_, err = connectWithAgentTimeout(
-		ctxConnDetails.Host,
-		ctxConnDetails.User,
-		ctxConnDetails.Port,
-		5*time.Second,
+// String presents a human friendly output for the HostConfig.
+func (h HostConfig) String() string {
+	return fmt.Sprintf(
+		"{host: %s, username: %s, port: %s, identityfile: %s}",
+		h.Host,
+		h.User,
+		h.Port,
+		h.IdentityFile,
 	)
-
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
-// FingerprintSHA256 generates the SHA256 fingerprint for a server SSH host key
-func FingerprintSHA256(key ssh.PublicKey) string {
-	hash := sha256.Sum256(key.Marshal())
-	b64hash := base64.StdEncoding.EncodeToString(hash[:])
-	trimmed := strings.TrimRight(b64hash, "=")
-	return fmt.Sprintf("SHA256:%s", trimmed)
-}
-
-// GetContextConnDetails retrieves SSH connection details from a docker context endpoint
-func GetContextConnDetails(serverName string) (*dockerSSHPkg.Spec, error) {
-	dockerContextStore := context.NewDefaultDockerContextStore()
-	contexts, err := dockerContextStore.Store.List()
-	if err != nil {
-		return &dockerSSHPkg.Spec{}, err
-	}
-
-	if strings.Contains(serverName, ":") {
-		serverName = strings.Split(serverName, ":")[0]
-	}
-
-	for _, ctx := range contexts {
-		endpoint, err := context.GetContextEndpoint(ctx)
-		if err != nil && strings.Contains(err.Error(), "does not exist") {
-			// No local context found, we can continue safely
-			continue
-		}
-		if ctx.Name == serverName {
-			ctxConnDetails, err := dockerSSHPkg.ParseURL(endpoint)
-			if err != nil {
-				return &dockerSSHPkg.Spec{}, err
-			}
-			logrus.Debugf("found context connection details %v for %s", ctxConnDetails, serverName)
-			return ctxConnDetails, nil
-		}
-	}
-
-	hostConfig, err := GetHostConfig(serverName, "", "", false)
-	if err != nil {
-		return &dockerSSHPkg.Spec{}, err
-	}
-
-	logrus.Debugf("couldn't find a docker context matching %s", serverName)
-	logrus.Debugf("searching ~/.ssh/config for a Host entry for %s", serverName)
-
-	connDetails := &dockerSSHPkg.Spec{
-		Host: hostConfig.Host,
-		User: hostConfig.User,
-		Port: hostConfig.Port,
-	}
-
-	logrus.Debugf("using %v from ~/.ssh/config for connection details", connDetails)
-
-	return connDetails, nil
-}
-
-// GetHostConfig retrieves a ~/.ssh/config config for a host.
-func GetHostConfig(hostname, username, port string, override bool) (HostConfig, error) {
+// GetHostConfig retrieves a ~/.ssh/config config for a host using /usr/bin/ssh
+// directly. We therefore maintain consistent interop with this standard
+// tooling. This is useful because SSH confuses a lot of people and having to
+// learn how two tools (`ssh` and `abra`) handle SSH connection details instead
+// of one (just `ssh`) is Not Cool. Here's to less bug reports on this topic!
+func GetHostConfig(hostname string) (HostConfig, error) {
 	var hostConfig HostConfig
 
-	if hostname == "" || override {
-		if sshHost := ssh_config.Get(hostname, "Hostname"); sshHost != "" {
-			hostname = sshHost
-		}
+	out, err := exec.Command("ssh", "-G", hostname).Output()
+	if err != nil {
+		return hostConfig, err
 	}
 
-	if username == "" || override {
-		if sshUser := ssh_config.Get(hostname, "User"); sshUser != "" {
-			username = sshUser
-		} else {
-			systemUser, err := user.Current()
-			if err != nil {
-				return hostConfig, err
+	for _, line := range strings.Split(string(out), "\n") {
+		entries := strings.Split(line, " ")
+		for idx, entry := range entries {
+			if entry == "hostname" {
+				hostConfig.Host = entries[idx+1]
 			}
-			username = systemUser.Username
-		}
-	}
-
-	if port == "" || override {
-		if sshPort := ssh_config.Get(hostname, "Port"); sshPort != "" {
-			// skip override probably correct port with dummy default value from
-			// ssh_config which is 22. only when the original port number is empty
-			// should we try this default. this might not cover all cases
-			// unfortunately.
-			if port != "" && sshPort != "22" {
-				port = sshPort
+			if entry == "user" {
+				hostConfig.User = entries[idx+1]
+			}
+			if entry == "port" {
+				hostConfig.Port = entries[idx+1]
+			}
+			if entry == "identityfile" {
+				if hostConfig.IdentityFile == "" {
+					hostConfig.IdentityFile = entries[idx+1]
+				}
 			}
 		}
 	}
 
-	if idf := ssh_config.Get(hostname, "IdentityFile"); idf != "" && idf != "~/.ssh/identity" {
-		var err error
-		idf, err = identityFileAbsPath(idf)
-		if err != nil {
-			return hostConfig, err
-		}
-		hostConfig.IdentityFile = idf
-	} else {
-		hostConfig.IdentityFile = ""
-	}
-
-	hostConfig.Host = hostname
-	hostConfig.Port = port
-	hostConfig.User = username
-
-	logrus.Debugf("constructed SSH config %s for %s", hostConfig, hostname)
+	logrus.Debugf("retrieved ssh config for %s: %s", hostname, hostConfig.String())
 
 	return hostConfig, nil
 }
 
-func identityFileAbsPath(relPath string) (string, error) {
-	var err error
-	var absPath string
-
-	if strings.HasPrefix(relPath, "~/") {
-		systemUser, err := user.Current()
-		if err != nil {
-			return absPath, err
-		}
-		absPath = filepath.Join(systemUser.HomeDir, relPath[2:])
+// Fatal is a error output wrapper which aims to make SSH failures easier to
+// parse through re-wording.
+func Fatal(hostname string, err error) error {
+	out := err.Error()
+	if strings.Contains(out, "Host key verification failed.") {
+		return fmt.Errorf("SSH host key verification failed for %s", hostname)
+	} else if strings.Contains(out, "Could not resolve hostname") {
+		return fmt.Errorf("could not resolve hostname for %s", hostname)
+	} else if strings.Contains(out, "Connection timed out") {
+		return fmt.Errorf("connection timed out for %s", hostname)
+	} else if strings.Contains(out, "Permission denied") {
+		return fmt.Errorf("ssh auth: permission denied for %s", hostname)
+	} else if strings.Contains(out, "Network is unreachable") {
+		return fmt.Errorf("unable to connect to %s, network is unreachable?", hostname)
 	} else {
-		absPath, err = filepath.Abs(relPath)
-		if err != nil {
-			return absPath, err
-		}
+		return err
 	}
-
-	logrus.Debugf("resolved %s to %s to read the ssh identity file", relPath, absPath)
-
-	return absPath, nil
 }

pkg/upstream/commandconn/connection.go

@@ -2,18 +2,15 @@ package commandconn
 
 import (
 	"context"
-	"fmt"
 	"net"
 	"net/url"
 
-	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	"github.com/docker/cli/cli/connhelper"
 	"github.com/docker/cli/cli/connhelper/ssh"
 	"github.com/docker/cli/cli/context/docker"
 	dCliContextStore "github.com/docker/cli/cli/context/store"
 	dClient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 // GetConnectionHelper returns Docker-specific connection helper for the given URL.
@@ -36,25 +33,6 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 			return nil, errors.Wrap(err, "ssh host connection is not valid")
 		}
 
-		if err := sshPkg.EnsureHostKey(ctxConnDetails.Host); err != nil {
-			return nil, err
-		}
-
-		hostConfig, err := sshPkg.GetHostConfig(
-			ctxConnDetails.Host,
-			ctxConnDetails.User,
-			ctxConnDetails.Port,
-			false,
-		)
-		if err != nil {
-			return nil, err
-		}
-		if hostConfig.IdentityFile != "" {
-			msg := "discovered %s as identity file for %s, using for ssh connection"
-			logrus.Debugf(msg, hostConfig.IdentityFile, ctxConnDetails.Host)
-			sshFlags = append(sshFlags, fmt.Sprintf("-o IdentityFile=%s", hostConfig.IdentityFile))
-		}
-
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)

pkg/upstream/stack/stack.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 	"time"
 
-	abraClient "coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"github.com/docker/cli/cli/command/service/progress"
+	"github.com/docker/cli/cli/command/stack/formatter"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -18,7 +18,7 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/client"
-	dockerclient "github.com/docker/docker/client"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -57,20 +57,10 @@ func GetStackServices(ctx context.Context, dockerclient client.APIClient, namesp
 }
 
 // GetDeployedServicesByLabel filters services by label
-func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
-	cl, err := abraClient.New(contextName)
-	if err != nil {
-		if strings.Contains(err.Error(), "does not exist") {
-			// No local context found, bail out gracefully
-			return StackStatus{[]swarm.Service{}, nil}
-		}
-		return StackStatus{[]swarm.Service{}, err}
-	}
-
-	ctx := context.Background()
+func GetDeployedServicesByLabel(cl *dockerClient.Client, contextName string, label string) StackStatus {
 	filters := filters.NewArgs()
 	filters.Add("label", label)
-	services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: filters})
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filters})
 	if err != nil {
 		return StackStatus{[]swarm.Service{}, err}
 	}
@@ -78,18 +68,8 @@ func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
 	return StackStatus{services, nil}
 }
 
-func GetAllDeployedServices(contextName string) StackStatus {
-	cl, err := abraClient.New(contextName)
-	if err != nil {
-		if strings.Contains(err.Error(), "does not exist") {
-			// No local context found, bail out gracefully
-			return StackStatus{[]swarm.Service{}, nil}
-		}
-		return StackStatus{[]swarm.Service{}, err}
-	}
-
-	ctx := context.Background()
-	services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: getAllStacksFilter()})
+func GetAllDeployedServices(cl *dockerClient.Client, contextName string) StackStatus {
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: getAllStacksFilter()})
 	if err != nil {
 		return StackStatus{[]swarm.Service{}, err}
 	}
@@ -98,7 +78,7 @@ func GetAllDeployedServices(contextName string) StackStatus {
 }
 
 // GetDeployedServicesByName filters services by name
-func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, stackName, serviceName string) StackStatus {
+func GetDeployedServicesByName(ctx context.Context, cl *dockerClient.Client, stackName, serviceName string) StackStatus {
 	filters := filters.NewArgs()
 	filters.Add("name", fmt.Sprintf("%s_%s", stackName, serviceName))
 
@@ -111,7 +91,7 @@ func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, sta
 }
 
 // IsDeployed chekcks whether an appp is deployed or not.
-func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string) (bool, string, error) {
+func IsDeployed(ctx context.Context, cl *dockerClient.Client, stackName string) (bool, string, error) {
 	version := "unknown"
 	isDeployed := false
 
@@ -142,7 +122,7 @@ func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string)
 }
 
 // pruneServices removes services that are no longer referenced in the source
-func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, services map[string]struct{}) {
+func pruneServices(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, services map[string]struct{}) {
 	oldServices, err := GetStackServices(ctx, cl, namespace.Name())
 	if err != nil {
 		logrus.Infof("Failed to list services: %s\n", err)
@@ -158,9 +138,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve
 }
 
 // RunDeploy is the swarm implementation of docker stack deploy
-func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
-	ctx := context.Background()
-
+func RunDeploy(cl *dockerClient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
 	if err := validateResolveImageFlag(&opts); err != nil {
 		return err
 	}
@@ -170,7 +148,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, a
 		opts.ResolveImage = ResolveImageNever
 	}
 
-	return deployCompose(ctx, cl, opts, cfg, appName, dontWait)
+	return deployCompose(context.Background(), cl, opts, cfg, appName, dontWait)
 }
 
 // validateResolveImageFlag validates the opts.resolveImage command line option
@@ -183,7 +161,7 @@ func validateResolveImageFlag(opts *Deploy) error {
 	}
 }
 
-func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
+func deployCompose(ctx context.Context, cl *dockerClient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
 	namespace := convert.NewNamespace(opts.Namespace)
 
 	if opts.Prune {
@@ -241,7 +219,7 @@ func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) ma
 	return serviceNetworks
 }
 
-func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAPIClient, externalNetworks []string) error {
+func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAPIClient, externalNetworks []string) error {
 	for _, networkName := range externalNetworks {
 		if !container.NetworkMode(networkName).IsUserDefined() {
 			// Networks that are not user defined always exist on all nodes as
@@ -250,7 +228,7 @@ func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAP
 		}
 		network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
 		switch {
-		case dockerclient.IsErrNotFound(err):
+		case dockerClient.IsErrNotFound(err):
 			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
 		case err != nil:
 			return err
@@ -261,7 +239,7 @@ func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAP
 	return nil
 }
 
-func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm.SecretSpec) error {
+func createSecrets(ctx context.Context, cl *dockerClient.Client, secrets []swarm.SecretSpec) error {
 	for _, secretSpec := range secrets {
 		secret, _, err := cl.SecretInspectWithRaw(ctx, secretSpec.Name)
 		switch {
@@ -270,7 +248,7 @@ func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm
 			if err := cl.SecretUpdate(ctx, secret.ID, secret.Meta.Version, secretSpec); err != nil {
 				return errors.Wrapf(err, "failed to update secret %s", secretSpec.Name)
 			}
-		case dockerclient.IsErrNotFound(err):
+		case dockerClient.IsErrNotFound(err):
 			// secret does not exist, then we create a new one.
 			logrus.Infof("Creating secret %s\n", secretSpec.Name)
 			if _, err := cl.SecretCreate(ctx, secretSpec); err != nil {
@@ -283,7 +261,7 @@ func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm
 	return nil
 }
 
-func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm.ConfigSpec) error {
+func createConfigs(ctx context.Context, cl *dockerClient.Client, configs []swarm.ConfigSpec) error {
 	for _, configSpec := range configs {
 		config, _, err := cl.ConfigInspectWithRaw(ctx, configSpec.Name)
 		switch {
@@ -292,7 +270,7 @@ func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm
 			if err := cl.ConfigUpdate(ctx, config.ID, config.Meta.Version, configSpec); err != nil {
 				return errors.Wrapf(err, "failed to update config %s", configSpec.Name)
 			}
-		case dockerclient.IsErrNotFound(err):
+		case dockerClient.IsErrNotFound(err):
 			// config does not exist, then we create a new one.
 			logrus.Infof("Creating config %s\n", configSpec.Name)
 			if _, err := cl.ConfigCreate(ctx, configSpec); err != nil {
@@ -305,7 +283,7 @@ func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm
 	return nil
 }
 
-func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
+func createNetworks(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
 	existingNetworks, err := getStackNetworks(ctx, cl, namespace.Name())
 	if err != nil {
 		return err
@@ -335,7 +313,7 @@ func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace conv
 
 func deployServices(
 	ctx context.Context,
-	cl *dockerclient.Client,
+	cl *dockerClient.Client,
 	services map[string]swarm.ServiceSpec,
 	namespace convert.Namespace,
 	sendAuth bool,
@@ -469,7 +447,7 @@ func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespa
 
 // https://github.com/docker/cli/blob/master/cli/command/service/helpers.go
 // https://github.com/docker/cli/blob/master/cli/command/service/progress/progress.go
-func WaitOnService(ctx context.Context, cl *dockerclient.Client, serviceID, appName string) error {
+func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appName string) error {
 	errChan := make(chan error, 1)
 	pipeReader, pipeWriter := io.Pipe()
 
@@ -507,3 +485,37 @@ If a service is failing to even start, try smoke out the error with:
 `, appName, timeout, appName, appName, appName))
 	}
 }
+
+// Copypasta from https://github.com/docker/cli/blob/master/cli/command/stack/swarm/list.go
+// GetStacks lists the swarm stacks.
+func GetStacks(cl *dockerClient.Client) ([]*formatter.Stack, error) {
+	services, err := cl.ServiceList(
+		context.Background(),
+		types.ServiceListOptions{Filters: getAllStacksFilter()})
+	if err != nil {
+		return nil, err
+	}
+	m := make(map[string]*formatter.Stack)
+	for _, service := range services {
+		labels := service.Spec.Labels
+		name, ok := labels[convert.LabelNamespace]
+		if !ok {
+			return nil, errors.Errorf("cannot get label %s for service %s",
+				convert.LabelNamespace, service.ID)
+		}
+		ztack, ok := m[name]
+		if !ok {
+			m[name] = &formatter.Stack{
+				Name:     name,
+				Services: 1,
+			}
+		} else {
+			ztack.Services++
+		}
+	}
+	var stacks []*formatter.Stack
+	for _, stack := range m {
+		stacks = append(stacks, stack)
+	}
+	return stacks, nil
+}

renovate.json

@@ -1,5 +1,12 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "packageRules": [{
+    "matchUpdateTypes": ["minor", "patch"],
+    "automerge": true
+  }],
+  "postUpdateOptions": [
+    "gomodTidy"
+  ],
   "ignoreDeps": [
     "github.com/urfave/cli"
   ]

scripts/installer/installer

@@ -2,7 +2,7 @@
 
 ABRA_VERSION="0.6.0-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.6.0-beta"
+RC_VERSION="0.7.0-rc2-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do