chore: release 0.4.0-alpha-rc4

Follows 1b37d2d5f5.

.e2e.env.sample

@@ -0,0 +1,4 @@
+GANDI_TOKEN=...
+HCLOUD_TOKEN=...
+REGISTRY_PASSWORD=...
+REGISTRY_USERNAME=...

.gitignore

@@ -1,6 +1,7 @@
-abra
-.vscode/
-vendor/
-.envrc
-dist/
 *fmtcoverage.html
+.e2e.env
+.envrc
+.vscode/
+abra
+dist/
+vendor/

Makefile

@@ -5,7 +5,7 @@ LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 export GOPRIVATE=coopcloud.tech
 
-all: run test install build clean format check static
+all: format check static build test
 
 run:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
@@ -43,3 +43,15 @@ loc-author:
    	sort -f | \
    	uniq -ic | \
    	sort -n
+
+int-core:
+	@docker run \
+		-v $$(pwd):/src \
+		--env-file .e2e.env \
+		debian:bullseye-slim \
+		sh -c "\
+			apt update && apt install -y wget curl git; echo ""; echo ""; \
+			git config --global user.email 'e2e@coopcloud.tech'; \
+			git config --global user.name 'e2e'; \
+			cd /src/tests/integration && bash core.sh -- --dev \
+		"

README.md

@@ -9,6 +9,20 @@ The Co-op Cloud utility belt 🎩🐇
 
 `abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create apps, deploy them, run backup and restore operations and a whole lot of other things. Please see [docs.coopcloud.tech](https://docs.coopcloud.tech) for more extensive documentation.
 
+## Quick install
+
+```bash
+curl https://install.abra.autonomic.zone | bash
+```
+
+Or using the latest release candidate (extra experimental!):
+
+```bash
+curl https://install.abra.autonomic.zone | bash -s -- --rc
+```
+
+Source for this script is in [scripts/installer/installer](./scripts/installer/installer).
+
 ## Hacking
 
 ### Getting started

cli/app/app.go

@@ -7,7 +7,7 @@ import (
 // AppCommand defines the `abra app` command and ets subcommands
 var AppCommand = &cli.Command{
 	Name:      "app",
-	Usage:     "Manage deployed apps",
+	Usage:     "Manage apps",
 	Aliases:   []string{"a"},
 	ArgsUsage: "<app>",
 	Description: `
@@ -18,6 +18,7 @@ to scaling apps up and spinning them down.
 	Subcommands: []*cli.Command{
 		appNewCommand,
 		appConfigCommand,
+		appRestartCommand,
 		appDeployCommand,
 		appUpgradeCommand,
 		appUndeployCommand,
@@ -34,5 +35,6 @@ to scaling apps up and spinning them down.
 		appSecretCommand,
 		appVolumeCommand,
 		appVersionCommand,
+		appErrorsCommand,
 	},
 }

cli/app/backup.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -37,10 +38,10 @@ var appBackupCommand = &cli.Command{
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<service>' and '--all' together"))
 		}
 
-		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		abraSh := path.Join(config.RECIPES_DIR, app.Type, "abra.sh")
 		if _, err := os.Stat(abraSh); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("'%s' does not exist?", abraSh)
+				logrus.Fatalf("%s does not exist?", abraSh)
 			}
 			logrus.Fatal(err)
 		}
@@ -61,7 +62,7 @@ var appBackupCommand = &cli.Command{
 			logrus.Fatal(err)
 		}
 		if !strings.Contains(string(bytes), execCmd) {
-			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+			logrus.Fatalf("%s doesn't have a %s function", app.Type, execCmd)
 		}
 
 		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
@@ -72,16 +73,5 @@ var appBackupCommand = &cli.Command{
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/check.go

@@ -1,12 +1,12 @@
 package app
 
 import (
-	"fmt"
 	"os"
 	"path"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -20,10 +20,10 @@ var appCheckCommand = &cli.Command{
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		envSamplePath := path.Join(config.ABRA_DIR, "apps", app.Type, ".env.sample")
+		envSamplePath := path.Join(config.RECIPES_DIR, app.Type, ".env.sample")
 		if _, err := os.Stat(envSamplePath); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("'%s' does not exist?", envSamplePath)
+				logrus.Fatalf("%s does not exist?", envSamplePath)
 			}
 			logrus.Fatal(err)
 		}
@@ -45,20 +45,9 @@ var appCheckCommand = &cli.Command{
 			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
 		}
 
-		logrus.Infof("all necessary environment variables defined for '%s'", app.Name)
+		logrus.Infof("all necessary environment variables defined for %s", app.Name)
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/config.go

@@ -2,11 +2,11 @@ package app
 
 import (
 	"errors"
-	"fmt"
 	"os"
 	"os/exec"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
@@ -31,7 +31,7 @@ var appConfigCommand = &cli.Command{
 
 		appFile, exists := files[appName]
 		if !exists {
-			logrus.Fatalf("cannot find app with name '%s'", appName)
+			logrus.Fatalf("cannot find app with name %s", appName)
 		}
 
 		ed, ok := os.LookupEnv("EDITOR")
@@ -55,16 +55,5 @@ var appConfigCommand = &cli.Command{
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cp.go

@@ -5,10 +5,12 @@ import (
 	"os"
 	"strings"
 
-	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/pkg/archive"
@@ -21,6 +23,18 @@ var appCpCommand = &cli.Command{
 	Aliases:   []string{"c"},
 	ArgsUsage: "<src> <dst>",
 	Usage:     "Copy files to/from a running app service",
+	Description: `
+This command supports copying files to and from any app service file system.
+
+If you want to copy a myfile.txt to the root of the app service:
+
+    abra app cp <app> myfile.txt app:/
+
+And if you want to copy that file back to your current working directory locally:
+
+		abra app cp <app> app:/myfile.txt .
+
+`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -64,61 +78,78 @@ var appCpCommand = &cli.Command{
 			logrus.Debugf("assuming transfer is going TO the container")
 		}
 
-		appFiles, err := config.LoadAppFiles("")
+		if !isToContainer {
+			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", dstPath)
+			}
+		}
+		err := configureAndCp(c, app, srcPath, dstPath, service, isToContainer)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
-		appEnv, err := config.GetApp(appFiles, app.Name)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		filters := filters.NewArgs()
-		filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
-		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(containers) != 1 {
-			logrus.Fatalf("expected 1 container but got %v", len(containers))
-		}
-		container := containers[0]
-
-		logrus.Debugf("retrieved '%s' as target container on '%s'", formatter.ShortenID(container.ID), app.Server)
-
-		if isToContainer {
-			if _, err := os.Stat(srcPath); err != nil {
-				logrus.Fatalf("'%s' does not exist?", srcPath)
-			}
-
-			toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-			content, err := archive.TarWithOptions(srcPath, toTarOpts)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-			if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			defer content.Close()
-			fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-			if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-				logrus.Fatal(err)
-			}
-		}
 		return nil
+
 	},
+	BashComplete: autocomplete.AppNameComplete,
+}
+
+func configureAndCp(
+	c *cli.Context,
+	app config.App,
+	srcPath string,
+	dstPath string,
+	service string,
+	isToContainer bool) error {
+	appFiles, err := config.LoadAppFiles("")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	appEnv, err := config.GetApp(appFiles, app.Name)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
+
+	container, err := container.GetContainer(c.Context, cl, filters, true)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+
+	if isToContainer {
+		if _, err := os.Stat(srcPath); err != nil {
+			logrus.Fatalf("%s does not exist?", srcPath)
+		}
+
+		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		content, err := archive.TarWithOptions(srcPath, toTarOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+		if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		defer content.Close()
+		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+	return nil
 }

cli/app/deploy.go

@@ -1,11 +1,8 @@
 package app
 
 import (
-	"fmt"
-
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/config"
-	"github.com/sirupsen/logrus"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"github.com/urfave/cli/v2"
 )
 
@@ -16,11 +13,13 @@ var appDeployCommand = &cli.Command{
 	Flags: []cli.Flag{
 		internal.ForceFlag,
 		internal.ChaosFlag,
+		internal.NoDomainChecksFlag,
+		internal.DontWaitConvergeFlag,
 	},
 	Description: `
-This command deploys a new instance of an app. It does not support changing the
-version of an existing deployed app, for this you need to look at the "abra app
-upgrade <app>" command.
+This command deploys an app. It does not support incrementing the version of a
+deployed app, for this you need to look at the "abra app upgrade <app>"
+command.
 
 You may pass "--force" to re-deploy the same version again. This can be useful
 if the container runtime has gotten into a weird state.
@@ -29,17 +28,6 @@ Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	Action: internal.DeployAction,
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	Action:       internal.DeployAction,
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/errors.go

@@ -0,0 +1,136 @@
+package app
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appErrorsCommand = &cli.Command{
+	Name:  "errors",
+	Usage: "List errors for a deployed app",
+	Description: `
+This command lists errors for a deployed app.
+
+This is a best-effort implementation and an attempt to gather a number of tips
+& tricks for finding errors together into one convenient command. When an app
+is failing to deploy or having issues, it could be a lot of things.
+
+This command currently takes into account:
+
+    Is the service deployed?
+    Is the service killed by an OOM error?
+    Is the service reporting an error (like in "ps --no-trunc" output)
+    Is the service healthcheck failing? what are the healthcheck logs?
+
+Got any more ideas? Please let us know:
+
+    https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose
+
+This command is best accompanied by "abra app logs <app>" which may reveal
+further information which can help you debug the cause of an app failure via
+the logs.
+
+`,
+	Aliases:      []string{"e"},
+	Flags:        []cli.Flag{internal.WatchFlag},
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(c.Context, cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		if !internal.Watch {
+			if err := checkErrors(c, cl, app); err != nil {
+				logrus.Fatal(err)
+			}
+			return nil
+		}
+
+		for {
+			if err := checkErrors(c, cl, app); err != nil {
+				logrus.Fatal(err)
+			}
+			time.Sleep(2 * time.Second)
+		}
+
+		return nil
+	},
+}
+
+func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
+	recipe, err := recipe.Get(app.Type)
+	if err != nil {
+		return err
+	}
+
+	for _, service := range recipe.Config.Services {
+		filters := filters.NewArgs()
+		filters.Add("name", service.Name)
+		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			return err
+		}
+
+		if len(containers) == 0 {
+			logrus.Warnf("%s is not up, something seems wrong", service.Name)
+			continue
+		}
+
+		container := containers[0]
+		containerState, err := cl.ContainerInspect(c.Context, container.ID)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if containerState.State.OOMKilled {
+			logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
+		}
+
+		if containerState.State.Error != "" {
+			logrus.Warnf("%s reports this error: %s", service.Name, containerState.State.Error)
+		}
+
+		if containerState.State.Health != nil {
+			if containerState.State.Health.Status != "healthy" {
+				logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
+				logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
+				for _, log := range containerState.State.Health.Log {
+					logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func getServiceName(names []string) string {
+	containerName := strings.Join(names, " ")
+	trimmed := strings.TrimPrefix(containerName, "/")
+	return strings.Split(trimmed, ".")[0]
+}

cli/app/list.go

@@ -5,9 +5,11 @@ import (
 	"sort"
 	"strings"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/ssh"
 	"coopcloud.tech/tagcmp"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -40,6 +42,25 @@ var listAppServerFlag = &cli.StringFlag{
 	Destination: &listAppServer,
 }
 
+type appStatus struct {
+	server  string
+	recipe  string
+	appName string
+	domain  string
+	status  string
+	version string
+	upgrade string
+}
+
+type serverStatus struct {
+	apps             []appStatus
+	appCount         int
+	versionCount     int
+	unversionedCount int
+	latestCount      int
+	upgradeCount     int
+}
+
 var appListCommand = &cli.Command{
 	Name:  "list",
 	Usage: "List all managed apps",
@@ -67,54 +88,79 @@ can take some time.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+
 		sort.Sort(config.ByServerAndType(apps))
 
 		statuses := make(map[string]map[string]string)
-		tableCol := []string{"Server", "Type", "Domain"}
+		var catl recipe.RecipeCatalogue
 		if status {
-			tableCol = append(tableCol, "Status", "Version", "Updates")
+			alreadySeen := make(map[string]bool)
+			for _, app := range apps {
+				if _, ok := alreadySeen[app.Server]; !ok {
+					if err := ssh.EnsureHostKey(app.Server); err != nil {
+						logrus.Fatal(fmt.Sprintf(internal.SSHFailMsg, app.Server))
+					}
+					alreadySeen[app.Server] = true
+				}
+			}
+
 			statuses, err = config.GetAppStatuses(appFiles)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+
+			var err error
+			catl, err = recipe.ReadRecipeCatalogue()
+			if err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		table := abraFormatter.CreateTable(tableCol)
-		table.SetAutoMergeCellsByColumnIndex([]int{0})
-
-		var (
-			versionedAppsCount   int
-			unversionedAppsCount int
-			onLatestCount        int
-			canUpgradeCount      int
-		)
-
+		var totalServersCount int
+		var totalAppsCount int
+		allStats := make(map[string]serverStatus)
 		for _, app := range apps {
-			var tableRow []string
+			var stats serverStatus
+			var ok bool
+			if stats, ok = allStats[app.Server]; !ok {
+				stats = serverStatus{}
+				if appType == "" {
+					// count server, no filtering
+					totalServersCount++
+				}
+			}
+
 			if app.Type == appType || appType == "" {
-				// If type flag is set, check for it, if not, Type == ""
-				tableRow = []string{app.Server, app.Type, app.Domain}
+				if appType != "" {
+					// only count server if matches filter
+					totalServersCount++
+				}
+
+				appStats := appStatus{}
+				stats.appCount++
+				totalAppsCount++
+
 				if status {
-					stackName := app.StackName()
 					status := "unknown"
 					version := "unknown"
-					if statusMeta, ok := statuses[stackName]; ok {
+					if statusMeta, ok := statuses[app.StackName()]; ok {
 						if currentVersion, exists := statusMeta["version"]; exists {
 							version = currentVersion
 						}
 						if statusMeta["status"] != "" {
 							status = statusMeta["status"]
 						}
-						tableRow = append(tableRow, status, version)
-						versionedAppsCount++
+						stats.versionCount++
 					} else {
-						tableRow = append(tableRow, status, version)
-						unversionedAppsCount++
+						stats.unversionedCount++
 					}
 
+					appStats.status = status
+					appStats.version = version
+
 					var newUpdates []string
 					if version != "unknown" {
-						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+						updates, err := recipe.GetRecipeCatalogueVersions(app.Type, catl)
 						if err != nil {
 							logrus.Fatal(err)
 						}
@@ -138,35 +184,80 @@ can take some time.
 
 					if len(newUpdates) == 0 {
 						if version == "unknown" {
-							tableRow = append(tableRow, "unknown")
+							appStats.upgrade = "unknown"
 						} else {
-							tableRow = append(tableRow, "on latest")
-							onLatestCount++
+							appStats.upgrade = "latest"
+							stats.latestCount++
 						}
 					} else {
-						// FIXME: jeezus golang why do you not have a list reverse function
-						for i, j := 0, len(newUpdates)-1; i < j; i, j = i+1, j-1 {
-							newUpdates[i], newUpdates[j] = newUpdates[j], newUpdates[i]
-						}
-						tableRow = append(tableRow, strings.Join(newUpdates, "\n"))
-						canUpgradeCount++
+						newUpdates = internal.ReverseStringList(newUpdates)
+						appStats.upgrade = strings.Join(newUpdates, "\n")
+						stats.upgradeCount++
 					}
 				}
+
+				appStats.server = app.Server
+				appStats.recipe = app.Type
+				appStats.appName = app.Name
+				appStats.domain = app.Domain
+
+				stats.apps = append(stats.apps, appStats)
 			}
-			table.Append(tableRow)
+
+			allStats[app.Server] = stats
 		}
 
-		stats := fmt.Sprintf(
-			"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
-			len(apps),
-			versionedAppsCount,
-			unversionedAppsCount,
-			onLatestCount,
-			canUpgradeCount,
-		)
+		alreadySeen := make(map[string]bool)
+		for _, app := range apps {
+			if _, ok := alreadySeen[app.Server]; ok {
+				continue
+			}
 
-		table.SetCaption(true, stats)
-		table.Render()
+			serverStat := allStats[app.Server]
+
+			tableCol := []string{"recipe", "domain", "app name"}
+			if status {
+				tableCol = append(tableCol, []string{"status", "version", "upgrade"}...)
+			}
+
+			table := formatter.CreateTable(tableCol)
+
+			for _, appStat := range serverStat.apps {
+				tableRow := []string{appStat.recipe, appStat.domain, appStat.appName}
+				if status {
+					tableRow = append(tableRow, []string{appStat.status, appStat.version, appStat.upgrade}...)
+				}
+				table.Append(tableRow)
+			}
+
+			if table.NumLines() > 0 {
+				table.Render()
+
+				if status {
+					fmt.Println(fmt.Sprintf(
+						"server: %s | total apps: %v | versioned: %v | unversioned: %v | latest: %v | upgrade: %v",
+						app.Server,
+						serverStat.appCount,
+						serverStat.versionCount,
+						serverStat.unversionedCount,
+						serverStat.latestCount,
+						serverStat.upgradeCount,
+					))
+				} else {
+					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.appCount))
+				}
+			}
+
+			if len(allStats) > 1 && table.NumLines() > 0 {
+				fmt.Println() // newline separator for multiple servers
+			}
+
+			alreadySeen[app.Server] = true
+		}
+
+		if len(allStats) > 1 {
+			fmt.Println(fmt.Sprintf("total servers: %v | total apps: %v ", totalServersCount, totalAppsCount))
+		}
 
 		return nil
 	},

cli/app/logs.go

@@ -7,8 +7,10 @@ import (
 	"sync"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/service"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
@@ -16,6 +18,15 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+var logOpts = types.ContainerLogsOptions{
+	Details:    false,
+	Follow:     true,
+	ShowStderr: true,
+	ShowStdout: true,
+	Tail:       "20",
+	Timestamps: true,
+}
+
 // stackLogs lists logs for all stack services
 func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
 	filters := filters.NewArgs()
@@ -30,19 +41,14 @@ func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
 	for _, service := range services {
 		wg.Add(1)
 		go func(s string) {
-			logOpts := types.ContainerLogsOptions{
-				Details:    true,
-				Follow:     true,
-				ShowStderr: true,
-				ShowStdout: true,
-				Tail:       "20",
-				Timestamps: true,
+			if internal.StdErrOnly {
+				logOpts.ShowStdout = false
 			}
+
 			logs, err := client.ServiceLogs(c.Context, s, logOpts)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			// defer after err check as any err returns a nil io.ReadCloser
 			defer logs.Close()
 
 			_, err = io.Copy(os.Stdout, logs)
@@ -51,7 +57,9 @@ func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
 			}
 		}(service.ID)
 	}
+
 	wg.Wait()
+
 	os.Exit(0)
 }
 
@@ -60,6 +68,10 @@ var appLogsCommand = &cli.Command{
 	Aliases:   []string{"l"},
 	ArgsUsage: "[<service>]",
 	Usage:     "Tail app logs",
+	Flags: []cli.Flag{
+		internal.StdErrOnlyFlag,
+	},
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -70,55 +82,41 @@ var appLogsCommand = &cli.Command{
 
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
-			logrus.Debug("tailing logs for all app services")
+			logrus.Debugf("tailing logs for all %s services", app.Type)
 			stackLogs(c, app.StackName(), cl)
-		}
-		logrus.Debugf("tailing logs for '%s'", serviceName)
-
-		service := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-		filters := filters.NewArgs()
-		filters.Add("name", service)
-		serviceOpts := types.ServiceListOptions{Filters: filters}
-		services, err := cl.ServiceList(c.Context, serviceOpts)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if len(services) != 1 {
-			logrus.Fatalf("expected 1 service but got %v", len(services))
-		}
-
-		logOpts := types.ContainerLogsOptions{
-			Details:    true,
-			Follow:     true,
-			ShowStderr: true,
-			ShowStdout: true,
-			Tail:       "20",
-			Timestamps: true,
-		}
-		logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		// defer after err check as any err returns a nil io.ReadCloser
-		defer logs.Close()
-
-		_, err = io.Copy(os.Stdout, logs)
-		if err != nil && err != io.EOF {
-			logrus.Fatal(err)
+		} else {
+			logrus.Debugf("tailing logs for %s", serviceName)
+			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+}
+
+func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
+	chosenService, err := service.GetService(c.Context, cl, filters, internal.NoInput)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if internal.StdErrOnly {
+		logOpts.ShowStdout = false
+	}
+
+	logs, err := cl.ServiceLogs(c.Context, chosenService.ID, logOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	defer logs.Close()
+
+	_, err = io.Copy(os.Stdout, logs)
+	if err != nil && err != io.EOF {
+		logrus.Fatal(err)
+	}
+
+	return nil
 }

cli/app/new.go

@@ -1,11 +1,8 @@
 package app
 
 import (
-	"fmt"
-
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
-	"github.com/sirupsen/logrus"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"github.com/urfave/cli/v2"
 )
 
@@ -41,18 +38,7 @@ var appNewCommand = &cli.Command{
 		internal.PassFlag,
 		internal.SecretsFlag,
 	},
-	ArgsUsage: "<recipe>",
-	Action:    internal.NewAction,
-	BashComplete: func(c *cli.Context) {
-		catl, err := catalogue.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for name := range catl {
-			fmt.Println(name)
-		}
-	},
+	ArgsUsage:    "<recipe>",
+	Action:       internal.NewAction,
+	BashComplete: autocomplete.RecipeNameComplete,
 }

cli/app/ps.go

@@ -1,72 +1,68 @@
 package app
 
 import (
-	"fmt"
 	"strings"
 	"time"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"github.com/docker/cli/cli/command/formatter"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/buger/goterm"
+	dockerFormatter "github.com/docker/cli/cli/command/formatter"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
-var watch bool
-var watchFlag = &cli.BoolFlag{
-	Name:        "watch",
-	Aliases:     []string{"w"},
-	Value:       false,
-	Usage:       "Watch status by polling repeatedly",
-	Destination: &watch,
-}
-
 var appPsCommand = &cli.Command{
-	Name:    "ps",
-	Usage:   "Check app status",
-	Aliases: []string{"p"},
+	Name:        "ps",
+	Usage:       "Check app status",
+	Description: "This command shows a more detailed status output of a specific deployed app.",
+	Aliases:     []string{"p"},
 	Flags: []cli.Flag{
-		watchFlag,
+		internal.WatchFlag,
 	},
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		if !watch {
-			showPSOutput(c)
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(c.Context, cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		if !internal.Watch {
+			showPSOutput(c, app, cl)
 			return nil
 		}
 
-		// TODO: how do we make this update in-place in an x-platform way?
+		goterm.Clear()
 		for {
-			showPSOutput(c)
+			goterm.MoveCursor(1, 1)
+			showPSOutput(c, app, cl)
+			goterm.Flush()
 			time.Sleep(2 * time.Second)
 		}
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
 }
 
 // showPSOutput renders ps output.
-func showPSOutput(c *cli.Context) {
-	app := internal.ValidateApp(c)
-
-	cl, err := client.New(app.Server)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
+func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
 	filters := filters.NewArgs()
 	filters.Add("name", app.StackName())
 
@@ -75,8 +71,8 @@ func showPSOutput(c *cli.Context) {
 		logrus.Fatal(err)
 	}
 
-	tableCol := []string{"image", "created", "status", "ports", "names"}
-	table := abraFormatter.CreateTable(tableCol)
+	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
+	table := formatter.CreateTable(tableCol)
 
 	for _, container := range containers {
 		var containerNames []string
@@ -86,11 +82,12 @@ func showPSOutput(c *cli.Context) {
 		}
 
 		tableRow := []string{
-			abraFormatter.RemoveSha(container.Image),
-			abraFormatter.HumanDuration(container.Created),
+			service.ContainerToServiceName(container.Names, app.StackName()),
+			formatter.RemoveSha(container.Image),
+			formatter.HumanDuration(container.Created),
 			container.Status,
-			formatter.DisplayablePorts(container.Ports),
-			strings.Join(containerNames, "\n"),
+			container.State,
+			dockerFormatter.DisplayablePorts(container.Ports),
 		}
 		table.Append(tableRow)
 	}

cli/app/remove.go

@@ -5,8 +5,9 @@ import (
 	"os"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -38,38 +39,31 @@ var appRemoveCommand = &cli.Command{
 		if !internal.Force {
 			response := false
 			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("about to delete %s, are you sure?", app.Name),
+				Message: fmt.Sprintf("about to remove %s, are you sure?", app.Name),
 			}
 			if err := survey.AskOne(prompt, &response); err != nil {
 				logrus.Fatal(err)
 			}
 			if !response {
-				logrus.Fatal("user aborted app removal")
+				logrus.Fatal("aborting as requested")
 			}
 		}
 
-		appFiles, err := config.LoadAppFiles("")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		if !internal.Force {
-			// FIXME: only query for app we are interested in, not all of them!
-			statuses, err := config.GetAppStatuses(appFiles)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			if statuses[app.Name]["status"] == "deployed" {
-				logrus.Fatalf("'%s' is still deployed. Run \"abra app %s undeploy\" or pass --force", app.Name, app.Name)
-			}
+
+		isDeployed, _, err := stack.IsDeployed(c.Context, cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if isDeployed {
+			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s \" or pass --force", app.Name, app.Name)
 		}
 
 		fs := filters.NewArgs()
-		fs.Add("name", app.Name)
+		fs.Add("name", app.StackName())
 		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: fs})
 		if err != nil {
 			logrus.Fatal(err)
@@ -85,9 +79,12 @@ var appRemoveCommand = &cli.Command{
 
 		if len(secrets) > 0 {
 			var secretNamesToRemove []string
+
 			if !internal.Force {
 				secretsPrompt := &survey.MultiSelect{
 					Message: "which secrets do you want to remove?",
+					Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
+					VimMode: true,
 					Options: secretNames,
 					Default: secretNames,
 				}
@@ -124,6 +121,8 @@ var appRemoveCommand = &cli.Command{
 				if !internal.Force {
 					volumesPrompt := &survey.MultiSelect{
 						Message: "which volumes do you want to remove?",
+						Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
+						VimMode: true,
 						Options: vols,
 						Default: vols,
 					}
@@ -142,7 +141,9 @@ var appRemoveCommand = &cli.Command{
 				logrus.Info("no volumes were removed")
 			}
 		} else {
-			logrus.Info("no volumes to remove")
+			if Volumes {
+				logrus.Info("no volumes to remove")
+			}
 		}
 
 		err = os.Remove(app.Path)
@@ -153,16 +154,5 @@ var appRemoveCommand = &cli.Command{
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/restart.go

@@ -0,0 +1,65 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	upstream "coopcloud.tech/abra/pkg/upstream/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appRestartCommand = &cli.Command{
+	Name:         "restart",
+	Usage:        "Restart an app",
+	Aliases:      []string{"re"},
+	ArgsUsage:    "<service>",
+	Description:  `This command restarts a service within a deployed app.`,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		serviceNameShort := c.Args().Get(1)
+		if serviceNameShort == "" {
+			err := errors.New("missing service?")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
+
+		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)
+		if err := upstream.RunServiceScale(c.Context, cl, serviceName, 0); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.WaitOnService(c.Context, cl, serviceName, app.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("%s has been scaled to 0 (restart logic)", serviceName)
+
+		logrus.Debugf("attempting to scale %s to 1 (restart logic)", serviceName)
+		if err := upstream.RunServiceScale(c.Context, cl, serviceName, 1); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.WaitOnService(c.Context, cl, serviceName, app.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("%s has been scaled to 1 (restart logic)", serviceName)
+
+		logrus.Infof("%s service successfully restarted", serviceNameShort)
+
+		return nil
+	},
+}

cli/app/restore.go

@@ -27,7 +27,7 @@ var restoreAllServicesFlag = &cli.BoolFlag{
 var appRestoreCommand = &cli.Command{
 	Name:      "restore",
 	Usage:     "Restore an app from a backup",
-	Aliases:   []string{"r"},
+	Aliases:   []string{"rs"},
 	Flags:     []cli.Flag{restoreAllServicesFlag},
 	ArgsUsage: "<service> [<backup file>]",
 	Action: func(c *cli.Context) error {
@@ -37,10 +37,10 @@ var appRestoreCommand = &cli.Command{
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use <service>/<backup file> and '--all' together"))
 		}
 
-		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		abraSh := path.Join(config.RECIPES_DIR, app.Type, "abra.sh")
 		if _, err := os.Stat(abraSh); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("'%s' does not exist?", abraSh)
+				logrus.Fatalf("%s does not exist?", abraSh)
 			}
 			logrus.Fatal(err)
 		}
@@ -60,7 +60,7 @@ var appRestoreCommand = &cli.Command{
 			logrus.Fatal(err)
 		}
 		if !strings.Contains(string(bytes), execCmd) {
-			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+			logrus.Fatalf("%s doesn't have a %s function", app.Type, execCmd)
 		}
 
 		backupFile := c.Args().Get(2)

cli/app/rollback.go

@@ -3,8 +3,9 @@ package app
 import (
 	"fmt"
 
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
@@ -19,11 +20,12 @@ import (
 var appRollbackCommand = &cli.Command{
 	Name:      "rollback",
 	Usage:     "Roll an app back to a previous version",
-	Aliases:   []string{"r", "downgrade"},
+	Aliases:   []string{"rl"},
 	ArgsUsage: "<app>",
 	Flags: []cli.Flag{
 		internal.ForceFlag,
 		internal.ChaosFlag,
+		internal.DontWaitConvergeFlag,
 	},
 	Description: `
 This command rolls an app back to a previous version if one exists.
@@ -38,28 +40,30 @@ Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if err := recipe.EnsureUpToDate(app.Type); err != nil {
+			logrus.Fatal(err)
+		}
+
+		r, err := recipe.Get(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(r); err != nil {
+			logrus.Fatal(err)
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+		logrus.Debugf("checking whether %s is already deployed", stackName)
 
 		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
@@ -67,19 +71,27 @@ recipes.
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("'%s' is not deployed?", app.Name)
+			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := recipe.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(versions) == 0 && !internal.Chaos {
+			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Type)
+		}
+
 		var availableDowngrades []string
-		if deployedVersion == "" {
-			deployedVersion = "unknown"
+		if deployedVersion == "unknown" {
 			availableDowngrades = versions
-			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
+			logrus.Warnf("failed to determine version of deployed %s", app.Name)
 		}
 
 		if deployedVersion != "unknown" && !internal.Chaos {
@@ -98,23 +110,21 @@ recipes.
 			}
 
 			if len(availableDowngrades) == 0 {
-				logrus.Fatal("no available downgrades, you're on latest")
+				logrus.Info("no available downgrades, you're on oldest ✌️")
+				return nil
 			}
 		}
 
-		// FIXME: jeezus golang why do you not have a list reverse function
-		for i, j := 0, len(availableDowngrades)-1; i < j; i, j = i+1, j-1 {
-			availableDowngrades[i], availableDowngrades[j] = availableDowngrades[j], availableDowngrades[i]
-		}
+		availableDowngrades = internal.ReverseStringList(availableDowngrades)
 
 		var chosenDowngrade string
 		if !internal.Chaos {
 			if internal.Force {
 				chosenDowngrade = availableDowngrades[0]
-				logrus.Debugf("choosing '%s' as version to downgrade to (--force)", chosenDowngrade)
+				logrus.Debugf("choosing %s as version to downgrade to (--force)", chosenDowngrade)
 			} else {
 				prompt := &survey.Select{
-					Message: fmt.Sprintf("Please select a downgrade (current version: '%s'):", deployedVersion),
+					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
 					Options: availableDowngrades,
 				}
 				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
@@ -138,7 +148,7 @@ recipes.
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Type, "abra.sh")
 		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
@@ -163,12 +173,12 @@ recipes.
 		}
 
 		if !internal.Force {
-			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade); err != nil {
+			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/run.go

@@ -5,8 +5,9 @@ import (
 	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -35,9 +36,10 @@ var appRunCommand = &cli.Command{
 		noTTYFlag,
 		userFlag,
 	},
-	Aliases:   []string{"r"},
-	ArgsUsage: "<service> <args>...",
-	Usage:     "Run a command in a service container",
+	Aliases:      []string{"r"},
+	ArgsUsage:    "<service> <args>...",
+	Usage:        "Run a command in a service container",
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -59,18 +61,11 @@ var appRunCommand = &cli.Command{
 		filters := filters.NewArgs()
 		filters.Add("name", stackAndServiceName)
 
-		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+		targetContainer, err := containerPkg.GetContainer(c.Context, cl, filters, true)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if len(containers) == 0 {
-			logrus.Fatalf("no containers matching '%s' found?", stackAndServiceName)
-		}
-		if len(containers) > 1 {
-			logrus.Fatalf("expected 1 container matching '%s' but got %d", stackAndServiceName, len(containers))
-		}
-
 		cmd := c.Args().Slice()[2:]
 		execCreateOpts := types.ExecConfig{
 			AttachStderr: true,
@@ -88,41 +83,16 @@ var appRunCommand = &cli.Command{
 			execCreateOpts.Tty = false
 		}
 
-		// FIXME: an absolutely monumental hack to instantiate another command-line
-		// client withing our command-line client so that we pass something down
-		// the tubes that satisfies the necessary interface requirements. We should
-		// refactor our vendored container code to not require all this cruft.  For
-		// now, It Works.
+		// FIXME: avoid instantiating a new CLI
 		dcli, err := command.NewDockerCli()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, containers[0].ID, &execCreateOpts); err != nil {
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		switch c.NArg() {
-		case 0:
-			appNames, err := config.GetAppNames()
-			if err != nil {
-				logrus.Warn(err)
-			}
-			for _, a := range appNames {
-				fmt.Println(a)
-			}
-		case 1:
-			appName := c.Args().First()
-			serviceNames, err := config.GetAppServiceNames(appName)
-			if err != nil {
-				logrus.Warn(err)
-			}
-			for _, s := range serviceNames {
-				fmt.Println(s)
-			}
-		}
-	},
 }

cli/app/secret.go

@@ -6,10 +6,10 @@ import (
 	"os"
 	"strconv"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -20,18 +20,19 @@ import (
 var allSecrets bool
 var allSecretsFlag = &cli.BoolFlag{
 	Name:        "all",
-	Aliases:     []string{"A"},
+	Aliases:     []string{"a"},
 	Value:       false,
 	Destination: &allSecrets,
 	Usage:       "Generate all secrets",
 }
 
 var appSecretGenerateCommand = &cli.Command{
-	Name:      "generate",
-	Aliases:   []string{"g"},
-	Usage:     "Generate secrets",
-	ArgsUsage: "<secret> <version>",
-	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
+	Name:         "generate",
+	Aliases:      []string{"g"},
+	Usage:        "Generate secrets",
+	ArgsUsage:    "<secret> <version>",
+	Flags:        []cli.Flag{allSecretsFlag, internal.PassFlag},
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -60,7 +61,7 @@ var appSecretGenerateCommand = &cli.Command{
 				}
 			}
 			if !matches {
-				logrus.Fatalf("'%s' doesn't exist in the env config?", secretName)
+				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
 		}
 
@@ -83,7 +84,7 @@ var appSecretGenerateCommand = &cli.Command{
 		}
 
 		tableCol := []string{"name", "value"}
-		table := abraFormatter.CreateTable(tableCol)
+		table := formatter.CreateTable(tableCol)
 		for name, val := range secretVals {
 			table.Append([]string{name, val})
 		}
@@ -95,11 +96,12 @@ var appSecretGenerateCommand = &cli.Command{
 }
 
 var appSecretInsertCommand = &cli.Command{
-	Name:      "insert",
-	Aliases:   []string{"i"},
-	Usage:     "Insert secret",
-	Flags:     []cli.Flag{internal.PassFlag},
-	ArgsUsage: "<app> <secret-name> <version> <data>",
+	Name:         "insert",
+	Aliases:      []string{"i"},
+	Usage:        "Insert secret",
+	Flags:        []cli.Flag{internal.PassFlag},
+	ArgsUsage:    "<app> <secret-name> <version> <data>",
+	BashComplete: autocomplete.AppNameComplete,
 	Description: `
 This command inserts a secret into an app environment.
 
@@ -139,11 +141,12 @@ Example:
 }
 
 var appSecretRmCommand = &cli.Command{
-	Name:      "remove",
-	Usage:     "Remove a secret",
-	Aliases:   []string{"rm"},
-	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
-	ArgsUsage: "<app> <secret-name>",
+	Name:         "remove",
+	Usage:        "Remove a secret",
+	Aliases:      []string{"rm"},
+	Flags:        []cli.Flag{allSecretsFlag, internal.PassFlag},
+	ArgsUsage:    "<app> <secret-name>",
+	BashComplete: autocomplete.AppNameComplete,
 	Description: `
 This command removes a secret from an app environment.
 
@@ -215,7 +218,7 @@ var appSecretLsCommand = &cli.Command{
 		secrets := secret.ReadSecretEnvVars(app.Env)
 
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
-		table := abraFormatter.CreateTable(tableCol)
+		table := formatter.CreateTable(tableCol)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -249,21 +252,15 @@ var appSecretLsCommand = &cli.Command{
 			table.Append(tableRow)
 		}
 
-		table.Render()
+		if table.NumLines() > 0 {
+			table.Render()
+		} else {
+			logrus.Warnf("no secrets stored for %s", app.Name)
+		}
+
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appSecretCommand = &cli.Command{

cli/app/undeploy.go

@@ -1,11 +1,9 @@
 package app
 
 import (
-	"fmt"
-
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -13,7 +11,7 @@ import (
 
 var appUndeployCommand = &cli.Command{
 	Name:    "undeploy",
-	Aliases: []string{"u"},
+	Aliases: []string{"un"},
 	Usage:   "Undeploy an app",
 	Description: `
 This does not destroy any of the application data. However, you should remain
@@ -29,7 +27,7 @@ volumes as eligiblef or pruning once undeployed.
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+		logrus.Debugf("checking whether %s is already deployed", stackName)
 
 		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
@@ -37,7 +35,7 @@ volumes as eligiblef or pruning once undeployed.
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("'%s' is not deployed?", stackName)
+			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
 		if err := internal.DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
@@ -51,16 +49,5 @@ volumes as eligiblef or pruning once undeployed.
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/upgrade.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
@@ -17,18 +18,19 @@ import (
 
 var appUpgradeCommand = &cli.Command{
 	Name:      "upgrade",
-	Aliases:   []string{"u"},
+	Aliases:   []string{"up"},
 	Usage:     "Upgrade an app",
 	ArgsUsage: "<app>",
 	Flags: []cli.Flag{
 		internal.ForceFlag,
 		internal.ChaosFlag,
+		internal.NoDomainChecksFlag,
 	},
 	Description: `
 This command supports upgrading an app. You can use it to choose and roll out a
 new upgrade to an existing app.
 
-This command specifically supports changing the version of running apps, as
+This command specifically supports incrementing the version of running apps, as
 opposed to "abra app deploy <app>" which will not change the version of a
 deployed app.
 
@@ -46,12 +48,25 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if err := recipe.EnsureUpToDate(app.Type); err != nil {
+			logrus.Fatal(err)
+		}
+
+		r, err := recipe.Get(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(r); err != nil {
+			logrus.Fatal(err)
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+		logrus.Debugf("checking whether %s is already deployed", stackName)
 
 		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
@@ -59,23 +74,27 @@ recipes.
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("'%s' is not deployed?", app.Name)
+			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := recipe.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no versions available '%s' in recipe catalogue?", app.Type)
+			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Type)
 		}
 
 		var availableUpgrades []string
-		if deployedVersion == "" {
-			deployedVersion = "unknown"
+		if deployedVersion == "uknown" {
 			availableUpgrades = versions
-			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
+			logrus.Warnf("failed to determine version of deployed %s", app.Name)
 		}
 
 		if deployedVersion != "unknown" && !internal.Chaos {
@@ -94,19 +113,21 @@ recipes.
 			}
 
 			if len(availableUpgrades) == 0 && !internal.Force {
-				logrus.Fatal("no available upgrades, you're on latest")
-				availableUpgrades = versions
+				logrus.Infof("no available upgrades, you're on latest (%s) ✌️", deployedVersion)
+				return nil
 			}
 		}
 
+		availableUpgrades = internal.ReverseStringList(availableUpgrades)
+
 		var chosenUpgrade string
 		if len(availableUpgrades) > 0 && !internal.Chaos {
 			if internal.Force {
 				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
-				logrus.Debugf("choosing '%s' as version to upgrade to", chosenUpgrade)
+				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
 			} else {
 				prompt := &survey.Select{
-					Message: fmt.Sprintf("Please select an upgrade (current version: '%s'):", deployedVersion),
+					Message: fmt.Sprintf("Please select an upgrade (current version: %s):", deployedVersion),
 					Options: availableUpgrades,
 				}
 				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
@@ -115,6 +136,14 @@ recipes.
 			}
 		}
 
+		// if release notes written after git tag published, read them before we
+		// check out the tag and then they'll appear to be missing. this covers
+		// when we obviously will forget to write release notes before publishing
+		releaseNotes, err := internal.GetReleaseNotes(app.Type, chosenUpgrade)
+		if err != nil {
+			return err
+		}
+
 		if !internal.Chaos {
 			if err := recipe.EnsureVersion(app.Type, chosenUpgrade); err != nil {
 				logrus.Fatal(err)
@@ -130,7 +159,7 @@ recipes.
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Type, "abra.sh")
 		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
@@ -154,26 +183,15 @@ recipes.
 			logrus.Fatal(err)
 		}
 
-		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade); err != nil {
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/version.go

@@ -1,14 +1,11 @@
 package app
 
 import (
-	"fmt"
-	"strings"
-
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
@@ -21,11 +18,13 @@ func getImagePath(image string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	path := reference.Path(img)
-	if strings.Contains(path, "library") {
-		path = strings.Split(path, "/")[1]
-	}
-	logrus.Debugf("parsed '%s' from '%s'", path, image)
+
+	path = recipe.StripTagMeta(path)
+
+	logrus.Debugf("parsed %s from %s", path, image)
+
 	return path, nil
 }
 
@@ -47,27 +46,27 @@ Cloud recipe version.
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+		logrus.Debugf("checking whether %s is already deployed", stackName)
 
 		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if deployedVersion == "" {
-			logrus.Fatalf("failed to determine version of deployed '%s'", app.Name)
+		if deployedVersion == "unknown" {
+			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("'%s' is not deployed?", app.Name)
+			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := catalogue.GetRecipeMeta(app.Type)
+		recipeMeta, err := recipe.GetRecipeMeta(app.Type)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versionsMeta := make(map[string]catalogue.ServiceMeta)
+		versionsMeta := make(map[string]recipe.ServiceMeta)
 		for _, recipeVersion := range recipeMeta.Versions {
 			if currentVersion, exists := recipeVersion[deployedVersion]; exists {
 				versionsMeta = currentVersion
@@ -75,30 +74,20 @@ Cloud recipe version.
 		}
 
 		if len(versionsMeta) == 0 {
-			logrus.Fatalf("PANIC: could not retrieve deployed version ('%s') from recipe catalogue?", deployedVersion)
+			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"name", "image", "version", "tag", "digest"}
-		table := abraFormatter.CreateTable(tableCol)
+		tableCol := []string{"version", "service", "image", "digest"}
+		table := formatter.CreateTable(tableCol)
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{serviceName, versionMeta.Image, deployedVersion, versionMeta.Tag, versionMeta.Digest})
+			table.Append([]string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Digest})
 		}
 
 		table.Render()
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/volume.go

@@ -1,21 +1,20 @@
 package app
 
 import (
-	"fmt"
-
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 var appVolumeListCommand = &cli.Command{
-	Name:    "list",
-	Usage:   "List volumes associated with an app",
-	Aliases: []string{"ls"},
+	Name:         "list",
+	Usage:        "List volumes associated with an app",
+	Aliases:      []string{"ls"},
+	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -24,7 +23,7 @@ var appVolumeListCommand = &cli.Command{
 			logrus.Fatal(err)
 		}
 
-		table := abraFormatter.CreateTable([]string{"driver", "volume name"})
+		table := formatter.CreateTable([]string{"driver", "volume name"})
 		var volTable [][]string
 		for _, volume := range volumeList {
 			volRow := []string{
@@ -35,16 +34,33 @@ var appVolumeListCommand = &cli.Command{
 		}
 
 		table.AppendBulk(volTable)
-		table.Render()
+
+		if table.NumLines() > 0 {
+			table.Render()
+		} else {
+			logrus.Warnf("no volumes created for %s", app.Name)
+		}
 
 		return nil
 	},
 }
 
 var appVolumeRemoveCommand = &cli.Command{
-	Name:    "remove",
-	Usage:   "Remove volume(s) associated with an app",
-	Aliases: []string{"rm"},
+	Name:  "remove",
+	Usage: "Remove volume(s) associated with an app",
+	Description: `
+This command supports removing volumes associated with an app. The app in
+question must be undeployed before you try to remove volumes. See "abra app
+undeploy <app>" for more.
+
+The command is interactive and will show a multiple select input which allows
+you to make a seclection. Use the "?" key to see more help on navigating this
+interface.
+
+Passing "--force" will select all volumes for removal. Be careful.
+`,
+	ArgsUsage: "<app>",
+	Aliases:   []string{"rm"},
 	Flags: []cli.Flag{
 		internal.ForceFlag,
 	},
@@ -61,6 +77,8 @@ var appVolumeRemoveCommand = &cli.Command{
 		if !internal.Force {
 			volumesPrompt := &survey.MultiSelect{
 				Message: "which volumes do you want to remove?",
+				Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
+				VimMode: true,
 				Options: volumeNames,
 				Default: volumeNames,
 			}
@@ -80,18 +98,7 @@ var appVolumeRemoveCommand = &cli.Command{
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
+	BashComplete: autocomplete.AppNameComplete,
 }
 
 var appVolumeCommand = &cli.Command{

cli/autocomplete.go

@@ -1,121 +0,0 @@
-package cli
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/config"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-// downloadFile downloads a file brah
-func downloadFile(filepath string, url string) (err error) {
-	out, err := os.Create(filepath)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad status: %s", resp.Status)
-	}
-
-	_, err = io.Copy(out, resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// AutoCompleteCommand helps people set up auto-complete in their shells
-var AutoCompleteCommand = &cli.Command{
-	Name:    "autocomplete",
-	Usage:   "Help set up shell autocompletion",
-	Aliases: []string{"ac"},
-	Description: `
-This command helps set up autocompletion in your shell by downloading the
-relevant autocompletion files and laying out what additional information must
-be loaded.
-
-Example:
-
-    abra autocomplete bash
-
-Supported shells are as follows:
-
-		fish
-		zsh
-    bash
-`,
-	ArgsUsage: "<shell>",
-	Action: func(c *cli.Context) error {
-		shellType := c.Args().First()
-
-		if shellType == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
-		}
-
-		supportedShells := map[string]bool{
-			"bash": true,
-			"zsh":  true,
-			"fish": true,
-		}
-
-		if _, ok := supportedShells[shellType]; !ok {
-			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
-		}
-
-		if shellType == "fish" {
-			shellType = "zsh" // handled the same on the autocompletion side
-		}
-
-		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
-		if err := os.Mkdir(autocompletionDir, 0755); err != nil {
-			if !os.IsExist(err) {
-				logrus.Fatal(err)
-			}
-			logrus.Debugf("'%s' already created, moving on...", autocompletionDir)
-		}
-
-		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
-		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
-			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
-			logrus.Infof("fetching %s", url)
-			if err := downloadFile(autocompletionFile, url); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		switch shellType {
-		case "bash":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install autocompletion
-sudo mkdir /etc/bash/completion.d/
-sudo cp %s /etc/bash_completion.d/abra
-echo "source /etc/bash/completion.d/abra" >> ~/.bashrc
-`, autocompletionFile))
-		case "zsh":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install autocompletion
-sudo mkdir /etc/zsh/completion.d/
-sudo cp %s /etc/zsh/completion.d/abra
-echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
-`, autocompletionFile))
-		}
-
-		return nil
-	},
-}

cli/catalogue/catalogue.go

@@ -1,13 +1,260 @@
 package catalogue
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/limit"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                  true,
+	"abra-apps":             true,
+	"abra-aur":              true,
+	"abra-bash":             true,
+	"abra-capsul":           true,
+	"abra-gandi":            true,
+	"abra-hetzner":          true,
+	"apps":                  true,
+	"aur-abra-git":          true,
+	"auto-apps-json":        true,
+	"auto-mirror":           true,
+	"backup-bot":            true,
+	"backup-bot-two":        true,
+	"comrade-renovate-bot":  true,
+	"coopcloud.tech":        true,
+	"coturn":                true,
+	"docker-cp-deploy":      true,
+	"docker-dind-bats-kcov": true,
+	"docs.coopcloud.tech":   true,
+	"drone-abra":            true,
+	"example":               true,
+	"gardening":             true,
+	"go-abra":               true,
+	"organising":            true,
+	"outline-with-patch":    true,
+	"pyabra":                true,
+	"radicle-seed-node":     true,
+	"recipes":               true,
+	"stack-ssh-deploy":      true,
+	"swarm-cronjob":         true,
+	"tagcmp":                true,
+	"traefik-cert-dumper":   true,
+	"tyop":                  true,
+}
+
+var catalogueGenerateCommand = &cli.Command{
+	Name:    "generate",
+	Aliases: []string{"g"},
+	Usage:   "Generate the recipe catalogue",
+	Flags: []cli.Flag{
+		internal.PublishFlag,
+		internal.DryFlag,
+		internal.SkipUpdatesFlag,
+		internal.RegistryUsernameFlag,
+		internal.RegistryPasswordFlag,
+	},
+	Description: `
+This command generates a new copy of the recipe catalogue which can be found on:
+
+    https://recipes.coopcloud.tech
+
+It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
+listing, parses README.md and git tags of those repositories to produce recipe
+metadata and produces a recipes JSON file.
+
+It is possible to generate new metadata for a single recipe by passing
+<recipe>. The existing local catalogue will be updated, not overwritten.
+
+It is quite easy to get rate limited by Docker Hub when running this command.
+If you have a Hub account you can have Abra log you in to avoid this. Pass
+"--user" and "--pass".
+
+Push your new release git.coopcloud.tech with "-p/--publish". This requires
+that you have permission to git push to these repositories and have your SSH
+keys configured on your account.
+`,
+	ArgsUsage: "[<recipe>]",
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		repos, err := recipe.ReadReposMetadata()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var barLength int
+		var logMsg string
+		if recipeName != "" {
+			barLength = 1
+			logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
+		} else {
+			barLength = len(repos)
+			logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
+		}
+
+		if !internal.SkipUpdates {
+			logrus.Warn(logMsg)
+			if err := updateRepositories(repos, recipeName); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		catl := make(recipe.RecipeCatalogue)
+		catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
+		for _, recipeMeta := range repos {
+			if recipeName != "" && recipeName != recipeMeta.Name {
+				catlBar.Add(1)
+				continue
+			}
+
+			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
+				catlBar.Add(1)
+				continue
+			}
+
+			versions, err := recipe.GetRecipeVersions(
+				recipeMeta.Name,
+				internal.RegistryUsername,
+				internal.RegistryPassword,
+			)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			features, category, err := recipe.GetRecipeFeaturesAndCategory(recipeMeta.Name)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
+			catl[recipeMeta.Name] = recipe.RecipeMeta{
+				Name:          recipeMeta.Name,
+				Repository:    recipeMeta.CloneURL,
+				SSHURL:        recipeMeta.SSHURL,
+				Icon:          recipeMeta.AvatarURL,
+				DefaultBranch: recipeMeta.DefaultBranch,
+				Description:   recipeMeta.Description,
+				Website:       recipeMeta.Website,
+				Versions:      versions,
+				Category:      category,
+				Features:      features,
+			}
+
+			catlBar.Add(1)
+		}
+
+		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if recipeName == "" {
+			if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			catlFS, err := recipe.ReadRecipeCatalogue()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			catlFS[recipeName] = catl[recipeName]
+
+			updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		logrus.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)
+
+		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
+		if internal.Publish {
+
+			isClean, err := gitPkg.IsClean(cataloguePath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if isClean {
+				if !internal.Dry {
+					logrus.Fatalf("no changes discovered in %s, nothing to publish?", cataloguePath)
+				}
+			}
+
+			msg := "chore: publish new catalogue release changes"
+			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
+				logrus.Fatal(err)
+			}
+
+			repo, err := git.PlainOpen(cataloguePath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			sshURL := fmt.Sprintf(config.SSH_URL_TEMPLATE, "recipes")
+			if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		repo, err := git.PlainOpen(cataloguePath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		head, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Dry && internal.Publish {
+			url := fmt.Sprintf("%s/recipes/commit/%s", config.REPOS_BASE_URL, head.Hash())
+			logrus.Infof("new changes published: %s", url)
+		}
+
+		if internal.Dry {
+			logrus.Info("dry run: no changes published")
+		}
+
+		return nil
+	},
+	BashComplete: autocomplete.RecipeNameComplete,
+}
+
 // CatalogueCommand defines the `abra catalogue` command and sub-commands.
 var CatalogueCommand = &cli.Command{
 	Name:        "catalogue",
-	Usage:       "Manage the recipe catalogue (for maintainers)",
+	Usage:       "Manage the recipe catalogue",
 	Aliases:     []string{"c"},
 	ArgsUsage:   "<recipe>",
 	Description: "This command helps recipe packagers interact with the recipe catalogue",
@@ -15,3 +262,62 @@ var CatalogueCommand = &cli.Command{
 		catalogueGenerateCommand,
 	},
 }
+
+func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error {
+	var barLength int
+	if recipeName != "" {
+		barLength = 1
+	} else {
+		barLength = len(repos)
+	}
+
+	cloneLimiter := limit.New(10)
+
+	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
+	ch := make(chan string, barLength)
+	for _, repoMeta := range repos {
+		go func(rm recipe.RepoMeta) {
+			cloneLimiter.Begin()
+			defer cloneLimiter.End()
+
+			if recipeName != "" && recipeName != rm.Name {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
+			if _, exists := CatalogueSkipList[rm.Name]; exists {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
+
+			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
+
+			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
+				logrus.Fatal(err)
+			}
+
+			isClean, err := gitPkg.IsClean(recipeDir)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !isClean {
+				logrus.Fatalf("%s has locally unstaged changes", rm.Name)
+			}
+
+			if err := recipe.EnsureUpToDate(rm.Name); err != nil {
+				logrus.Fatal(err)
+			}
+
+			ch <- rm.Name
+			retrieveBar.Add(1)
+		}(repoMeta)
+	}
+
+	for range repos {
+		<-ch // wait for everything
+	}
+
+	return nil
+}

cli/catalogue/generate.go

@@ -1,261 +0,0 @@
-package catalogue
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-
-	"coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/config"
-	gitPkg "coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/limit"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                  true,
-	"abra-bash":             true,
-	"abra-apps":             true,
-	"abra-aur":              true,
-	"abra-capsul":           true,
-	"abra-gandi":            true,
-	"abra-hetzner":          true,
-	"apps":                  true,
-	"aur-abra-git":          true,
-	"auto-apps-json":        true,
-	"auto-mirror":           true,
-	"backup-bot":            true,
-	"backup-bot-two":        true,
-	"coopcloud.tech":        true,
-	"coturn":                true,
-	"docker-cp-deploy":      true,
-	"docker-dind-bats-kcov": true,
-	"docs.coopcloud.tech":   true,
-	"example":               true,
-	"gardening":             true,
-	"go-abra":               true,
-	"organising":            true,
-	"pyabra":                true,
-	"radicle-seed-node":     true,
-	"stack-ssh-deploy":      true,
-	"swarm-cronjob":         true,
-	"tagcmp":                true,
-	"tyop":                  true,
-}
-
-var commit bool
-var commitFlag = &cli.BoolFlag{
-	Name:        "commit",
-	Usage:       "Commits new generated catalogue changes",
-	Value:       false,
-	Aliases:     []string{"c"},
-	Destination: &commit,
-}
-
-var catalogueGenerateCommand = &cli.Command{
-	Name:    "generate",
-	Aliases: []string{"g"},
-	Usage:   "Generate a new copy of the catalogue",
-	Flags: []cli.Flag{
-		internal.PushFlag,
-		commitFlag,
-		internal.CommitMessageFlag,
-	},
-	Description: `
-This command generates a new copy of the recipe catalogue which can be found on:
-
-    https://recipes.coopcloud.tech
-
-It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
-listing, parses README and tags to produce recipe metadata and produces a
-apps.json file which is placed in your ~/.abra/catalogue/recipes.json.
-
-It is possible to generate new metadata for a single recipe by passing
-<recipe>. The existing local catalogue will be updated, not overwritten.
-
-A new catalogue copy can be published to the recipes repository by passing the
-"--commit" and "--push" flags. The recipes repository is available here:
-
-    https://git.coopcloud.tech/coop-cloud/recipes
-
-`,
-	ArgsUsage: "[<recipe>]",
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-
-		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
-		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
-		if err := gitPkg.Clone(catalogueDir, url); err != nil {
-			return err
-		}
-
-		repos, err := catalogue.ReadReposMetadata()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
-
-		cloneLimiter := limit.New(10)
-		retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
-		ch := make(chan string, len(repos))
-		for _, repoMeta := range repos {
-			go func(rm catalogue.RepoMeta) {
-				cloneLimiter.Begin()
-				defer cloneLimiter.End()
-
-				if recipeName != "" && recipeName != rm.Name {
-					ch <- rm.Name
-					retrieveBar.Add(1)
-					return
-				}
-				if _, exists := CatalogueSkipList[rm.Name]; exists {
-					ch <- rm.Name
-					retrieveBar.Add(1)
-					return
-				}
-
-				recipeDir := path.Join(config.ABRA_DIR, "apps", rm.Name)
-
-				if err := gitPkg.Clone(recipeDir, rm.SSHURL); err != nil {
-					logrus.Fatal(err)
-				}
-
-				if err := gitPkg.EnsureUpToDate(recipeDir); err != nil {
-					logrus.Fatal(err)
-				}
-
-				ch <- rm.Name
-				retrieveBar.Add(1)
-			}(repoMeta)
-		}
-
-		for range repos {
-			<-ch // wait for everything
-		}
-
-		catl := make(catalogue.RecipeCatalogue)
-		catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...")
-		for _, recipeMeta := range repos {
-			if recipeName != "" && recipeName != recipeMeta.Name {
-				catlBar.Add(1)
-				continue
-			}
-
-			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
-			versions, err := catalogue.GetRecipeVersions(recipeMeta.Name)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			catl[recipeMeta.Name] = catalogue.RecipeMeta{
-				Name:          recipeMeta.Name,
-				Repository:    recipeMeta.CloneURL,
-				Icon:          recipeMeta.AvatarURL,
-				DefaultBranch: recipeMeta.DefaultBranch,
-				Description:   recipeMeta.Description,
-				Website:       recipeMeta.Website,
-				Versions:      versions,
-				// Category:      ..., // FIXME: parse & load
-				// Features:      ..., // FIXME: parse & load
-			}
-			catlBar.Add(1)
-		}
-
-		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if _, err := os.Stat(config.APPS_JSON); err != nil && os.IsNotExist(err) {
-			if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if recipeName != "" {
-				catlFS, err := catalogue.ReadRecipeCatalogue()
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				catlFS[recipeName] = catl[recipeName]
-
-				updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				if err := ioutil.WriteFile(config.APPS_JSON, updatedRecipesJSON, 0644); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
-		cataloguePath := path.Join(config.ABRA_DIR, "catalogue", "recipes.json")
-		logrus.Infof("generated new recipe catalogue in %s", cataloguePath)
-
-		if commit {
-			repoPath := path.Join(config.ABRA_DIR, "catalogue")
-			commitRepo, err := git.PlainOpen(repoPath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			commitWorktree, err := commitRepo.Worktree()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if internal.CommitMessage == "" {
-				prompt := &survey.Input{
-					Message: "commit message",
-					Default: "chore: publish new catalogue changes",
-				}
-				if err := survey.AskOne(prompt, &internal.CommitMessage); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			err = commitWorktree.AddGlob("**.json")
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			logrus.Debug("staged **.json for commit")
-
-			_, err = commitWorktree.Commit(internal.CommitMessage, &git.CommitOptions{})
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			logrus.Info("changes commited")
-
-			if err := commitRepo.Push(&git.PushOptions{}); err != nil {
-				logrus.Fatal(err)
-			}
-			logrus.Info("changes pushed")
-		}
-
-		return nil
-	},
-	BashComplete: func(c *cli.Context) {
-		catl, err := catalogue.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for name := range catl {
-			fmt.Println(name)
-		}
-	},
-}

cli/cli.go

@@ -2,8 +2,10 @@
 package cli
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path"
 
 	"coopcloud.tech/abra/cli/app"
@@ -13,47 +15,140 @@ import (
 	"coopcloud.tech/abra/cli/record"
 	"coopcloud.tech/abra/cli/server"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/web"
 	logrusStack "github.com/Gurpartap/logrus-stack"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
-// Verbose stores the variable from VerboseFlag.
-var Verbose bool
+// AutoCompleteCommand helps people set up auto-complete in their shells
+var AutoCompleteCommand = &cli.Command{
+	Name:    "autocomplete",
+	Usage:   "Configure shell autocompletion (recommended)",
+	Aliases: []string{"ac"},
+	Description: `
+This command helps set up autocompletion in your shell by downloading the
+relevant autocompletion files and laying out what additional information must
+be loaded.
 
-// VerboseFlag turns on/off verbose logging down to the INFO level.
-var VerboseFlag = &cli.BoolFlag{
-	Name:        "verbose",
-	Aliases:     []string{"V"},
-	Value:       false,
-	Destination: &Verbose,
-	Usage:       "Show INFO messages",
+Example:
+
+    abra autocomplete bash
+
+Supported shells are as follows:
+
+		fizsh
+		zsh
+    bash
+
+`,
+	ArgsUsage: "<shell>",
+	Action: func(c *cli.Context) error {
+		shellType := c.Args().First()
+
+		if shellType == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
+		}
+
+		supportedShells := map[string]bool{
+			"bash":  true,
+			"zsh":   true,
+			"fizsh": true,
+		}
+
+		if _, ok := supportedShells[shellType]; !ok {
+			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
+		}
+
+		if shellType == "fizsh" {
+			shellType = "zsh" // handled the same on the autocompletion side
+		}
+
+		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
+		if err := os.Mkdir(autocompletionDir, 0764); err != nil {
+			if !os.IsExist(err) {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("%s already created", autocompletionDir)
+		}
+
+		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
+		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
+			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
+			logrus.Infof("fetching %s", url)
+			if err := web.GetFile(autocompletionFile, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		switch shellType {
+		case "bash":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install autocompletion
+sudo mkdir /etc/bash_completion.d/
+sudo cp %s /etc/bash_completion.d/abra
+echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
+# And finally run "abra app ps <hit tab key>" to test things are working, you should see app names listed!
+`, autocompletionFile))
+		case "zsh":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install autocompletion
+sudo mkdir /etc/zsh/completion.d/
+sudo cp %s /etc/zsh/completion.d/abra
+echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
+# And finally run "abra app ps <hit tab key>" to test things are working, you should see app names listed!
+`, autocompletionFile))
+		}
+
+		return nil
+	},
 }
 
-// Debug stores the variable from DebugFlag.
-var Debug bool
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = &cli.Command{
+	Name:    "upgrade",
+	Usage:   "Upgrade Abra itself",
+	Aliases: []string{"u"},
+	Description: `
+This command allows you to upgrade Abra in-place with the latest stable or
+release candidate.
 
-// DebugFlag turns on/off verbose logging down to the DEBUG level.
-var DebugFlag = &cli.BoolFlag{
-	Name:        "debug",
-	Aliases:     []string{"d"},
-	Value:       false,
-	Destination: &Debug,
-	Usage:       "Show DEBUG messages",
+If you would like to install the latest release candidate, please pass the
+"--rc" option. Please bear in mind that the latest release candidate may have
+some catastrophic bugs contained in it. In any case, thank you very much for
+the testing efforts!
+`,
+	Flags: []cli.Flag{internal.RCFlag},
+	Action: func(c *cli.Context) error {
+		mainURL := "https://install.abra.coopcloud.tech"
+		cmd := exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash", mainURL))
+
+		if internal.RC {
+			releaseCandidateURL := "https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer"
+			cmd = exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash -s -- --rc", releaseCandidateURL))
+		}
+
+		logrus.Debugf("attempting to run %s", cmd)
+
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
 }
 
 func newAbraApp(version, commit string) *cli.App {
 	app := &cli.App{
 		Name: "abra",
 		Usage: `The Co-op Cloud command-line utility belt 🎩🐇
-
     ____                           ____ _                 _
    / ___|___         ___  _ __    / ___| | ___  _   _  __| |
   | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
   | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
    \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
                          |_|
-		`,
+`,
 		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
 		Commands: []*cli.Command{
 			app.AppCommand,
@@ -65,13 +160,16 @@ func newAbraApp(version, commit string) *cli.App {
 			AutoCompleteCommand,
 		},
 		Flags: []cli.Flag{
-			VerboseFlag,
-			DebugFlag,
+			internal.DebugFlag,
 			internal.NoInputFlag,
 		},
 		Authors: []*cli.Author{
+			// If you're looking at this and you hack on Abra and you're not listed
+			// here, please do add yourself! This is a community project, let's show
+			// some love
 			{Name: "3wordchant"},
 			{Name: "decentral1se"},
+			{Name: "kawaiipunk"},
 			{Name: "knoflook"},
 			{Name: "roxxers"},
 		},
@@ -80,7 +178,7 @@ func newAbraApp(version, commit string) *cli.App {
 	app.EnableBashCompletion = true
 
 	app.Before = func(c *cli.Context) error {
-		if Debug {
+		if internal.Debug {
 			logrus.SetLevel(logrus.DebugLevel)
 			logrus.SetFormatter(&logrus.TextFormatter{})
 			logrus.SetOutput(os.Stderr)
@@ -89,23 +187,21 @@ func newAbraApp(version, commit string) *cli.App {
 
 		paths := []string{
 			config.ABRA_DIR,
-			path.Join(config.ABRA_DIR, "servers"),
-			path.Join(config.ABRA_DIR, "apps"),
-			path.Join(config.ABRA_DIR, "vendor"),
+			path.Join(config.SERVERS_DIR),
+			path.Join(config.RECIPES_DIR),
+			path.Join(config.VENDOR_DIR),
 		}
 
 		for _, path := range paths {
-			if err := os.Mkdir(path, 0755); err != nil {
+			if err := os.Mkdir(path, 0764); err != nil {
 				if !os.IsExist(err) {
 					logrus.Fatal(err)
 				}
-				logrus.Debugf("'%s' already created, moving on...", path)
 				continue
 			}
-			logrus.Debugf("'%s' is missing, creating...", path)
 		}
 
-		logrus.Debugf("abra version '%s', commit '%s'", version, commit)
+		logrus.Debugf("abra version %s, commit %s", version, commit)
 
 		return nil
 	}

cli/internal/common.go

@@ -1,261 +0,0 @@
-package internal
-
-import (
-	"github.com/urfave/cli/v2"
-)
-
-// Secrets stores the variable from SecretsFlag
-var Secrets bool
-
-// SecretsFlag turns on/off automatically generating secrets
-var SecretsFlag = &cli.BoolFlag{
-	Name:        "secrets",
-	Aliases:     []string{"S"},
-	Value:       false,
-	Usage:       "Automatically generate secrets",
-	Destination: &Secrets,
-}
-
-// Pass stores the variable from PassFlag
-var Pass bool
-
-// PassFlag turns on/off storing generated secrets in pass
-var PassFlag = &cli.BoolFlag{
-	Name:        "pass",
-	Aliases:     []string{"P"},
-	Value:       false,
-	Usage:       "Store the generated secrets in a local pass store",
-	Destination: &Pass,
-}
-
-// Context is temp
-var Context string
-
-// ContextFlag is temp
-var ContextFlag = &cli.StringFlag{
-	Name:        "context",
-	Value:       "",
-	Aliases:     []string{"c"},
-	Destination: &Context,
-}
-
-// Force force functionality without asking.
-var Force bool
-
-// ForceFlag turns on/off force functionality.
-var ForceFlag = &cli.BoolFlag{
-	Name:        "force",
-	Value:       false,
-	Aliases:     []string{"f"},
-	Destination: &Force,
-}
-
-// Chaos engages chaos mode.
-var Chaos bool
-
-// ChaosFlag turns on/off chaos functionality.
-var ChaosFlag = &cli.BoolFlag{
-	Name:        "chaos",
-	Value:       false,
-	Aliases:     []string{"ch"},
-	Usage:       "Deploy uncommitted recipes changes. Use with care!",
-	Destination: &Chaos,
-}
-
-// DNSProvider specifies a DNS provider.
-var DNSProvider string
-
-// DNSProviderFlag selects a DNS provider.
-var DNSProviderFlag = &cli.StringFlag{
-	Name:        "provider",
-	Value:       "",
-	Aliases:     []string{"p"},
-	Usage:       "DNS provider",
-	Destination: &DNSProvider,
-}
-
-var NoInput bool
-var NoInputFlag = &cli.BoolFlag{
-	Name:        "no-input",
-	Value:       false,
-	Aliases:     []string{"n"},
-	Usage:       "Toggle non-interactive mode",
-	Destination: &NoInput,
-}
-
-var DNSType string
-
-var DNSTypeFlag = &cli.StringFlag{
-	Name:        "type",
-	Value:       "",
-	Aliases:     []string{"t"},
-	Usage:       "Domain name record type (e.g. A)",
-	Destination: &DNSType,
-}
-
-var DNSName string
-
-var DNSNameFlag = &cli.StringFlag{
-	Name:        "name",
-	Value:       "",
-	Aliases:     []string{"n"},
-	Usage:       "Domain name record name (e.g. mysubdomain)",
-	Destination: &DNSName,
-}
-
-var DNSValue string
-
-var DNSValueFlag = &cli.StringFlag{
-	Name:        "value",
-	Value:       "",
-	Aliases:     []string{"v"},
-	Usage:       "Domain name record value (e.g. 192.168.1.1)",
-	Destination: &DNSValue,
-}
-
-var DNSTTL int
-
-var DNSTTLFlag = &cli.IntFlag{
-	Name:        "ttl",
-	Value:       86400,
-	Aliases:     []string{"T"},
-	Usage:       "Domain name TTL value)",
-	Destination: &DNSTTL,
-}
-
-var DNSPriority int
-
-var DNSPriorityFlag = &cli.IntFlag{
-	Name:        "priority",
-	Value:       10,
-	Aliases:     []string{"P"},
-	Usage:       "Domain name priority value",
-	Destination: &DNSPriority,
-}
-
-var ServerProvider string
-
-var ServerProviderFlag = &cli.StringFlag{
-	Name:        "provider",
-	Aliases:     []string{"p"},
-	Usage:       "3rd party server provider",
-	Destination: &ServerProvider,
-}
-
-var CapsulInstanceURL string
-
-var CapsulInstanceURLFlag = &cli.StringFlag{
-	Name:        "capsul-url",
-	Value:       "yolo.servers.coop",
-	Aliases:     []string{"cu"},
-	Usage:       "capsul instance URL",
-	Destination: &CapsulInstanceURL,
-}
-
-var CapsulName string
-
-var CapsulNameFlag = &cli.StringFlag{
-	Name:        "capsul-name",
-	Value:       "",
-	Aliases:     []string{"cn"},
-	Usage:       "capsul name",
-	Destination: &CapsulName,
-}
-
-var CapsulType string
-
-var CapsulTypeFlag = &cli.StringFlag{
-	Name:        "capsul-type",
-	Value:       "f1-xs",
-	Aliases:     []string{"ct"},
-	Usage:       "capsul type",
-	Destination: &CapsulType,
-}
-
-var CapsulImage string
-
-var CapsulImageFlag = &cli.StringFlag{
-	Name:        "capsul-image",
-	Value:       "debian10",
-	Aliases:     []string{"ci"},
-	Usage:       "capsul image",
-	Destination: &CapsulImage,
-}
-
-var CapsulSSHKeys cli.StringSlice
-
-var CapsulSSHKeysFlag = &cli.StringSliceFlag{
-	Name:        "capsul-ssh-keys",
-	Aliases:     []string{"cs"},
-	Usage:       "capsul SSH key",
-	Destination: &CapsulSSHKeys,
-}
-
-var CapsulAPIToken string
-
-var CapsulAPITokenFlag = &cli.StringFlag{
-	Name:        "capsul-token",
-	Aliases:     []string{"ca"},
-	Usage:       "capsul API token",
-	EnvVars:     []string{"CAPSUL_TOKEN"},
-	Destination: &CapsulAPIToken,
-}
-
-var HetznerCloudName string
-
-var HetznerCloudNameFlag = &cli.StringFlag{
-	Name:        "hetzner-name",
-	Value:       "",
-	Aliases:     []string{"hn"},
-	Usage:       "hetzner cloud name",
-	Destination: &HetznerCloudName,
-}
-
-var HetznerCloudType string
-
-var HetznerCloudTypeFlag = &cli.StringFlag{
-	Name:        "hetzner-type",
-	Aliases:     []string{"ht"},
-	Usage:       "hetzner cloud type",
-	Destination: &HetznerCloudType,
-	Value:       "cx11",
-}
-
-var HetznerCloudImage string
-
-var HetznerCloudImageFlag = &cli.StringFlag{
-	Name:        "hetzner-image",
-	Aliases:     []string{"hi"},
-	Usage:       "hetzner cloud image",
-	Value:       "debian-10",
-	Destination: &HetznerCloudImage,
-}
-
-var HetznerCloudSSHKeys cli.StringSlice
-
-var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
-	Name:        "hetzner-ssh-keys",
-	Aliases:     []string{"hs"},
-	Usage:       "hetzner cloud SSH keys (e.g. me@foo.com)",
-	Destination: &HetznerCloudSSHKeys,
-}
-
-var HetznerCloudLocation string
-
-var HetznerCloudLocationFlag = &cli.StringFlag{
-	Name:        "hetzner-location",
-	Aliases:     []string{"hl"},
-	Usage:       "hetzner cloud server location",
-	Value:       "hel1",
-	Destination: &HetznerCloudLocation,
-}
-
-var HetznerCloudAPIToken string
-
-var HetznerCloudAPITokenFlag = &cli.StringFlag{
-	Name:        "hetzner-token",
-	Aliases:     []string{"ha"},
-	Usage:       "hetzner cloud API token",
-	EnvVars:     []string{"HCLOUD_TOKEN"},
-	Destination: &HetznerCloudAPIToken,
-}

cli/internal/deploy.go

@@ -2,12 +2,17 @@ package internal
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
 	"strings"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
@@ -18,44 +23,64 @@ import (
 // DeployAction is the main command-line action for this package
 func DeployAction(c *cli.Context) error {
 	app := ValidateApp(c)
-	stackName := app.StackName()
+
+	if !Chaos {
+		if err := recipe.EnsureUpToDate(app.Type); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	r, err := recipe.Get(app.Type)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := lint.LintForErrors(r); err != nil {
+		logrus.Fatal(err)
+	}
 
 	cl, err := client.New(app.Server)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("checking whether '%s' is already deployed", stackName)
+	logrus.Debugf("checking whether %s is already deployed", app.StackName())
 
-	isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+	isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, app.StackName())
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
 	if isDeployed {
-		if Force {
-			logrus.Warnf("'%s' already deployed but continuing (--force)", stackName)
-		} else if Chaos {
-			logrus.Warnf("'%s' already deployed but continuing (--chaos)", stackName)
+		if Force || Chaos {
+			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
 		} else {
-			logrus.Fatalf("'%s' is already deployed", stackName)
+			logrus.Fatalf("%s is already deployed", app.Name)
 		}
 	}
 
 	version := deployedVersion
-	if version == "" && !Chaos {
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+	if version == "unknown" && !Chaos {
+		catl, err := recipe.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		if len(versions) > 0 {
 			version = versions[len(versions)-1]
-			logrus.Debugf("choosing '%s' as version to deploy", version)
+			logrus.Debugf("choosing %s as version to deploy", version)
 			if err := recipe.EnsureVersion(app.Type, version); err != nil {
 				logrus.Fatal(err)
 			}
 		} else {
-			version = "latest commit"
+			head, err := git.GetRecipeHead(app.Type)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			version = formatter.SmallSHA(head.String())
 			logrus.Warn("no versions detected, using latest commit")
 			if err := recipe.EnsureLatest(app.Type); err != nil {
 				logrus.Fatal(err)
@@ -63,8 +88,14 @@ func DeployAction(c *cli.Context) error {
 		}
 	}
 
-	if version == "" && !Chaos {
-		logrus.Debugf("choosing '%s' as version to deploy", version)
+	if version == "unknown" && !Chaos {
+		logrus.Debugf("choosing %s as version to deploy", version)
+		if err := recipe.EnsureVersion(app.Type, version); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if version != "unknown" && !Chaos {
 		if err := recipe.EnsureVersion(app.Type, version); err != nil {
 			logrus.Fatal(err)
 		}
@@ -79,7 +110,7 @@ func DeployAction(c *cli.Context) error {
 		}
 	}
 
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Type, "abra.sh")
 	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 	if err != nil {
 		logrus.Fatal(err)
@@ -94,7 +125,7 @@ func DeployAction(c *cli.Context) error {
 	}
 	deployOpts := stack.Deploy{
 		Composefiles: composeFiles,
-		Namespace:    stackName,
+		Namespace:    app.StackName(),
 		Prune:        false,
 		ResolveImage: stack.ResolveImageAlways,
 	}
@@ -107,7 +138,21 @@ func DeployAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+	if !NoDomainChecks {
+		domainName := app.Env["DOMAIN"]
+		ipv4, err := dns.EnsureIPv4(domainName)
+		if err != nil || ipv4 == "" {
+			logrus.Fatalf("could not find an IP address assigned to %s?", domainName)
+		}
+
+		if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		logrus.Warn("skipping domain checks as requested")
+	}
+
+	if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, DontWaitConverge); err != nil {
 		logrus.Fatal(err)
 	}
 
@@ -116,8 +161,8 @@ func DeployAction(c *cli.Context) error {
 
 // DeployOverview shows a deployment overview
 func DeployOverview(app config.App, version, message string) error {
-	tableCol := []string{"server", "compose", "domain", "stack", "version"}
-	table := abraFormatter.CreateTable(tableCol)
+	tableCol := []string{"server", "compose", "domain", "app name", "version"}
+	table := formatter.CreateTable(tableCol)
 
 	deployConfig := "compose.yml"
 	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
@@ -129,7 +174,7 @@ func DeployOverview(app config.App, version, message string) error {
 		server = "local"
 	}
 
-	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), version})
+	table.Append([]string{server, deployConfig, app.Domain, app.Name, version})
 	table.Render()
 
 	if NoInput {
@@ -153,9 +198,9 @@ func DeployOverview(app config.App, version, message string) error {
 }
 
 // NewVersionOverview shows an upgrade or downgrade overview
-func NewVersionOverview(app config.App, currentVersion, newVersion string) error {
-	tableCol := []string{"server", "compose", "domain", "stack", "current version", "to be deployed"}
-	table := abraFormatter.CreateTable(tableCol)
+func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
+	tableCol := []string{"server", "compose", "domain", "app name", "current version", "to be deployed"}
+	table := formatter.CreateTable(tableCol)
 
 	deployConfig := "compose.yml"
 	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
@@ -167,9 +212,24 @@ func NewVersionOverview(app config.App, currentVersion, newVersion string) error
 		server = "local"
 	}
 
-	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), currentVersion, newVersion})
+	table.Append([]string{server, deployConfig, app.Domain, app.Name, currentVersion, newVersion})
 	table.Render()
 
+	if releaseNotes == "" {
+		var err error
+		releaseNotes, err = GetReleaseNotes(app.Type, newVersion)
+		if err != nil {
+			return err
+		}
+	}
+
+	if releaseNotes != "" && newVersion != "" {
+		fmt.Println()
+		fmt.Println(fmt.Sprintf("%s release notes:\n\n%s", newVersion, releaseNotes))
+	} else {
+		logrus.Warnf("no release notes available for %s", newVersion)
+	}
+
 	if NoInput {
 		return nil
 	}
@@ -189,3 +249,22 @@ func NewVersionOverview(app config.App, currentVersion, newVersion string) error
 
 	return nil
 }
+
+// GetReleaseNotes prints release notes for a recipe version
+func GetReleaseNotes(recipeName, version string) (string, error) {
+	if version == "" {
+		return "", nil
+	}
+
+	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
+
+	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
+		releaseNotes, err := ioutil.ReadFile(fpath)
+		if err != nil {
+			return "", err
+		}
+		return string(releaseNotes), nil
+	}
+
+	return "", nil
+}

cli/internal/errors_test.go

@@ -1,38 +0,0 @@
-package internal
-
-import (
-	"github.com/urfave/cli/v2"
-)
-
-// Testing functions that call os.Exit
-// https://stackoverflow.com/questions/26225513/how-to-test-os-exit-scenarios-in-go
-// https://talks.golang.org/2014/testing.slide#23
-
-var testapp = &cli.App{
-	Name:  "abra",
-	Usage: `The Co-op Cloud command-line utility belt 🎩🐇`,
-}
-
-// not testing output as that changes. just if it exits with code 1
-// does not work because of some weird errors on cli's part. Its a hard lib to test effectively.
-// func TestShowSubcommandHelpAndError(t *testing.T) {
-// 	if os.Getenv("HelpAndError") == "1" {
-// 		ShowSubcommandHelpAndError(cli.NewContext(testapp, nil, nil), errors.New("Test error"))
-// 		return
-// 	}
-// 	cmd := exec.Command(os.Args[0], "-test.run=TestShowSubcommandHelpAndError")
-// 	cmd.Env = append(os.Environ(), "HelpAndError=1")
-// 	var out bytes.Buffer
-// 	cmd.Stderr = &out
-// 	err := cmd.Run()
-// 	println(out.String())
-// 	if !strings.Contains(out.String(), "Test error") {
-
-// 		t.Fatalf("expected command to show the error causing the exit, did not get correct stdout output")
-// 	}
-
-// 	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
-// 		return
-// 	}
-// 	t.Fatalf("process ran with err %v, want exit status 1", err)
-// }

cli/internal/flags.go

@@ -0,0 +1,488 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Secrets stores the variable from SecretsFlag
+var Secrets bool
+
+// SecretsFlag turns on/off automatically generating secrets
+var SecretsFlag = &cli.BoolFlag{
+	Name:        "secrets",
+	Aliases:     []string{"ss"},
+	Value:       false,
+	Usage:       "Automatically generate secrets",
+	Destination: &Secrets,
+}
+
+// Pass stores the variable from PassFlag
+var Pass bool
+
+// PassFlag turns on/off storing generated secrets in pass
+var PassFlag = &cli.BoolFlag{
+	Name:        "pass",
+	Aliases:     []string{"p"},
+	Value:       false,
+	Usage:       "Store the generated secrets in a local pass store",
+	Destination: &Pass,
+}
+
+// Context is temp
+var Context string
+
+// ContextFlag is temp
+var ContextFlag = &cli.StringFlag{
+	Name:        "context",
+	Value:       "",
+	Aliases:     []string{"c"},
+	Destination: &Context,
+}
+
+// Force force functionality without asking.
+var Force bool
+
+// ForceFlag turns on/off force functionality.
+var ForceFlag = &cli.BoolFlag{
+	Name:        "force",
+	Value:       false,
+	Aliases:     []string{"f"},
+	Usage:       "Perform action without further prompt. Use with care!",
+	Destination: &Force,
+}
+
+// Chaos engages chaos mode.
+var Chaos bool
+
+// ChaosFlag turns on/off chaos functionality.
+var ChaosFlag = &cli.BoolFlag{
+	Name:        "chaos",
+	Value:       false,
+	Aliases:     []string{"ch"},
+	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Destination: &Chaos,
+}
+
+// DNSProvider specifies a DNS provider.
+var DNSProvider string
+
+// DNSProviderFlag selects a DNS provider.
+var DNSProviderFlag = &cli.StringFlag{
+	Name:        "provider",
+	Value:       "",
+	Aliases:     []string{"p"},
+	Usage:       "DNS provider",
+	Destination: &DNSProvider,
+}
+
+var NoInput bool
+var NoInputFlag = &cli.BoolFlag{
+	Name:        "no-input",
+	Value:       false,
+	Aliases:     []string{"n"},
+	Usage:       "Toggle non-interactive mode",
+	Destination: &NoInput,
+}
+
+var DNSType string
+
+var DNSTypeFlag = &cli.StringFlag{
+	Name:        "type",
+	Value:       "",
+	Aliases:     []string{"t"},
+	Usage:       "Domain name record type (e.g. A)",
+	Destination: &DNSType,
+}
+
+var DNSName string
+
+var DNSNameFlag = &cli.StringFlag{
+	Name:        "name",
+	Value:       "",
+	Aliases:     []string{"n"},
+	Usage:       "Domain name record name (e.g. mysubdomain)",
+	Destination: &DNSName,
+}
+
+var DNSValue string
+
+var DNSValueFlag = &cli.StringFlag{
+	Name:        "value",
+	Value:       "",
+	Aliases:     []string{"v"},
+	Usage:       "Domain name record value (e.g. 192.168.1.1)",
+	Destination: &DNSValue,
+}
+
+var DNSTTL string
+var DNSTTLFlag = &cli.StringFlag{
+	Name:        "ttl",
+	Value:       "600s",
+	Aliases:     []string{"T"},
+	Usage:       "Domain name TTL value (seconds)",
+	Destination: &DNSTTL,
+}
+
+var DNSPriority int
+
+var DNSPriorityFlag = &cli.IntFlag{
+	Name:        "priority",
+	Value:       10,
+	Aliases:     []string{"P"},
+	Usage:       "Domain name priority value",
+	Destination: &DNSPriority,
+}
+
+var ServerProvider string
+
+var ServerProviderFlag = &cli.StringFlag{
+	Name:        "provider",
+	Aliases:     []string{"p"},
+	Usage:       "3rd party server provider",
+	Destination: &ServerProvider,
+}
+
+var CapsulInstanceURL string
+
+var CapsulInstanceURLFlag = &cli.StringFlag{
+	Name:        "capsul-url",
+	Value:       "yolo.servers.coop",
+	Aliases:     []string{"cu"},
+	Usage:       "capsul instance URL",
+	Destination: &CapsulInstanceURL,
+}
+
+var CapsulName string
+
+var CapsulNameFlag = &cli.StringFlag{
+	Name:        "capsul-name",
+	Value:       "",
+	Aliases:     []string{"cn"},
+	Usage:       "capsul name",
+	Destination: &CapsulName,
+}
+
+var CapsulType string
+
+var CapsulTypeFlag = &cli.StringFlag{
+	Name:        "capsul-type",
+	Value:       "f1-xs",
+	Aliases:     []string{"ct"},
+	Usage:       "capsul type",
+	Destination: &CapsulType,
+}
+
+var CapsulImage string
+
+var CapsulImageFlag = &cli.StringFlag{
+	Name:        "capsul-image",
+	Value:       "debian10",
+	Aliases:     []string{"ci"},
+	Usage:       "capsul image",
+	Destination: &CapsulImage,
+}
+
+var CapsulSSHKeys cli.StringSlice
+
+var CapsulSSHKeysFlag = &cli.StringSliceFlag{
+	Name:        "capsul-ssh-keys",
+	Aliases:     []string{"cs"},
+	Usage:       "capsul SSH key",
+	Destination: &CapsulSSHKeys,
+}
+
+var CapsulAPIToken string
+
+var CapsulAPITokenFlag = &cli.StringFlag{
+	Name:        "capsul-token",
+	Aliases:     []string{"ca"},
+	Usage:       "capsul API token",
+	EnvVars:     []string{"CAPSUL_TOKEN"},
+	Destination: &CapsulAPIToken,
+}
+
+var HetznerCloudName string
+
+var HetznerCloudNameFlag = &cli.StringFlag{
+	Name:        "hetzner-name",
+	Value:       "",
+	Aliases:     []string{"hn"},
+	Usage:       "hetzner cloud name",
+	Destination: &HetznerCloudName,
+}
+
+var HetznerCloudType string
+
+var HetznerCloudTypeFlag = &cli.StringFlag{
+	Name:        "hetzner-type",
+	Aliases:     []string{"ht"},
+	Usage:       "hetzner cloud type",
+	Destination: &HetznerCloudType,
+	Value:       "cx11",
+}
+
+var HetznerCloudImage string
+
+var HetznerCloudImageFlag = &cli.StringFlag{
+	Name:        "hetzner-image",
+	Aliases:     []string{"hi"},
+	Usage:       "hetzner cloud image",
+	Value:       "debian-10",
+	Destination: &HetznerCloudImage,
+}
+
+var HetznerCloudSSHKeys cli.StringSlice
+
+var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
+	Name:        "hetzner-ssh-keys",
+	Aliases:     []string{"hs"},
+	Usage:       "hetzner cloud SSH keys (e.g. me@foo.com)",
+	Destination: &HetznerCloudSSHKeys,
+}
+
+var HetznerCloudLocation string
+
+var HetznerCloudLocationFlag = &cli.StringFlag{
+	Name:        "hetzner-location",
+	Aliases:     []string{"hl"},
+	Usage:       "hetzner cloud server location",
+	Value:       "hel1",
+	Destination: &HetznerCloudLocation,
+}
+
+var HetznerCloudAPIToken string
+
+var HetznerCloudAPITokenFlag = &cli.StringFlag{
+	Name:        "hetzner-token",
+	Aliases:     []string{"ha"},
+	Usage:       "hetzner cloud API token",
+	EnvVars:     []string{"HCLOUD_TOKEN"},
+	Destination: &HetznerCloudAPIToken,
+}
+
+// Debug stores the variable from DebugFlag.
+var Debug bool
+
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
+var DebugFlag = &cli.BoolFlag{
+	Name:        "debug",
+	Aliases:     []string{"d"},
+	Value:       false,
+	Destination: &Debug,
+	Usage:       "Show DEBUG messages",
+}
+
+// RC signifies the latest release candidate
+var RC bool
+
+// RCFlag chooses the latest release candidate for install
+var RCFlag = &cli.BoolFlag{
+	Name:        "rc",
+	Value:       false,
+	Destination: &RC,
+	Usage:       "Insatll the latest release candidate",
+}
+
+var Major bool
+var MajorFlag = &cli.BoolFlag{
+	Name:        "major",
+	Usage:       "Increase the major part of the version",
+	Value:       false,
+	Aliases:     []string{"ma", "x"},
+	Destination: &Major,
+}
+
+var Minor bool
+var MinorFlag = &cli.BoolFlag{
+	Name:        "minor",
+	Usage:       "Increase the minor part of the version",
+	Value:       false,
+	Aliases:     []string{"mi", "y"},
+	Destination: &Minor,
+}
+
+var Patch bool
+var PatchFlag = &cli.BoolFlag{
+	Name:        "patch",
+	Usage:       "Increase the patch part of the version",
+	Value:       false,
+	Aliases:     []string{"pa", "z"},
+	Destination: &Patch,
+}
+
+var Dry bool
+var DryFlag = &cli.BoolFlag{
+	Name:        "dry-run",
+	Usage:       "Only reports changes that would be made",
+	Value:       false,
+	Aliases:     []string{"d"},
+	Destination: &Dry,
+}
+
+var Publish bool
+var PublishFlag = &cli.BoolFlag{
+	Name:        "publish",
+	Usage:       "Publish changes to git.coopcloud.tech",
+	Value:       false,
+	Aliases:     []string{"p"},
+	Destination: &Publish,
+}
+
+var Domain string
+var DomainFlag = &cli.StringFlag{
+	Name:        "domain",
+	Aliases:     []string{"d"},
+	Value:       "",
+	Usage:       "Choose a domain name",
+	Destination: &Domain,
+}
+
+var NewAppServer string
+var NewAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &NewAppServer,
+}
+
+var NewAppName string
+var NewAppNameFlag = &cli.StringFlag{
+	Name:        "app-name",
+	Aliases:     []string{"a"},
+	Value:       "",
+	Usage:       "Choose an app name",
+	Destination: &NewAppName,
+}
+
+var NoDomainChecks bool
+var NoDomainChecksFlag = &cli.BoolFlag{
+	Name:        "no-domain-checks",
+	Aliases:     []string{"nd"},
+	Value:       false,
+	Usage:       "Disable app domain sanity checks",
+	Destination: &NoDomainChecks,
+}
+
+var StdErrOnly bool
+var StdErrOnlyFlag = &cli.BoolFlag{
+	Name:        "stderr",
+	Aliases:     []string{"s"},
+	Value:       false,
+	Usage:       "Only tail stderr",
+	Destination: &StdErrOnly,
+}
+
+var AutoDNSRecord bool
+var AutoDNSRecordFlag = &cli.BoolFlag{
+	Name:        "auto",
+	Aliases:     []string{"a"},
+	Value:       false,
+	Usage:       "Automatically configure DNS records",
+	Destination: &AutoDNSRecord,
+}
+
+var DontWaitConverge bool
+var DontWaitConvergeFlag = &cli.BoolFlag{
+	Name:        "no-converge-checks",
+	Aliases:     []string{"nc"},
+	Value:       false,
+	Usage:       "Don't wait for converge logic checks",
+	Destination: &DontWaitConverge,
+}
+
+var Watch bool
+var WatchFlag = &cli.BoolFlag{
+	Name:        "watch",
+	Aliases:     []string{"w"},
+	Value:       false,
+	Usage:       "Watch status by polling repeatedly",
+	Destination: &Watch,
+}
+
+var OnlyErrors bool
+var OnlyErrorFlag = &cli.BoolFlag{
+	Name:        "errors",
+	Aliases:     []string{"e"},
+	Value:       false,
+	Usage:       "Only show errors",
+	Destination: &OnlyErrors,
+}
+
+var SkipUpdates bool
+var SkipUpdatesFlag = &cli.BoolFlag{
+	Name:        "skip-updates",
+	Aliases:     []string{"s"},
+	Value:       false,
+	Usage:       "Skip updating recipe repositories",
+	Destination: &SkipUpdates,
+}
+
+var RegistryUsername string
+var RegistryUsernameFlag = &cli.StringFlag{
+	Name:        "username",
+	Aliases:     []string{"user"},
+	Value:       "",
+	Usage:       "Registry username",
+	EnvVars:     []string{"REGISTRY_USERNAME"},
+	Destination: &RegistryUsername,
+}
+
+var RegistryPassword string
+var RegistryPasswordFlag = &cli.StringFlag{
+	Name:        "password",
+	Aliases:     []string{"pass"},
+	Value:       "",
+	Usage:       "Registry password",
+	EnvVars:     []string{"REGISTRY_PASSWORD"},
+	Destination: &RegistryUsername,
+}
+
+// SSHFailMsg is a hopefully helpful SSH failure message
+var SSHFailMsg = `
+Woops, Abra is unable to connect to connect to %s.
+
+Here are a few tips for debugging your local SSH config. Abra uses plain 'ol
+SSH to make connections to servers, so if your SSH config is working, Abra is
+working.
+
+In the first place, Abra will always try to read your Docker context connection
+string for SSH connection details. You can view your server context configs
+with the following command. Are they correct?
+
+    abra server ls
+
+Is your ssh-agent running? You can start it by running the following command:
+
+    eval "$(ssh-agent)"
+
+If your SSH private key loaded? You can check by running the following command:
+
+    ssh-add -L
+
+If, you can add it with:
+
+    ssh-add ~/.ssh/<private-key-part>
+
+If you are using a non-default public/private key, you can configure this in
+your ~/.ssh/config file which Abra will read in order to figure out connection
+details:
+
+Host foo.coopcloud.tech
+  Hostname foo.coopcloud.tech
+  User bar
+  Port 12345
+  IdentityFile ~/.ssh/bar@foo.coopcloud.tech
+
+If you're only using password authentication, you can use the following config:
+
+Host foo.coopcloud.tech
+  Hostname foo.coopcloud.tech
+  User bar
+  Port 12345
+  PreferredAuthentications=password
+  PubkeyAuthentication=no
+
+Good luck!
+
+`

cli/internal/list.go

@@ -0,0 +1,10 @@
+package internal
+
+// ReverseStringList reverses a list of a strings. Roll on Go generics.
+func ReverseStringList(strings []string) []string {
+	for i, j := 0, len(strings)-1; i < j; i, j = i+1, j-1 {
+		strings[i], strings[j] = strings[j], strings[i]
+	}
+
+	return strings
+}

cli/internal/new.go

@@ -4,44 +4,20 @@ import (
 	"fmt"
 	"path"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/secret"
+	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
+// AppSecrets represents all app secrest
 type AppSecrets map[string]string
 
-var Domain string
-var DomainFlag = &cli.StringFlag{
-	Name:        "domain",
-	Aliases:     []string{"d"},
-	Value:       "",
-	Usage:       "Choose a domain name",
-	Destination: &Domain,
-}
-
-var NewAppServer string
-var NewAppServerFlag = &cli.StringFlag{
-	Name:        "server",
-	Aliases:     []string{"s"},
-	Value:       "",
-	Usage:       "Show apps of a specific server",
-	Destination: &NewAppServer,
-}
-
-var NewAppName string
-var NewAppNameFlag = &cli.StringFlag{
-	Name:        "app-name",
-	Aliases:     []string{"a"},
-	Value:       "",
-	Usage:       "Choose an app name",
-	Destination: &NewAppName,
-}
-
 // RecipeName is used for configuring recipe name programmatically
 var RecipeName string
 
@@ -118,7 +94,7 @@ func ensureAppNameFlag() error {
 	if NewAppName == "" && !NoInput {
 		prompt := &survey.Input{
 			Message: "Specify app name:",
-			Default: config.SanitiseAppName(Domain),
+			Default: Domain,
 		}
 		if err := survey.AskOne(prompt, &NewAppName); err != nil {
 			return err
@@ -136,7 +112,7 @@ func ensureAppNameFlag() error {
 func NewAction(c *cli.Context) error {
 	recipe := ValidateRecipeWithPrompt(c)
 
-	if err := config.EnsureAbraDirExists(); err != nil {
+	if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
 		logrus.Fatal(err)
 	}
 
@@ -154,22 +130,26 @@ func NewAction(c *cli.Context) error {
 
 	sanitisedAppName := config.SanitiseAppName(NewAppName)
 	if len(sanitisedAppName) > 45 {
-		logrus.Fatalf("'%s' cannot be longer than 45 characters", sanitisedAppName)
+		logrus.Fatalf("%s cannot be longer than 45 characters", sanitisedAppName)
 	}
-	logrus.Debugf("'%s' sanitised as '%s' for new app", NewAppName, sanitisedAppName)
+	logrus.Debugf("%s sanitised as %s for new app", NewAppName, sanitisedAppName)
 
-	if err := config.TemplateAppEnvSample(recipe.Name, NewAppName, NewAppServer, Domain, recipe.Name); err != nil {
+	if err := config.TemplateAppEnvSample(recipe.Name, NewAppName, NewAppServer, Domain); err != nil {
 		logrus.Fatal(err)
 	}
 
 	if Secrets {
+		if err := ssh.EnsureHostKey(NewAppServer); err != nil {
+			logrus.Fatal(err)
+		}
+
 		secrets, err := createSecrets(sanitisedAppName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		secretCols := []string{"Name", "Value"}
-		secretTable := abraFormatter.CreateTable(secretCols)
+		secretTable := formatter.CreateTable(secretCols)
 		for secret := range secrets {
 			secretTable.Append([]string{secret, secrets[secret]})
 		}
@@ -183,9 +163,9 @@ func NewAction(c *cli.Context) error {
 		NewAppServer = "local"
 	}
 
-	tableCol := []string{"Name", "Domain", "Type", "Server"}
-	table := abraFormatter.CreateTable(tableCol)
-	table.Append([]string{sanitisedAppName, Domain, recipe.Name, NewAppServer})
+	tableCol := []string{"server", "type", "domain", "app name"}
+	table := formatter.CreateTable(tableCol)
+	table.Append([]string{NewAppServer, recipe.Name, Domain, NewAppName})
 
 	fmt.Println("")
 	fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
@@ -193,10 +173,10 @@ func NewAction(c *cli.Context) error {
 	table.Render()
 	fmt.Println("")
 	fmt.Println("You can configure this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app config %s", sanitisedAppName))
+	fmt.Println(fmt.Sprintf("\n    abra app config %s", NewAppName))
 	fmt.Println("")
 	fmt.Println("You can deploy this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", sanitisedAppName))
+	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", NewAppName))
 	fmt.Println("")
 
 	return nil

cli/internal/recipe.go

@@ -2,104 +2,53 @@ package internal
 
 import (
 	"fmt"
-	"strings"
 
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
 )
 
-var Major bool
-var MajorFlag = &cli.BoolFlag{
-	Name:        "major",
-	Usage:       "Increase the major part of the version",
-	Value:       false,
-	Aliases:     []string{"ma", "x"},
-	Destination: &Major,
-}
-
-var Minor bool
-var MinorFlag = &cli.BoolFlag{
-	Name:        "minor",
-	Usage:       "Increase the minor part of the version",
-	Value:       false,
-	Aliases:     []string{"mi", "y"},
-	Destination: &Minor,
-}
-
-var Patch bool
-var PatchFlag = &cli.BoolFlag{
-	Name:        "patch",
-	Usage:       "Increase the patch part of the version",
-	Value:       false,
-	Aliases:     []string{"p", "z"},
-	Destination: &Patch,
-}
-
-var Dry bool
-var DryFlag = &cli.BoolFlag{
-	Name:        "dry-run",
-	Usage:       "No changes are made, only reports changes that would be made",
-	Value:       false,
-	Aliases:     []string{"d"},
-	Destination: &Dry,
-}
-
-var Push bool
-var PushFlag = &cli.BoolFlag{
-	Name:        "push",
-	Usage:       "Git push changes",
-	Value:       false,
-	Aliases:     []string{"P"},
-	Destination: &Push,
-}
-
-var CommitMessage string
-var CommitMessageFlag = &cli.StringFlag{
-	Name:        "commit-message",
-	Usage:       "Commit message (implies --commit)",
-	Aliases:     []string{"cm"},
-	Destination: &CommitMessage,
-}
-
-var Commit bool
-var CommitFlag = &cli.BoolFlag{
-	Name:        "commit",
-	Usage:       "Commits compose.**yml file changes to recipe repository",
-	Value:       false,
-	Aliases:     []string{"c"},
-	Destination: &Commit,
-}
-
-var TagMessage string
-var TagMessageFlag = &cli.StringFlag{
-	Name:        "tag-comment",
-	Usage:       "Description for release tag",
-	Aliases:     []string{"t", "tm"},
-	Destination: &TagMessage,
-}
-
 // PromptBumpType prompts for version bump type
 func PromptBumpType(tagString string) error {
 	if (!Major && !Minor && !Patch) && tagString == "" {
 		fmt.Printf(`
-semver cheat sheet (more via semver.org):
-  major: new features/bug fixes, backwards incompatible
-  minor: new features/bug fixes, backwards compatible
-  patch: bug fixes, backwards compatible
+You need to make a decision about what kind of an update this new recipe
+version is. If someone else performs this upgrade, do they have to do some
+migration work or take care of some breaking changes? This can be signaled in
+the version you specify on the recipe deploy label and is called a semantic
+version.
+
+Here is a semver cheat sheet (more on https://semver.org):
+
+    major: new features/bug fixes, backwards incompatible (e.g 1.0.0 -> 2.0.0).
+           the upgrade won't work without some preparation work and others need
+           to take care when performing it. "it could go wrong".
+
+    minor: new features/bug fixes, backwards compatible (e.g. 0.1.0 -> 0.2.0).
+           the upgrade should Just Work and there are no breaking changes in
+           the app and the recipe config. "it should go fine".
+
+    patch: bug fixes, backwards compatible (e.g. 0.0.1 -> 0.0.2). this upgrade
+           should also Just Work and is mostly to do with minor bug fixes
+           and/or security patches. "nothing to worry about".
 
 `)
+
 		var chosenBumpType string
 		prompt := &survey.Select{
 			Message: fmt.Sprintf("select recipe version increment type"),
 			Options: []string{"major", "minor", "patch"},
 		}
+
 		if err := survey.AskOne(prompt, &chosenBumpType); err != nil {
 			return err
 		}
+
 		SetBumpType(chosenBumpType)
 	}
+
 	return nil
 }
 
@@ -133,20 +82,27 @@ func SetBumpType(bumpType string) {
 	}
 }
 
-// GetMainApp retrieves the main 'app' image name
-func GetMainApp(recipe recipe.Recipe) string {
-	var app string
+// GetMainAppImage retrieves the main 'app' image name
+func GetMainAppImage(recipe recipe.Recipe) (string, error) {
+	var path string
 
 	for _, service := range recipe.Config.Services {
-		name := service.Name
-		if name == "app" {
-			app = strings.Split(service.Image, ":")[0]
+		if service.Name == "app" {
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return "", err
+			}
+
+			path = reference.Path(img)
+			path = recipePkg.StripTagMeta(path)
+
+			return path, nil
 		}
 	}
 
-	if app == "" {
-		logrus.Fatalf("%s has no main 'app' service?", recipe.Name)
+	if path == "" {
+		return path, fmt.Errorf("%s has no main 'app' service?", recipe.Name)
 	}
 
-	return app
+	return path, nil
 }

cli/internal/record.go

@@ -1,106 +0,0 @@
-package internal
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/urfave/cli/v2"
-)
-
-// EnsureDNSProvider ensures a DNS provider is chosen.
-func EnsureDNSProvider() error {
-	if DNSProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select DNS provider",
-			Options: []string{"gandi"},
-		}
-
-		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
-			return err
-		}
-	}
-
-	if DNSProvider == "" {
-		return fmt.Errorf("missing DNS provider?")
-	}
-
-	return nil
-}
-
-// EnsureDNSTypeFlag ensures a DNS type flag is present.
-func EnsureDNSTypeFlag(c *cli.Context) error {
-	if DNSType == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record type",
-			Default: "A",
-		}
-		if err := survey.AskOne(prompt, &DNSType); err != nil {
-			return err
-		}
-	}
-
-	if DNSType == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSNameFlag ensures a DNS name flag is present.
-func EnsureDNSNameFlag(c *cli.Context) error {
-	if DNSName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record name",
-			Default: "mysubdomain",
-		}
-		if err := survey.AskOne(prompt, &DNSName); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSValueFlag ensures a DNS value flag is present.
-func EnsureDNSValueFlag(c *cli.Context) error {
-	if DNSValue == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record value",
-			Default: "192.168.1.2",
-		}
-		if err := survey.AskOne(prompt, &DNSValue); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
-	}
-
-	return nil
-}
-
-// EnsureZoneArgument ensures a zone argument is present.
-func EnsureZoneArgument(c *cli.Context) (string, error) {
-	var zone string
-	if c.Args().First() == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name zone",
-			Default: "example.com",
-		}
-		if err := survey.AskOne(prompt, &zone); err != nil {
-			return zone, err
-		}
-	}
-
-	if zone == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
-	}
-
-	return zone, nil
-}

cli/internal/server.go

@@ -1,208 +0,0 @@
-package internal
-
-import (
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/urfave/cli/v2"
-)
-
-// EnsureServerProvider ensures a 3rd party server provider is chosen.
-func EnsureServerProvider() error {
-	if ServerProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select server provider",
-			Options: []string{"capsul", "hetzner-cloud"},
-		}
-
-		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
-			return err
-		}
-	}
-
-	if ServerProvider == "" {
-		return fmt.Errorf("missing server provider?")
-	}
-
-	return nil
-}
-
-// EnsureNewCapsulVPSFlags ensure all flags are present.
-func EnsureNewCapsulVPSFlags(c *cli.Context) error {
-	if CapsulName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul name",
-		}
-		if err := survey.AskOne(prompt, &CapsulName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul instance URL",
-			Default: CapsulInstanceURL,
-		}
-		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul type",
-			Default: CapsulType,
-		}
-		if err := survey.AskOne(prompt, &CapsulType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul image",
-			Default: CapsulImage,
-		}
-		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
-			return err
-		}
-	}
-
-	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify capsul SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
-			return err
-		}
-		CapsulSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
-	}
-
-	if CapsulAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("CAPSUL_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify capsul API token",
-			}
-			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
-				return err
-			}
-		} else {
-			CapsulAPIToken = token
-		}
-	}
-
-	if CapsulName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
-	}
-	if CapsulInstanceURL == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
-	}
-	if CapsulType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
-	}
-	if CapsulImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
-	}
-	if len(CapsulSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
-	}
-	if CapsulAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
-	}
-
-	return nil
-}
-
-// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
-func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
-	if HetznerCloudName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS type",
-			Default: HetznerCloudType,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS image",
-			Default: HetznerCloudImage,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
-			return err
-		}
-	}
-
-	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &sshKeys); err != nil {
-			return err
-		}
-		HetznerCloudSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS location",
-			Default: HetznerCloudLocation,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
-			return err
-		}
-	}
-
-	if HetznerCloudAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			HetznerCloudAPIToken = token
-		}
-	}
-
-	if HetznerCloudName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
-	}
-	if HetznerCloudType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
-	}
-	if HetznerCloudImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
-	}
-	if len(HetznerCloudSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud ssh keys?"))
-	}
-	if HetznerCloudLocation == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
-	}
-	if HetznerCloudAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
-	}
-
-	return nil
-}

cli/internal/validate.go

@@ -2,12 +2,14 @@ package internal
 
 import (
 	"errors"
+	"fmt"
+	"os"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/app"
-	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -21,17 +23,24 @@ func ValidateRecipe(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
 
 	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	recipe, err := recipe.Get(recipeName)
+	chosenRecipe, err := recipe.Get(recipeName)
 	if err != nil {
-		logrus.Fatal(err)
+		if c.Command.Name == "generate" {
+			if strings.Contains(err.Error(), "missing a compose") {
+				logrus.Fatal(err)
+			}
+			logrus.Warn(err)
+		} else {
+			logrus.Fatal(err)
+		}
 	}
 
-	logrus.Debugf("validated '%s' as recipe argument", recipeName)
+	logrus.Debugf("validated %s as recipe argument", recipeName)
 
-	return recipe
+	return chosenRecipe
 }
 
 // ValidateRecipeWithPrompt ensures a recipe argument is present before
@@ -40,14 +49,33 @@ func ValidateRecipeWithPrompt(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
 
 	if recipeName == "" && !NoInput {
-		catl, err := catalogue.ReadRecipeCatalogue()
+		var recipes []string
+
+		catl, err := recipe.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		var recipes []string
+
+		knownRecipes := make(map[string]bool)
 		for name := range catl {
-			recipes = append(recipes, name)
+			knownRecipes[name] = true
 		}
+
+		localRecipes, err := recipe.GetRecipesLocal()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, recipeLocal := range localRecipes {
+			if _, ok := knownRecipes[recipeLocal]; !ok {
+				knownRecipes[recipeLocal] = true
+			}
+		}
+
+		for recipeName := range knownRecipes {
+			recipes = append(recipes, recipeName)
+		}
+
 		prompt := &survey.Select{
 			Message: "Select recipe",
 			Options: recipes,
@@ -63,17 +91,17 @@ func ValidateRecipeWithPrompt(c *cli.Context) recipe.Recipe {
 	}
 
 	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 	}
 
-	recipe, err := recipe.Get(recipeName)
+	chosenRecipe, err := recipe.Get(recipeName)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("validated '%s' as recipe argument", recipeName)
+	logrus.Debugf("validated %s as recipe argument", recipeName)
 
-	return recipe
+	return chosenRecipe
 }
 
 // ValidateApp ensures the app name arg is valid.
@@ -98,7 +126,11 @@ func ValidateApp(c *cli.Context) config.App {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("validated '%s' as app argument", appName)
+	if err := ssh.EnsureHostKey(app.Server); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated %s as app argument", appName)
 
 	return app
 }
@@ -121,7 +153,7 @@ func ValidateDomain(c *cli.Context) (string, error) {
 		ShowSubcommandHelpAndError(c, errors.New("no domain provided"))
 	}
 
-	logrus.Debugf("validated '%s' as domain argument", domainName)
+	logrus.Debugf("validated %s as domain argument", domainName)
 
 	return domainName, nil
 }
@@ -163,7 +195,301 @@ func ValidateServer(c *cli.Context) (string, error) {
 		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
 	}
 
-	logrus.Debugf("validated '%s' as server argument", serverName)
+	logrus.Debugf("validated %s as server argument", serverName)
 
 	return serverName, nil
 }
+
+// EnsureDNSProvider ensures a DNS provider is chosen.
+func EnsureDNSProvider() error {
+	if DNSProvider == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select DNS provider",
+			Options: []string{"gandi"},
+		}
+
+		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
+			return err
+		}
+	}
+
+	if DNSProvider == "" {
+		return fmt.Errorf("missing DNS provider?")
+	}
+
+	return nil
+}
+
+// EnsureDNSTypeFlag ensures a DNS type flag is present.
+func EnsureDNSTypeFlag(c *cli.Context) error {
+	if DNSType == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record type",
+			Default: "A",
+		}
+		if err := survey.AskOne(prompt, &DNSType); err != nil {
+			return err
+		}
+	}
+
+	if DNSType == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
+	}
+
+	return nil
+}
+
+// EnsureDNSNameFlag ensures a DNS name flag is present.
+func EnsureDNSNameFlag(c *cli.Context) error {
+	if DNSName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record name",
+			Default: "mysubdomain",
+		}
+		if err := survey.AskOne(prompt, &DNSName); err != nil {
+			return err
+		}
+	}
+
+	if DNSName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
+	}
+
+	return nil
+}
+
+// EnsureDNSValueFlag ensures a DNS value flag is present.
+func EnsureDNSValueFlag(c *cli.Context) error {
+	if DNSValue == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record value",
+			Default: "192.168.1.2",
+		}
+		if err := survey.AskOne(prompt, &DNSValue); err != nil {
+			return err
+		}
+	}
+
+	if DNSValue == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
+	}
+
+	return nil
+}
+
+// EnsureZoneArgument ensures a zone argument is present.
+func EnsureZoneArgument(c *cli.Context) (string, error) {
+	zone := c.Args().First()
+
+	if zone == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify a domain name zone",
+			Default: "example.com",
+		}
+
+		if err := survey.AskOne(prompt, &zone); err != nil {
+			return zone, err
+		}
+	}
+
+	if zone == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
+	}
+
+	return zone, nil
+}
+
+// EnsureServerProvider ensures a 3rd party server provider is chosen.
+func EnsureServerProvider() error {
+	if ServerProvider == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select server provider",
+			Options: []string{"capsul", "hetzner-cloud"},
+		}
+
+		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
+			return err
+		}
+	}
+
+	if ServerProvider == "" {
+		return fmt.Errorf("missing server provider?")
+	}
+
+	return nil
+}
+
+// EnsureNewCapsulVPSFlags ensure all flags are present.
+func EnsureNewCapsulVPSFlags(c *cli.Context) error {
+	if CapsulName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul name",
+		}
+		if err := survey.AskOne(prompt, &CapsulName); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul instance URL",
+			Default: CapsulInstanceURL,
+		}
+		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul type",
+			Default: CapsulType,
+		}
+		if err := survey.AskOne(prompt, &CapsulType); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul image",
+			Default: CapsulImage,
+		}
+		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
+			return err
+		}
+	}
+
+	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
+		var sshKeys string
+		prompt := &survey.Input{
+			Message: "specify capsul SSH keys (e.g. me@foo.com)",
+			Default: "",
+		}
+		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
+			return err
+		}
+		CapsulSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
+	}
+
+	if CapsulAPIToken == "" && !NoInput {
+		token, ok := os.LookupEnv("CAPSUL_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify capsul API token",
+			}
+			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
+				return err
+			}
+		} else {
+			CapsulAPIToken = token
+		}
+	}
+
+	if CapsulName == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
+	}
+	if CapsulInstanceURL == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
+	}
+	if CapsulType == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
+	}
+	if CapsulImage == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
+	}
+	if len(CapsulSSHKeys.Value()) == 0 {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
+	}
+	if CapsulAPIToken == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
+	}
+
+	return nil
+}
+
+// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
+func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
+	if HetznerCloudName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS name",
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS type",
+			Default: HetznerCloudType,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS image",
+			Default: HetznerCloudImage,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
+			return err
+		}
+	}
+
+	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
+		var sshKeys string
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
+			Default: "",
+		}
+		if err := survey.AskOne(prompt, &sshKeys); err != nil {
+			return err
+		}
+		HetznerCloudSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS location",
+			Default: HetznerCloudLocation,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
+			return err
+		}
+	}
+
+	if HetznerCloudAPIToken == "" && !NoInput {
+		token, ok := os.LookupEnv("HCLOUD_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify hetzner cloud API token",
+			}
+			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
+				return err
+			}
+		} else {
+			HetznerCloudAPIToken = token
+		}
+	}
+
+	if HetznerCloudName == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
+	}
+	if HetznerCloudType == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
+	}
+	if HetznerCloudImage == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
+	}
+	if HetznerCloudLocation == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
+	}
+	if HetznerCloudAPIToken == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
+	}
+
+	return nil
+}

cli/recipe/lint.go

@@ -2,112 +2,74 @@ package recipe
 
 import (
 	"fmt"
-	"os"
-	"strconv"
 
-	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/tagcmp"
-	"github.com/docker/distribution/reference"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/lint"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 var recipeLintCommand = &cli.Command{
-	Name:      "lint",
-	Usage:     "Lint a recipe",
-	Aliases:   []string{"l"},
-	ArgsUsage: "<recipe>",
+	Name:         "lint",
+	Usage:        "Lint a recipe",
+	Aliases:      []string{"l"},
+	ArgsUsage:    "<recipe>",
+	Flags:        []cli.Flag{internal.OnlyErrorFlag},
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
-		expectedVersion := false
-		if recipe.Config.Version == "3.8" {
-			expectedVersion = true
-		}
-
-		envSampleProvided := false
-		envSample := fmt.Sprintf("%s/%s/.env.sample", config.APPS_DIR, recipe.Name)
-		if _, err := os.Stat(envSample); !os.IsNotExist(err) {
-			envSampleProvided = true
-		} else if err != nil {
+		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
 			logrus.Fatal(err)
 		}
 
-		serviceNamedApp := false
-		traefikEnabled := false
-		healthChecksForAllServices := true
-		allImagesTagged := true
-		noUnstableTags := true
-		semverLikeTags := true
-		for _, service := range recipe.Config.Services {
-			if service.Name == "app" {
-				serviceNamedApp = true
-			}
+		tableCol := []string{"ref", "rule", "satisfied", "severity", "resolve"}
+		table := formatter.CreateTable(tableCol)
 
-			for label := range service.Deploy.Labels {
-				if label == "traefik.enable" {
-					if service.Deploy.Labels[label] == "true" {
-						traefikEnabled = true
-					}
+		hasError := false
+		bar := formatter.CreateProgressbar(-1, "running recipe lint rules...")
+		for level := range lint.LintRules {
+			for _, rule := range lint.LintRules[level] {
+				ok, err := rule.Function(recipe)
+				if err != nil {
+					logrus.Warn(err)
 				}
-			}
 
-			img, err := reference.ParseNormalizedNamed(service.Image)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			if reference.IsNameOnly(img) {
-				allImagesTagged = false
-			}
+				if !ok && rule.Level == "error" {
+					hasError = true
+				}
 
-			var tag string
-			switch img.(type) {
-			case reference.NamedTagged:
-				tag = img.(reference.NamedTagged).Tag()
-			case reference.Named:
-				noUnstableTags = false
-			}
+				var result string
+				if ok {
+					result = "yes"
+				} else {
+					result = "NO"
+				}
 
-			if tag == "latest" {
-				noUnstableTags = false
-			}
-
-			if !tagcmp.IsParsable(tag) {
-				semverLikeTags = false
-			}
-
-			if service.HealthCheck == nil {
-				healthChecksForAllServices = false
+				if internal.OnlyErrors {
+					if !ok && rule.Level == "error" {
+						table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
+						bar.Add(1)
+					}
+				} else {
+					table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
+					bar.Add(1)
+				}
 			}
 		}
 
-		tableCol := []string{"rule", "satisfied"}
-		table := formatter.CreateTable(tableCol)
-		table.Append([]string{"compose files have the expected version", strconv.FormatBool(expectedVersion)})
-		table.Append([]string{"environment configuration is provided", strconv.FormatBool(envSampleProvided)})
-		table.Append([]string{"recipe contains a service named 'app'", strconv.FormatBool(serviceNamedApp)})
-		table.Append([]string{"traefik routing enabled on at least one service", strconv.FormatBool(traefikEnabled)})
-		table.Append([]string{"all services have a healthcheck enabled", strconv.FormatBool(healthChecksForAllServices)})
-		table.Append([]string{"all images are using a tag", strconv.FormatBool(allImagesTagged)})
-		table.Append([]string{"no usage of unstable 'latest' tags", strconv.FormatBool(noUnstableTags)})
-		table.Append([]string{"all tags are using a semver-like format", strconv.FormatBool(semverLikeTags)})
-		table.Render()
+		if table.NumLines() > 0 {
+			fmt.Println()
+			table.Render()
+		}
+
+		if hasError {
+			logrus.Warn("watch out, some critical errors are present in your recipe config")
+		}
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		catl, err := catalogue.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for name := range catl {
-			fmt.Println(name)
-		}
-	},
 }

cli/recipe/list.go

@@ -2,37 +2,82 @@ package recipe
 
 import (
 	"fmt"
+	"path"
 	"sort"
+	"strconv"
+	"strings"
 
-	"coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
+var pattern string
+var patternFlag = &cli.StringFlag{
+	Name:        "pattern",
+	Value:       "",
+	Aliases:     []string{"p"},
+	Usage:       "Simple string to filter recipes",
+	Destination: &pattern,
+}
+
 var recipeListCommand = &cli.Command{
 	Name:    "list",
 	Usage:   "List available recipes",
 	Aliases: []string{"ls"},
+	Flags: []cli.Flag{
+		patternFlag,
+	},
 	Action: func(c *cli.Context) error {
-		catl, err := catalogue.ReadRecipeCatalogue()
+		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		catl, err := recipe.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err.Error())
 		}
 
 		recipes := catl.Flatten()
-		sort.Sort(catalogue.ByRecipeName(recipes))
+		sort.Sort(recipe.ByRecipeName(recipes))
 
-		tableCol := []string{"name", "category", "status"}
+		tableCol := []string{"name", "category", "status", "healthcheck", "backups", "email", "tests", "SSO"}
 		table := formatter.CreateTable(tableCol)
 
+		len := 0
 		for _, recipe := range recipes {
-			status := fmt.Sprintf("%v", recipe.Features.Status)
-			tableRow := []string{recipe.Name, recipe.Category, status}
-			table.Append(tableRow)
+			tableRow := []string{
+				recipe.Name,
+				recipe.Category,
+				strconv.Itoa(recipe.Features.Status),
+				recipe.Features.Healthcheck,
+				recipe.Features.Backups,
+				recipe.Features.Email,
+				recipe.Features.Tests,
+				recipe.Features.SSO,
+			}
+
+			if pattern != "" {
+				if strings.Contains(recipe.Name, pattern) {
+					table.Append(tableRow)
+					len++
+				}
+			} else {
+				table.Append(tableRow)
+				len++
+			}
 		}
 
-		table.Render()
+		table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
+
+		if table.NumLines() > 0 {
+			table.Render()
+		}
 
 		return nil
 	},

cli/recipe/new.go

@@ -1,8 +1,10 @@
 package recipe
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"text/template"
@@ -14,6 +16,20 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+// recipeMetadata is the recipe metadata for the README.md
+type recipeMetadata struct {
+	Name        string
+	Description string
+	Category    string
+	Status      string
+	Image       string
+	Healthcheck string
+	Backups     string
+	Email       string
+	Tests       string
+	SSO         string
+}
+
 var recipeNewCommand = &cli.Command{
 	Name:      "new",
 	Usage:     "Create a new recipe",
@@ -29,19 +45,17 @@ Abra uses our built-in example repository which is available here:
 Files within the example repository make use of the Golang templating system
 which Abra uses to inject values into the generated recipe folder (e.g. name of
 recipe and domain in the sample environment config).
-
-The new example repository is cloned to ~/.abra/apps/<recipe>.
 `,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
 
 		if recipeName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
 		}
 
-		directory := path.Join(config.APPS_DIR, recipeName)
+		directory := path.Join(config.RECIPES_DIR, recipeName)
 		if _, err := os.Stat(directory); !os.IsNotExist(err) {
-			logrus.Fatalf("'%s' recipe directory already exists?", directory)
+			logrus.Fatalf("%s recipe directory already exists?", directory)
 		}
 
 		url := fmt.Sprintf("%s/example.git", config.REPOS_BASE_URL)
@@ -49,44 +63,73 @@ The new example repository is cloned to ~/.abra/apps/<recipe>.
 			logrus.Fatal(err)
 		}
 
-		gitRepo := path.Join(config.APPS_DIR, recipeName, ".git")
+		gitRepo := path.Join(config.RECIPES_DIR, recipeName, ".git")
 		if err := os.RemoveAll(gitRepo); err != nil {
 			logrus.Fatal(err)
 		}
-		logrus.Debugf("removed git repo in '%s'", gitRepo)
+		logrus.Debugf("removed example git repo in %s", gitRepo)
+
+		meta := newRecipeMeta(recipeName)
 
 		toParse := []string{
-			path.Join(config.APPS_DIR, recipeName, "README.md"),
-			path.Join(config.APPS_DIR, recipeName, ".env.sample"),
-			path.Join(config.APPS_DIR, recipeName, ".drone.yml"),
+			path.Join(config.RECIPES_DIR, recipeName, "README.md"),
+			path.Join(config.RECIPES_DIR, recipeName, ".env.sample"),
 		}
 		for _, path := range toParse {
-			file, err := os.OpenFile(path, os.O_RDWR, 0755)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
 			tpl, err := template.ParseFiles(path)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-			// TODO: ask for description and probably other things so that the
-			// template repository is more "ready" to go than the current best-guess
-			// mode of templating
-			if err := tpl.Execute(file, struct {
-				Name        string
-				Description string
-			}{recipeName, "TODO"}); err != nil {
+			var templated bytes.Buffer
+			if err := tpl.Execute(&templated, meta); err != nil {
 				logrus.Fatal(err)
 			}
+
+			if err := ioutil.WriteFile(path, templated.Bytes(), 0644); err != nil {
+				logrus.Fatal(err)
+			}
+
 		}
 
-		logrus.Infof(
-			"new recipe '%s' created in %s, happy hacking!\n",
-			recipeName, path.Join(config.APPS_DIR, recipeName),
-		)
+		newGitRepo := path.Join(config.RECIPES_DIR, recipeName)
+		if err := git.Init(newGitRepo, true); err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Print(fmt.Sprintf(`
+Your new %s recipe has been created in %s.
+
+In order to share your recipe, you can upload it the git repository to:
+
+    https://git.coopcloud.tech/coop-cloud/%s
+
+If you're not sure how to do that, come chat with us:
+
+    https://docs.coopcloud.tech/contact
+
+See "abra recipe -h" for additional recipe maintainer commands.
+
+Happy Hacking!
+
+`, recipeName, path.Join(config.RECIPES_DIR, recipeName), recipeName))
 
 		return nil
 	},
 }
+
+// newRecipeMeta creates a new recipeMetadata instance with defaults
+func newRecipeMeta(recipeName string) recipeMetadata {
+	return recipeMetadata{
+		Name:        recipeName,
+		Description: "> One line description of the recipe",
+		Category:    "Apps",
+		Status:      "0",
+		Image:       fmt.Sprintf("[`%s`](https://hub.docker.com/r/%s), 4, upstream", recipeName, recipeName),
+		Healthcheck: "No",
+		Backups:     "No",
+		Email:       "No",
+		Tests:       "No",
+		SSO:         "No",
+	}
+}

cli/recipe/recipe.go

@@ -7,13 +7,18 @@ import (
 // RecipeCommand defines all recipe related sub-commands.
 var RecipeCommand = &cli.Command{
 	Name:      "recipe",
-	Usage:     "Manage recipes (for maintainers)",
+	Usage:     "Manage recipes",
 	ArgsUsage: "<recipe>",
 	Aliases:   []string{"r"},
 	Description: `
-A recipe is a blueprint for an app. It is a bunch of configuration files which
+A recipe is a blueprint for an app. It is a bunch of config files which
 describe how to deploy and maintain an app. Recipes are maintained by the Co-op
 Cloud community and you can use Abra to read them and create apps for you.
+
+Anyone who uses a recipe can become a maintainer. Maintainers typically make
+sure the recipe is in good working order and the config upgraded in a timely
+manner. Abra supports convenient automation for recipe maintainenace, see the
+"abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands.
 `,
 	Subcommands: []*cli.Command{
 		recipeListCommand,

cli/recipe/release.go

@@ -6,16 +6,17 @@ import (
 	"strconv"
 	"strings"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -26,270 +27,96 @@ var recipeReleaseCommand = &cli.Command{
 	Aliases:   []string{"rl"},
 	ArgsUsage: "<recipe> [<version>]",
 	Description: `
-This command is used to specify a new tag for a recipe. These tags are used to
-identify different versions of the recipe and are published on the Co-op Cloud
-recipe catalogue.
-
-These tags take the following form:
+This command is used to specify a new version of a recipe. These versions are
+then published on the Co-op Cloud recipe catalogue. These versions take the
+following form:
 
     a.b.c+x.y.z
 
-Where the "a.b.c" part is maintained as a semantic version of the recipe by the
-recipe maintainer. And the "x.y.z" part is the image tag of the recipe "app"
-service (the main container which contains the software to be used).
+Where the "a.b.c" part is a semantic version determined by the maintainer. And
+the "x.y.z" part is the image tag of the recipe "app" service (the main
+container which contains the software to be used).
 
 We maintain a semantic versioning scheme ("a.b.c") alongside the libre app
-versioning scheme in order to maximise the chances that the nature of recipe
-updates are properly communicated.
-
-Abra does its best to read the "a.b.c" version scheme and communicate what
-action needs to be taken when performing different operations such as an update
-or a rollback of an app.
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra recipe release gitea
+versioning scheme ("x.y.z") in order to maximise the chances that the nature of
+recipe updates are properly communicated. I.e. developers of an app might
+publish a minor version but that might lead to changes in the recipe which are
+major and therefore require intervention while doing the upgrade work.
 
+Publish your new release to git.coopcloud.tech with "-p/--publish". This
+requires that you have permission to git push to these repositories and have
+your SSH keys configured on your account.
 `,
 	Flags: []cli.Flag{
 		internal.DryFlag,
 		internal.MajorFlag,
 		internal.MinorFlag,
 		internal.PatchFlag,
-		internal.PushFlag,
-		internal.CommitFlag,
-		internal.CommitMessageFlag,
-		internal.TagMessageFlag,
+		internal.PublishFlag,
 	},
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipeWithPrompt(c)
-		directory := path.Join(config.APPS_DIR, recipe.Name)
-		tagString := c.Args().Get(1)
-		mainApp := internal.GetMainApp(recipe)
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		mainAppVersion := imagesTmp[mainApp]
 
-		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+		mainApp, err := internal.GetMainAppImage(recipe)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		mainAppVersion := imagesTmp[mainApp]
 		if mainAppVersion == "" {
-			logrus.Fatalf("main 'app' service version for %s is empty?", recipe.Name)
+			logrus.Fatalf("main app service version for %s is empty?", recipe.Name)
 		}
 
+		tagString := c.Args().Get(1)
 		if tagString != "" {
 			if _, err := tagcmp.Parse(tagString); err != nil {
-				logrus.Fatal("invalid tag specified")
+				logrus.Fatalf("cannot parse %s, invalid tag specified?", tagString)
 			}
 		}
 
-		if (!internal.Major && !internal.Minor && !internal.Patch) && tagString != "" {
-			logrus.Fatal("please specify <version> or bump type (--major/--minor/--patch)")
-		}
-
 		if (internal.Major || internal.Minor || internal.Patch) && tagString != "" {
 			logrus.Fatal("cannot specify tag and bump type at the same time")
 		}
 
-		// bumpType is used to decide what part of the tag should be incremented
-		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
-		if bumpType != 0 {
-			// a bitwise check if the number is a power of 2
-			if (bumpType & (bumpType - 1)) != 0 {
-				logrus.Fatal("you can only use one of: --major, --minor, --patch.")
-			}
-		}
-
-		if err := internal.PromptBumpType(tagString); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if internal.TagMessage == "" {
-			prompt := &survey.Input{
-				Message: "tag message",
-				Default: fmt.Sprintf("chore: publish new %s version", internal.GetBumpType()),
-			}
-			if err := survey.AskOne(prompt, &internal.TagMessage); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		var createTagOptions git.CreateTagOptions
-		createTagOptions.Message = internal.TagMessage
-
-		if !internal.Commit {
-			prompt := &survey.Confirm{
-				Message: "git commit changes also?",
-			}
-			if err := survey.AskOne(prompt, &internal.Commit); err != nil {
-				return err
-			}
-		}
-
-		if !internal.Push {
-			prompt := &survey.Confirm{
-				Message: "git push changes also?",
-			}
-			if err := survey.AskOne(prompt, &internal.Push); err != nil {
-				return err
-			}
-		}
-
-		if internal.Commit || internal.CommitMessage != "" {
-			commitRepo, err := git.PlainOpen(directory)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			commitWorktree, err := commitRepo.Worktree()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if internal.CommitMessage == "" {
-				prompt := &survey.Input{
-					Message: "commit message",
-					Default: fmt.Sprintf("chore: publish new %s version", internal.GetBumpType()),
-				}
-				if err := survey.AskOne(prompt, &internal.CommitMessage); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			err = commitWorktree.AddGlob("compose.**yml")
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			logrus.Debug("staged compose.**yml for commit")
-
-			if !internal.Dry {
-				_, err = commitWorktree.Commit(internal.CommitMessage, &git.CommitOptions{})
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				logrus.Info("changes commited")
-			} else {
-				logrus.Info("dry run only: NOT committing changes")
-			}
-		}
-
-		repo, err := git.PlainOpen(directory)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		head, err := repo.Head()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		if tagString != "" {
-			tag, err := tagcmp.Parse(tagString)
-			if err != nil {
+			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
 				logrus.Fatal(err)
 			}
-			if tag.MissingMinor {
-				tag.Minor = "0"
-				tag.MissingMinor = false
-			}
-			if tag.MissingPatch {
-				tag.Patch = "0"
-				tag.MissingPatch = false
-			}
-			tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
-			if internal.Dry {
-				hash := abraFormatter.SmallSHA(head.Hash().String())
-				logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", tagString, hash))
-				return nil
-			}
-
-			repo.CreateTag(tagString, head.Hash(), &createTagOptions)
-			hash := abraFormatter.SmallSHA(head.Hash().String())
-			logrus.Info(fmt.Sprintf("created tag %s at %s", tagString, hash))
-			if internal.Push && !internal.Dry {
-				if err := repo.Push(&git.PushOptions{}); err != nil {
-					logrus.Fatal(err)
-				}
-				logrus.Info(fmt.Sprintf("pushed tag %s to remote", tagString))
-			} else {
-				logrus.Info("dry run only: NOT pushing changes")
-			}
-
-			return nil
 		}
 
-		// get the latest tag with its hash, name etc
-		var lastGitTag tagcmp.Tag
-		iter, err := repo.Tags()
+		tags, err := recipe.Tags()
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		if err := iter.ForEach(func(ref *plumbing.Reference) error {
-			obj, err := repo.TagObject(ref.Hash())
+
+		if tagString == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+			var err error
+			tagString, err = getLabelVersion(recipe, false)
 			if err != nil {
-				return err
-			}
-			tagcmpTag, err := tagcmp.Parse(obj.Name)
-			if err != nil {
-				return err
-			}
-			if (lastGitTag == tagcmp.Tag{}) {
-				lastGitTag = tagcmpTag
-			} else if tagcmpTag.IsGreaterThan(lastGitTag) {
-				lastGitTag = tagcmpTag
-			}
-			return nil
-		}); err != nil {
-			logrus.Fatal(err)
-		}
-
-		newTag := lastGitTag
-		var newtagString string
-		if bumpType > 0 {
-			if internal.Patch {
-				now, err := strconv.Atoi(newTag.Patch)
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				newTag.Patch = strconv.Itoa(now + 1)
-			} else if internal.Minor {
-				now, err := strconv.Atoi(newTag.Minor)
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				newTag.Patch = "0"
-				newTag.Minor = strconv.Itoa(now + 1)
-			} else if internal.Major {
-				now, err := strconv.Atoi(newTag.Major)
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				newTag.Patch = "0"
-				newTag.Minor = "0"
-				newTag.Major = strconv.Itoa(now + 1)
-			}
-		}
-
-		newTag.Metadata = mainAppVersion
-		newtagString = newTag.String()
-		if internal.Dry {
-			hash := abraFormatter.SmallSHA(head.Hash().String())
-			logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", newtagString, hash))
-			return nil
-		}
-
-		repo.CreateTag(newtagString, head.Hash(), &createTagOptions)
-		hash := abraFormatter.SmallSHA(head.Hash().String())
-		logrus.Info(fmt.Sprintf("created tag %s at %s", newtagString, hash))
-		if internal.Push && !internal.Dry {
-			if err := repo.Push(&git.PushOptions{}); err != nil {
 				logrus.Fatal(err)
 			}
-			logrus.Info(fmt.Sprintf("pushed tag %s to remote", newtagString))
+		}
+
+		if len(tags) > 0 {
+			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
+			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
+				logrus.Fatal(err)
+			}
 		} else {
-			logrus.Info("gry run only: NOT pushing changes")
+			logrus.Warnf("no tag specified and no previous tag available for %s, assuming this is the initial release", recipe.Name)
+
+			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+				if cleanUpErr := cleanUpTag(tagString, recipe.Name); err != nil {
+					logrus.Fatal(cleanUpErr)
+				}
+				logrus.Fatal(err)
+			}
 		}
 
 		return nil
@@ -300,6 +127,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
 	var services = make(map[string]string)
 
+	missingTag := false
 	for _, service := range recipe.Config.Services {
 		if service.Image == "" {
 			continue
@@ -311,24 +139,74 @@ func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
 		}
 
 		path := reference.Path(img)
-		if strings.Contains(path, "library") {
-			path = strings.Split(path, "/")[1]
-		}
+
+		path = recipePkg.StripTagMeta(path)
 
 		var tag string
 		switch img.(type) {
 		case reference.NamedTagged:
 			tag = img.(reference.NamedTagged).Tag()
 		case reference.Named:
-			logrus.Fatalf("%s service is missing image tag?", path)
+			if service.Name == "app" {
+				missingTag = true
+			}
+			continue
 		}
 
 		services[path] = tag
 	}
 
+	if missingTag {
+		return services, fmt.Errorf("app service is missing image tag?")
+	}
+
 	return services, nil
 }
 
+// createReleaseFromTag creates a new release based on a supplied recipe version string
+func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string) error {
+	var err error
+
+	directory := path.Join(config.RECIPES_DIR, recipe.Name)
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return err
+	}
+
+	tag, err := tagcmp.Parse(tagString)
+	if err != nil {
+		return err
+	}
+
+	if tag.MissingMinor {
+		tag.Minor = "0"
+		tag.MissingMinor = false
+	}
+
+	if tag.MissingPatch {
+		tag.Patch = "0"
+		tag.MissingPatch = false
+	}
+
+	if tagString == "" {
+		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
+	}
+
+	if err := commitRelease(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := tagRelease(tagString, repo); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := pushRelease(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}
+
 // btoi converts a boolean value into an integer
 func btoi(b bool) int {
 	if b {
@@ -337,3 +215,242 @@ func btoi(b bool) int {
 
 	return 0
 }
+
+// getTagCreateOptions constructs git tag create options
+func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
+	msg := fmt.Sprintf("chore: publish %s release", tag)
+	return git.CreateTagOptions{Message: msg}, nil
+}
+
+func commitRelease(recipe recipe.Recipe, tag string) error {
+	if internal.Dry {
+		logrus.Debugf("dry run: no changes committed")
+		return nil
+	}
+
+	isClean, err := gitPkg.IsClean(recipe.Dir())
+	if err != nil {
+		return err
+	}
+
+	if isClean {
+		if !internal.Dry {
+			return fmt.Errorf("no changes discovered in %s, nothing to publish?", recipe.Dir())
+		}
+	}
+
+	if internal.Publish {
+		msg := fmt.Sprintf("chore: publish %s release", tag)
+		repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
+		if err := gitPkg.Commit(repoPath, "compose.**yml", msg, internal.Dry); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func tagRelease(tagString string, repo *git.Repository) error {
+	if internal.Dry {
+		logrus.Debugf("dry run: no git tag created (%s)", tagString)
+		return nil
+	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return err
+	}
+
+	createTagOptions, err := getTagCreateOptions(tagString)
+	if err != nil {
+		return err
+	}
+
+	_, err = repo.CreateTag(tagString, head.Hash(), &createTagOptions)
+	if err != nil {
+		return err
+	}
+
+	hash := formatter.SmallSHA(head.Hash().String())
+	logrus.Debugf(fmt.Sprintf("created tag %s at %s", tagString, hash))
+
+	return nil
+}
+
+func pushRelease(recipe recipe.Recipe, tagString string) error {
+	if internal.Dry {
+		logrus.Info("dry run: no changes published")
+		return nil
+	}
+
+	if !internal.Publish && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: "publish new release?",
+		}
+
+		if err := survey.AskOne(prompt, &internal.Publish); err != nil {
+			return err
+		}
+	}
+
+	if internal.Publish {
+		if err := recipe.Push(internal.Dry); err != nil {
+			return err
+		}
+
+		if !internal.Dry {
+			url := fmt.Sprintf("%s/%s/src/tag/%s", config.REPOS_BASE_URL, recipe.Name, tagString)
+			logrus.Infof("new release published: %s", url)
+		} else {
+			logrus.Info("dry run: no changes published")
+		}
+	}
+
+	return nil
+}
+
+func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recipe.Recipe, tags []string) error {
+	directory := path.Join(config.RECIPES_DIR, recipe.Name)
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return err
+	}
+
+	bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+	if bumpType != 0 {
+		if (bumpType & (bumpType - 1)) != 0 {
+			return fmt.Errorf("you can only use one of: --major, --minor, --patch")
+		}
+	}
+
+	var lastGitTag tagcmp.Tag
+	if tagString == "" {
+		if err := internal.PromptBumpType(tagString); err != nil {
+			return err
+		}
+	}
+
+	for _, tag := range tags {
+		parsed, err := tagcmp.Parse(tag)
+		if err != nil {
+			return err
+		}
+
+		if (lastGitTag == tagcmp.Tag{}) {
+			lastGitTag = parsed
+		} else if parsed.IsGreaterThan(lastGitTag) {
+			lastGitTag = parsed
+		}
+	}
+
+	newTag := lastGitTag
+	if internal.Patch {
+		now, err := strconv.Atoi(newTag.Patch)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = strconv.Itoa(now + 1)
+	} else if internal.Minor {
+		now, err := strconv.Atoi(newTag.Minor)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = "0"
+		newTag.Minor = strconv.Itoa(now + 1)
+	} else if internal.Major {
+		now, err := strconv.Atoi(newTag.Major)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = "0"
+		newTag.Minor = "0"
+		newTag.Major = strconv.Itoa(now + 1)
+	}
+
+	if internal.Major || internal.Minor || internal.Patch {
+		newTag.Metadata = mainAppVersion
+		tagString = newTag.String()
+	}
+
+	if lastGitTag.String() == tagString {
+		logrus.Fatalf("latest git tag (%s) and synced lable (%s) are the same?", lastGitTag, tagString)
+	}
+
+	if !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: fmt.Sprintf("current: %s, new: %s, correct?", lastGitTag, tagString),
+		}
+
+		var ok bool
+		if err := survey.AskOne(prompt, &ok); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !ok {
+			logrus.Fatal("exiting as requested")
+		}
+	}
+
+	if err := commitRelease(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := tagRelease(tagString, repo); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := pushRelease(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}
+
+// cleanUpTag removes a freshly created tag
+func cleanUpTag(tag, recipeName string) error {
+	directory := path.Join(config.RECIPES_DIR, recipeName)
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return err
+	}
+
+	if err := repo.DeleteTag(tag); err != nil {
+		if !strings.Contains(err.Error(), "not found") {
+			return err
+		}
+	}
+
+	logrus.Debugf("removed freshly created tag %s", tag)
+
+	return nil
+}
+
+func getLabelVersion(recipe recipe.Recipe, prompt bool) (string, error) {
+	initTag, err := recipePkg.GetVersionLabelLocal(recipe)
+	if err != nil {
+		return "", err
+	}
+
+	if initTag == "" {
+		logrus.Fatalf("unable to read version for %s from synced label. Did you try running \"abra recipe sync %s\" already?", recipe.Name, recipe.Name)
+	}
+
+	logrus.Warnf("discovered %s as currently synced recipe label", initTag)
+
+	if prompt && !internal.NoInput {
+		var response bool
+		prompt := &survey.Confirm{Message: fmt.Sprintf("use %s as the new version?", initTag)}
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return "", err
+		}
+
+		if !response {
+			return "", fmt.Errorf("please fix your synced label for %s and re-run this command", recipe.Name)
+		}
+	}
+
+	return initTag, nil
+}

cli/recipe/sync.go

@@ -6,7 +6,7 @@ import (
 	"strconv"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -18,7 +18,7 @@ import (
 
 var recipeSyncCommand = &cli.Command{
 	Name:      "sync",
-	Usage:     "Ensure recipe version labels are up-to-date",
+	Usage:     "Sync recipe version label",
 	Aliases:   []string{"s"},
 	ArgsUsage: "<recipe> [<version>]",
 	Flags: []cli.Flag{
@@ -29,28 +29,27 @@ var recipeSyncCommand = &cli.Command{
 	},
 	Description: `
 This command will generate labels for the main recipe service (i.e. by
-convention, the service named "app") which corresponds to the following format:
+convention, the service named 'app') which corresponds to the following format:
 
     coop-cloud.${STACK_NAME}.version=<version>
 
-The <version> is determined by the recipe maintainer and is specified on the
-command-line. The <recipe> configuration will be updated on the local file
-system.
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra recipe sync gitea
-
+Where <version> can be specifed on the command-line or Abra can attempt to
+auto-generate it for you. The <recipe> configuration will be updated on the
+local file system.
 `,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipeWithPrompt(c)
 
-		mainApp := internal.GetMainApp(recipe)
+		mainApp, err := internal.GetMainAppImage(recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
+
 		mainAppVersion := imagesTmp[mainApp]
 
 		tags, err := recipe.Tags()
@@ -60,15 +59,35 @@ You may invoke this command in "wizard" mode and be prompted for input:
 
 		nextTag := c.Args().Get(1)
 		if len(tags) == 0 && nextTag == "" {
-			logrus.Warnf("no tags found for %s", recipe.Name)
+			logrus.Warnf("no git tags found for %s", recipe.Name)
+			fmt.Println(fmt.Sprintf(`
+The following options are two types of initial semantic version that you can
+pick for %s that will be published in the recipe catalogue. This follows the
+semver convention (more on https://semver.org), here is a short cheatsheet
+
+    0.1.0: development release, still hacking. when you make a major upgrade
+           you increment the "y" part (i.e. 0.1.0 -> 0.2.0) and only move to
+           using the "x" part when things are stable.
+
+    1.0.0: public release, assumed to be working. you already have a stable
+           and reliable deployment of this app and feel relatively confident
+           about it.
+
+If you want people to be able alpha test your current config for %s but don't
+think it is quite reliable, go with 0.1.0 and people will know that things are
+likely to change.
+
+`, recipe.Name, recipe.Name))
 			var chosenVersion string
 			edPrompt := &survey.Select{
 				Message: "which version do you want to begin with?",
 				Options: []string{"0.1.0", "1.0.0"},
 			}
+
 			if err := survey.AskOne(edPrompt, &chosenVersion); err != nil {
 				logrus.Fatal(err)
 			}
+
 			nextTag = fmt.Sprintf("%s+%s", chosenVersion, mainAppVersion)
 		}
 
@@ -79,30 +98,35 @@ You may invoke this command in "wizard" mode and be prompted for input:
 		}
 
 		if nextTag == "" {
-			recipeDir := path.Join(config.APPS_DIR, recipe.Name)
+			recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
 			repo, err := git.PlainOpen(recipeDir)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+
 			var lastGitTag tagcmp.Tag
 			iter, err := repo.Tags()
 			if err != nil {
 				logrus.Fatal(err)
 			}
+
 			if err := iter.ForEach(func(ref *plumbing.Reference) error {
 				obj, err := repo.TagObject(ref.Hash())
 				if err != nil {
 					return err
 				}
+
 				tagcmpTag, err := tagcmp.Parse(obj.Name)
 				if err != nil {
 					return err
 				}
+
 				if (lastGitTag == tagcmp.Tag{}) {
 					lastGitTag = tagcmpTag
 				} else if tagcmpTag.IsGreaterThan(lastGitTag) {
 					lastGitTag = tagcmpTag
 				}
+
 				return nil
 			}); err != nil {
 				logrus.Fatal(err)
@@ -113,7 +137,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 			if bumpType != 0 {
 				// a bitwise check if the number is a power of 2
 				if (bumpType & (bumpType - 1)) != 0 {
-					logrus.Fatal("you can only use one of: --major, --minor, --patch.")
+					logrus.Fatal("you can only use one version flag: --major, --minor or --patch")
 				}
 			}
 
@@ -124,12 +148,14 @@ You may invoke this command in "wizard" mode and be prompted for input:
 					if err != nil {
 						logrus.Fatal(err)
 					}
+
 					newTag.Patch = strconv.Itoa(now + 1)
 				} else if internal.Minor {
 					now, err := strconv.Atoi(newTag.Minor)
 					if err != nil {
 						logrus.Fatal(err)
 					}
+
 					newTag.Patch = "0"
 					newTag.Minor = strconv.Itoa(now + 1)
 				} else if internal.Major {
@@ -137,6 +163,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 					if err != nil {
 						logrus.Fatal(err)
 					}
+
 					newTag.Patch = "0"
 					newTag.Minor = "0"
 					newTag.Major = strconv.Itoa(now + 1)
@@ -153,44 +180,16 @@ You may invoke this command in "wizard" mode and be prompted for input:
 		}
 
 		mainService := "app"
-		var services []string
-		hasAppService := false
-		for _, service := range recipe.Config.Services {
-			services = append(services, service.Name)
-			if service.Name == "app" {
-				hasAppService = true
-				logrus.Debugf("detected app service in %s", recipe.Name)
-			}
-		}
-
-		if !hasAppService {
-			logrus.Fatalf("%s has no main 'app' service?", recipe.Name)
-		}
-
-		logrus.Debugf("selecting %s as the service to sync version label", mainService)
-
 		label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", nextTag)
 		if !internal.Dry {
-			if err := recipe.UpdateLabel(mainService, label); err != nil {
+			if err := recipe.UpdateLabel("compose.y*ml", mainService, label); err != nil {
 				logrus.Fatal(err)
 			}
-			logrus.Infof("synced label '%s' to service '%s'", label, mainService)
 		} else {
-			logrus.Infof("dry run only: NOT syncing label %s for recipe %s", nextTag, recipe.Name)
+			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
 		return nil
 	},
-	BashComplete: func(c *cli.Context) {
-		catl, err := catalogue.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for name := range catl {
-			fmt.Println(name)
-		}
-	},
+	BashComplete: autocomplete.RecipeNameComplete,
 }

cli/recipe/upgrade.go

@@ -9,9 +9,10 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
@@ -36,15 +37,25 @@ update the relevant compose file tags on the local file system.
 Some image tags cannot be parsed because they do not follow some sort of
 semver-like convention. In this case, all possible tags will be listed and it
 is up to the end-user to decide.
+
+The command is interactive and will show a select input which allows you to
+make a seclection. Use the "?" key to see more help on navigating this
+interface.
+
+You may invoke this command in "wizard" mode and be prompted for input:
+
+    abra recipe upgrade
+
 `,
-	ArgsUsage: "<recipe>",
+	BashComplete: autocomplete.RecipeNameComplete,
+	ArgsUsage:    "<recipe>",
 	Flags: []cli.Flag{
 		internal.PatchFlag,
 		internal.MinorFlag,
 		internal.MajorFlag,
 	},
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c)
+		recipe := internal.ValidateRecipeWithPrompt(c)
 
 		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
 		if bumpType != 0 {
@@ -56,7 +67,7 @@ is up to the end-user to decide.
 
 		// check for versions file and load pinned versions
 		versionsPresent := false
-		recipeDir := path.Join(config.ABRA_DIR, "apps", recipe.Name)
+		recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
 		versionsPath := path.Join(recipeDir, "versions")
 		var servicePins = make(map[string]imgPin)
 		if _, err := os.Stat(versionsPath); err == nil {
@@ -92,11 +103,6 @@ is up to the end-user to decide.
 		}
 
 		for _, service := range recipe.Config.Services {
-			catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
 			img, err := reference.ParseNormalizedNamed(service.Image)
 			if err != nil {
 				logrus.Fatal(err)
@@ -107,25 +113,28 @@ is up to the end-user to decide.
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			logrus.Debugf("retrieved '%s' from remote registry for '%s'", regVersions, image)
+			logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
 
-			if strings.Contains(image, "library") {
-				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
-				// postgres:<tag>, i.e. images which do not have a username in the
-				// first position of the string
-				image = strings.Split(image, "/")[1]
-			}
-			semverLikeTag := true
-			if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
-				logrus.Debugf("'%s' not considered semver-like", img.(reference.NamedTagged).Tag())
-				semverLikeTag = false
+			image = recipePkg.StripTagMeta(image)
+
+			switch img.(type) {
+			case reference.NamedTagged:
+				if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+					logrus.Debugf("%s not considered semver-like", img.(reference.NamedTagged).Tag())
+				}
+			default:
+				logrus.Warnf("unable to read tag for image %s, is it missing? skipping upgrade for %s", image, service.Name)
+				continue
 			}
 
 			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
-			if err != nil && semverLikeTag {
-				logrus.Fatal(err)
+			if err != nil {
+				logrus.Warnf("unable to parse %s, error was: %s, skipping upgrade for %s", image, err.Error(), service.Name)
+				continue
 			}
-			logrus.Debugf("parsed '%s' for '%s'", tag, service.Name)
+
+			logrus.Debugf("parsed %s for %s", tag, service.Name)
+
 			var compatible []tagcmp.Tag
 			for _, regVersion := range regVersions {
 				other, err := tagcmp.Parse(regVersion.Name)
@@ -138,16 +147,21 @@ is up to the end-user to decide.
 				}
 			}
 
-			logrus.Debugf("detected potential upgradable tags '%s' for '%s'", compatible, service.Name)
+			logrus.Debugf("detected potential upgradable tags %s for %s", compatible, service.Name)
 
 			sort.Sort(tagcmp.ByTagDesc(compatible))
 
-			if len(compatible) == 0 && semverLikeTag {
-				logrus.Info(fmt.Sprintf("no new versions available for '%s', '%s' is the latest", image, tag))
+			if len(compatible) == 0 {
+				logrus.Info(fmt.Sprintf("no new versions available for %s, %s is the latest", image, tag))
 				continue // skip on to the next tag and don't update any compose files
 			}
 
-			var compatibleStrings []string
+			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			compatibleStrings := []string{"skip"}
 			for _, compat := range compatible {
 				skip := false
 				for _, catlVersion := range catlVersions {
@@ -160,7 +174,7 @@ is up to the end-user to decide.
 				}
 			}
 
-			logrus.Debugf("detected compatible upgradable tags '%s' for '%s'", compatibleStrings, service.Name)
+			logrus.Debugf("detected compatible upgradable tags %s for %s", compatibleStrings, service.Name)
 
 			var upgradeTag string
 			_, ok := servicePins[service.Name]
@@ -177,13 +191,13 @@ is up to the end-user to decide.
 						}
 					}
 					if contains {
-						logrus.Infof("Upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
+						logrus.Infof("upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
 					} else {
 						logrus.Infof("service %s, image %s pinned to %s, no compatible upgrade found", service.Name, servicePins[service.Name].image, pinnedTagString)
 						continue
 					}
 				} else {
-					logrus.Fatalf("Service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
+					logrus.Fatalf("service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
 					continue
 				}
 			} else {
@@ -200,16 +214,16 @@ is up to the end-user to decide.
 						}
 					}
 					if upgradeTag == "" {
-						logrus.Warnf("not upgrading from '%s' to '%s' for '%s', because the upgrade type is more serious than what user wants.", tag.String(), compatible[0].String(), image)
+						logrus.Warnf("not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants", tag.String(), compatible[0].String(), image)
 						continue
 					}
 				} else {
-					msg := fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
+					msg := fmt.Sprintf("upgrade to which tag? (service: %s, image: %s, tag: %s)", service.Name, image, tag)
 					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
 						tag := img.(reference.NamedTagged).Tag()
-						logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of '%s', listing all tags", tag))
+						logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of %s, listing all tags", tag))
 						msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
-						compatibleStrings = []string{}
+						compatibleStrings = []string{"skip"}
 						for _, regVersion := range regVersions {
 							compatibleStrings = append(compatibleStrings, regVersion.Name)
 						}
@@ -217,6 +231,8 @@ is up to the end-user to decide.
 
 					prompt := &survey.Select{
 						Message: msg,
+						Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
+						VimMode: true,
 						Options: compatibleStrings,
 					}
 					if err := survey.AskOne(prompt, &upgradeTag); err != nil {
@@ -224,10 +240,17 @@ is up to the end-user to decide.
 					}
 				}
 			}
-			if err := recipe.UpdateTag(image, upgradeTag); err != nil {
-				logrus.Fatal(err)
+			if upgradeTag != "skip" {
+				ok, err := recipe.UpdateTag(image, upgradeTag)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if ok {
+					logrus.Infof("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)
+				}
+			} else {
+				logrus.Warnf("not upgrading %s, skipping as requested", image)
 			}
-			logrus.Infof("tag upgraded from '%s' to '%s' for '%s'", tag.String(), upgradeTag, image)
 		}
 
 		return nil

cli/recipe/version.go

@@ -1,36 +1,49 @@
 package recipe
 
 import (
-	"coopcloud.tech/abra/cli/formatter"
+	"fmt"
+	"path"
+
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
 var recipeVersionCommand = &cli.Command{
-	Name:      "versions",
-	Usage:     "List recipe versions",
-	Aliases:   []string{"v"},
-	ArgsUsage: "<recipe>",
+	Name:         "versions",
+	Usage:        "List recipe versions",
+	Aliases:      []string{"v"},
+	ArgsUsage:    "<recipe>",
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
-		catalogue, err := catalogue.ReadRecipeCatalogue()
+		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		catalogue, err := recipePkg.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		recipeMeta, ok := catalogue[recipe.Name]
 		if !ok {
-			logrus.Fatalf("'%s' recipe doesn't exist?", recipe.Name)
+			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
 		}
 
 		tableCol := []string{"Version", "Service", "Image", "Tag", "Digest"}
 		table := formatter.CreateTable(tableCol)
 
-		for _, serviceVersion := range recipeMeta.Versions {
-			for tag, meta := range serviceVersion {
+		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
+			for tag, meta := range recipeMeta.Versions[i] {
 				for service, serviceMeta := range meta {
 					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag, serviceMeta.Digest})
 				}
@@ -38,7 +51,12 @@ var recipeVersionCommand = &cli.Command{
 		}
 
 		table.SetAutoMergeCells(true)
-		table.Render()
+
+		if table.NumLines() > 0 {
+			table.Render()
+		} else {
+			logrus.Fatalf("%s has no published versions?", recipe.Name)
+		}
 
 		return nil
 	},

cli/record/list.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"strconv"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/libdns/gandi"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -46,7 +46,7 @@ are listed. This zone must already be created on your provider account.
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
 		}
 
 		records, err := provider.GetRecords(c.Context, zone)
@@ -55,7 +55,7 @@ are listed. This zone must already be created on your provider account.
 		}
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := abraFormatter.CreateTable(tableCol)
+		table := formatter.CreateTable(tableCol)
 
 		for _, record := range records {
 			value := record.Value

cli/record/new.go

@@ -3,11 +3,11 @@ package record
 import (
 	"fmt"
 	"strconv"
-	"time"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/dns"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/libdns/gandi"
 	"github.com/libdns/libdns"
 	"github.com/sirupsen/logrus"
@@ -27,6 +27,7 @@ var RecordNewCommand = &cli.Command{
 		internal.DNSValueFlag,
 		internal.DNSTTLFlag,
 		internal.DNSPriorityFlag,
+		internal.AutoDNSRecordFlag,
 	},
 	Description: `
 This command creates a new domain name record for a specific zone.
@@ -38,9 +39,16 @@ Example:
 
     abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
 
+Typically, you need two records, an A record which points at the zone (@.) and
+a wildcard record for your apps (*.). Pass "--auto" to have Abra automatically
+set this up.
+
+    abra record new --auto foo.com -p gandi -v 192.168.178.44
+
 You may also invoke this command in "wizard" mode and be prompted for input
 
     abra record new
+
 `,
 	Action: func(c *cli.Context) error {
 		zone, err := internal.EnsureZoneArgument(c)
@@ -60,7 +68,26 @@ You may also invoke this command in "wizard" mode and be prompted for input
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
+		}
+
+		if internal.AutoDNSRecord {
+			ipv4, err := dns.EnsureIPv4(zone)
+			if err != nil {
+				logrus.Debugf("no ipv4 associated with %s, prompting for input", zone)
+				if err := internal.EnsureDNSValueFlag(c); err != nil {
+					logrus.Fatal(err)
+				}
+				ipv4 = internal.DNSValue
+			}
+
+			logrus.Infof("automatically configuring @./*. A records for %s for %s (--auto)", zone, ipv4)
+
+			if err := autoConfigure(c, &provider, zone, ipv4); err != nil {
+				logrus.Fatal(err)
+			}
+
+			return nil
 		}
 
 		if err := internal.EnsureDNSTypeFlag(c); err != nil {
@@ -75,11 +102,16 @@ You may also invoke this command in "wizard" mode and be prompted for input
 			logrus.Fatal(err)
 		}
 
+		ttl, err := dns.GetTTL(internal.DNSTTL)
+		if err != nil {
+			return err
+		}
+
 		record := libdns.Record{
 			Type:  internal.DNSType,
 			Name:  internal.DNSName,
 			Value: internal.DNSValue,
-			TTL:   time.Duration(internal.DNSTTL),
+			TTL:   ttl,
 		}
 
 		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
@@ -95,7 +127,7 @@ You may also invoke this command in "wizard" mode and be prompted for input
 			if existingRecord.Type == record.Type &&
 				existingRecord.Name == record.Name &&
 				existingRecord.Value == record.Value {
-				logrus.Fatal("provider library reports that this record already exists?")
+				logrus.Fatalf("%s record for %s already exists?", record.Type, zone)
 			}
 		}
 
@@ -104,6 +136,9 @@ You may also invoke this command in "wizard" mode and be prompted for input
 			zone,
 			[]libdns.Record{record},
 		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
 		if len(createdRecords) == 0 {
 			logrus.Fatal("provider library reports that no record was created?")
@@ -112,7 +147,7 @@ You may also invoke this command in "wizard" mode and be prompted for input
 		createdRecord := createdRecords[0]
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := abraFormatter.CreateTable(tableCol)
+		table := formatter.CreateTable(tableCol)
 
 		value := createdRecord.Value
 		if len(createdRecord.Value) > 30 {
@@ -134,3 +169,84 @@ You may also invoke this command in "wizard" mode and be prompted for input
 		return nil
 	},
 }
+
+func autoConfigure(c *cli.Context, provider *gandi.Provider, zone, ipv4 string) error {
+	ttl, err := dns.GetTTL(internal.DNSTTL)
+	if err != nil {
+		return err
+	}
+
+	atRecord := libdns.Record{
+		Type:  "A",
+		Name:  "@",
+		Value: ipv4,
+		TTL:   ttl,
+	}
+
+	wildcardRecord := libdns.Record{
+		Type:  "A",
+		Name:  "*",
+		Value: ipv4,
+		TTL:   ttl,
+	}
+
+	records := []libdns.Record{atRecord, wildcardRecord}
+
+	tableCol := []string{"type", "name", "value", "TTL", "priority"}
+	table := formatter.CreateTable(tableCol)
+
+	for _, record := range records {
+		existingRecords, err := provider.GetRecords(c.Context, zone)
+		if err != nil {
+			return err
+		}
+
+		discovered := false
+		for _, existingRecord := range existingRecords {
+			if existingRecord.Type == record.Type &&
+				existingRecord.Name == record.Name &&
+				existingRecord.Value == record.Value {
+				logrus.Warnf("%s record: %s %s for %s already exists?", record.Type, record.Name, record.Value, zone)
+				discovered = true
+			}
+		}
+
+		if discovered {
+			continue
+		}
+
+		createdRecords, err := provider.SetRecords(
+			c.Context,
+			zone,
+			[]libdns.Record{record},
+		)
+		if err != nil {
+			return err
+		}
+
+		if len(createdRecords) == 0 {
+			return fmt.Errorf("provider library reports that no record was created?")
+		}
+
+		createdRecord := createdRecords[0]
+
+		value := createdRecord.Value
+		if len(createdRecord.Value) > 30 {
+			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
+		}
+
+		table.Append([]string{
+			createdRecord.Type,
+			createdRecord.Name,
+			value,
+			createdRecord.TTL.String(),
+			strconv.Itoa(createdRecord.Priority),
+		})
+	}
+
+	if table.NumLines() > 0 {
+		table.Render()
+	}
+
+	return nil
+}

cli/record/record.go

@@ -7,7 +7,7 @@ import (
 // RecordCommand supports managing DNS entries.
 var RecordCommand = &cli.Command{
 	Name:      "record",
-	Usage:     "Manage domain name records via 3rd party providers",
+	Usage:     "Manage domain name records",
 	Aliases:   []string{"rc"},
 	ArgsUsage: "<record>",
 	Description: `

cli/record/remove.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"strconv"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/libdns/gandi"
 	"github.com/libdns/libdns"
@@ -59,7 +59,7 @@ You may also invoke this command in "wizard" mode and be prompted for input
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
 		}
 
 		if err := internal.EnsureDNSTypeFlag(c); err != nil {
@@ -88,7 +88,7 @@ You may also invoke this command in "wizard" mode and be prompted for input
 		}
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := abraFormatter.CreateTable(tableCol)
+		table := formatter.CreateTable(tableCol)
 
 		value := toDelete.Value
 		if len(toDelete.Value) > 30 {
@@ -105,17 +105,19 @@ You may also invoke this command in "wizard" mode and be prompted for input
 
 		table.Render()
 
-		response := false
-		prompt := &survey.Confirm{
-			Message: "continue with record deletion?",
-		}
+		if !internal.NoInput {
+			response := false
+			prompt := &survey.Confirm{
+				Message: "continue with record deletion?",
+			}
 
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return err
-		}
+			if err := survey.AskOne(prompt, &response); err != nil {
+				return err
+			}
 
-		if !response {
-			logrus.Fatal("exiting as requested")
+			if !response {
+				logrus.Fatal("exiting as requested")
+			}
 		}
 
 		_, err = provider.DeleteRecords(c.Context, zone, []libdns.Record{toDelete})

cli/server/add.go

@@ -1,23 +1,20 @@
 package server
 
 import (
-	"context"
 	"errors"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
 	"os/user"
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
-	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/server"
 	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
@@ -100,7 +97,7 @@ func cleanUp(domainName string) {
 	}
 
 	logrus.Warnf("cleaning up server directory for %s", domainName)
-	if err := os.RemoveAll(filepath.Join(config.ABRA_SERVER_FOLDER, domainName)); err != nil {
+	if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, domainName)); err != nil {
 		logrus.Fatal(err)
 	}
 }
@@ -119,7 +116,17 @@ func installDockerLocal(c *cli.Context) error {
 		logrus.Fatal("exiting as requested")
 	}
 
-	cmd := exec.Command("bash", "-c", "curl -s https://get.docker.com | bash")
+	for _, exe := range []string{"wget", "bash"} {
+		exists, err := ensureLocalExecutable(exe)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			return fmt.Errorf("%s missing, please install it", exe)
+		}
+	}
+
+	cmd := exec.Command("bash", "-c", "wget -O- https://get.docker.com | bash")
 	if err := internal.RunCmd(cmd); err != nil {
 		return err
 	}
@@ -138,15 +145,17 @@ func newLocalServer(c *cli.Context, domainName string) error {
 	}
 
 	if provision {
-		out, err := exec.Command("which", "docker").Output()
+		exists, err := ensureLocalExecutable("docker")
 		if err != nil {
 			return err
 		}
-		if string(out) == "" {
+
+		if !exists {
 			if err := installDockerLocal(c); err != nil {
 				return err
 			}
 		}
+
 		if err := initSwarmLocal(c, cl, domainName); err != nil {
 			if !strings.Contains(err.Error(), "proxy already exists") {
 				logrus.Fatal(err)
@@ -197,59 +206,127 @@ func newClient(c *cli.Context, domainName string) (*dockerClient.Client, error)
 }
 
 func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, domainName string) error {
-	result, err := sshCl.Exec("which docker")
-	if err != nil && string(result) != "" {
+	exists, err := ensureRemoteExecutable("docker", sshCl)
+	if err != nil {
 		return err
 	}
 
-	if string(result) == "" {
+	if !exists {
 		fmt.Println(fmt.Sprintf(dockerInstallMsg, domainName))
 
 		response := false
 		prompt := &survey.Confirm{
 			Message: fmt.Sprintf("attempt install docker on %s?", domainName),
 		}
+
 		if err := survey.AskOne(prompt, &response); err != nil {
 			return err
 		}
+
 		if !response {
 			logrus.Fatal("exiting as requested")
 		}
 
-		cmd := "curl -s https://get.docker.com | bash"
+		exes := []string{"wget", "bash"}
+		if askSudoPass {
+			exes = append(exes, "ssh-askpass")
+		}
+
+		for _, exe := range exes {
+			exists, err := ensureRemoteExecutable(exe, sshCl)
+			if err != nil {
+				return err
+			}
+			if !exists {
+				return fmt.Errorf("%s missing on remote, please install it", exe)
+			}
+		}
 
 		var sudoPass string
 		if askSudoPass {
+			cmd := "wget -O- https://get.docker.com | bash"
+
 			prompt := &survey.Password{
 				Message: "sudo password?",
 			}
+
 			if err := survey.AskOne(prompt, &sudoPass); err != nil {
 				return err
 			}
-			logrus.Debugf("running '%s' on %s now with sudo password", cmd, domainName)
+
+			logrus.Debugf("running %s on %s now with sudo password", cmd, domainName)
+
+			if sudoPass == "" {
+				return fmt.Errorf("missing sudo password but requested --ask-sudo-pass?")
+			}
+
+			logrus.Warn("installing docker, this could take some time...")
+
 			if err := ssh.RunSudoCmd(cmd, sudoPass, sshCl); err != nil {
+				fmt.Print(fmt.Sprintf(`
+Abra was unable to bootstrap Docker, see below for logs:
+
+
+%s
+
+If nothing works, you try running the Docker install script manually on your server:
+
+    wget -O- https://get.docker.com | bash
+
+`, string(err.Error())))
+				logrus.Fatal("Process exited with status 1")
+			}
+
+			logrus.Infof("docker is installed on %s", domainName)
+
+			remoteUser := sshCl.SSHClient.Conn.User()
+			logrus.Infof("adding %s to docker group", remoteUser)
+			permsCmd := fmt.Sprintf("sudo usermod -aG docker %s", remoteUser)
+			if err := ssh.RunSudoCmd(permsCmd, sudoPass, sshCl); err != nil {
 				return err
 			}
 		} else {
-			logrus.Debugf("running '%s' on %s now without sudo password", cmd, domainName)
-			if err := ssh.Exec(cmd, sshCl); err != nil {
-				return err
+			cmd := "wget -O- https://get.docker.com | bash"
+
+			logrus.Debugf("running %s on %s now without sudo password", cmd, domainName)
+
+			logrus.Warn("installing docker, this could take some time...")
+
+			if out, err := sshCl.Exec(cmd); err != nil {
+				fmt.Print(fmt.Sprintf(`
+Abra was unable to bootstrap Docker, see below for logs:
+
+
+%s
+
+This could be due to a number of things but one of the most common is that your
+server user account does not have sudo access, and if it does, you need to pass
+"--ask-sudo-pass" in order to supply Abra with your password.
+
+If nothing works, you try running the Docker install script manually on your server:
+
+    wget -O- https://get.docker.com | bash
+
+`, string(out)))
+				logrus.Fatal(err)
 			}
+
+			logrus.Infof("docker is installed on %s", domainName)
 		}
 	}
 
-	logrus.Infof("docker is installed on %s", domainName)
-
 	return nil
 }
 
 func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string) error {
 	initReq := swarm.InitRequest{ListenAddr: "0.0.0.0:2377"}
 	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
-		if !strings.Contains(err.Error(), "is already part of a swarm") {
+		if strings.Contains(err.Error(), "is already part of a swarm") ||
+			strings.Contains(err.Error(), "must specify a listening address") {
+			logrus.Infof("swarm mode already initialised on %s", domainName)
+		} else {
 			return err
 		}
-		logrus.Info("swarm mode already initialised on local server")
 	} else {
 		logrus.Infof("initialised swarm mode on local server")
 	}
@@ -268,42 +345,22 @@ func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string)
 }
 
 func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error {
-	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
-	freifunkDNS := "5.1.66.255:53"
-
-	resolver := &net.Resolver{
-		PreferGo: false,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Millisecond * time.Duration(10000),
-			}
-			return d.DialContext(ctx, "udp", freifunkDNS)
-		},
-	}
-
-	logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
-
-	ips, err := resolver.LookupIPAddr(c.Context, domainName)
+	ipv4, err := dns.EnsureIPv4(domainName)
 	if err != nil {
 		return err
 	}
 
-	if len(ips) == 0 {
-		return fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
-	}
-
-	ipv4 := ips[0].IP.To4().String()
-	logrus.Debugf("discovered the following ipv4 addr: %s", ipv4)
-
 	initReq := swarm.InitRequest{
 		ListenAddr:    "0.0.0.0:2377",
 		AdvertiseAddr: ipv4,
 	}
 	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
-		if !strings.Contains(err.Error(), "is already part of a swarm") {
+		if strings.Contains(err.Error(), "is already part of a swarm") ||
+			strings.Contains(err.Error(), "must specify a listening address") {
+			logrus.Infof("swarm mode already initialised on %s", domainName)
+		} else {
 			return err
 		}
-		logrus.Infof("swarm mode already initialised on %s", domainName)
 	} else {
 		logrus.Infof("initialised swarm mode on %s", domainName)
 	}
@@ -340,16 +397,8 @@ func deployTraefik(c *cli.Context, cl *dockerClient.Client, domainName string) e
 	internal.NewAppName = fmt.Sprintf("%s_%s", "traefik", config.SanitiseAppName(domainName))
 
 	appEnvPath := path.Join(config.ABRA_DIR, "servers", internal.Domain, fmt.Sprintf("%s.env", internal.NewAppName))
-	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
-		fmt.Println(fmt.Sprintf(`
-	You specified "--traefik/-t" and that means that Abra will now try to
-	automatically create a new Traefik app on %s.
-	`, internal.NewAppServer))
-
-		tableCol := []string{"recipe", "domain", "server", "name"}
-		table := abraFormatter.CreateTable(tableCol)
-		table.Append([]string{internal.RecipeName, internal.Domain, internal.NewAppServer, internal.NewAppName})
-
+	if _, err := os.Stat(appEnvPath); os.IsNotExist(err) {
+		logrus.Info(fmt.Sprintf("-t/--traefik specified, automatically deploying traefik to %s", internal.NewAppServer))
 		if err := internal.NewAction(c); err != nil {
 			logrus.Fatal(err)
 		}
@@ -429,12 +478,12 @@ You may omit flags to avoid performing this provisioning logic.
 	ArgsUsage: "<domain> [<user>] [<port>]",
 	Action: func(c *cli.Context) error {
 		if c.Args().Len() > 0 && local || !internal.ValidateSubCmdFlags(c) {
-			err := errors.New("cannot use '<domain>' and '--local' together")
+			err := errors.New("cannot use <domain> and --local together")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
 		if sshAuth != "password" && sshAuth != "identity-file" {
-			err := errors.New("--ssh-auth only accepts 'identity-file' or 'password'")
+			err := errors.New("--ssh-auth only accepts identity-file or password")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
@@ -508,3 +557,23 @@ You may omit flags to avoid performing this provisioning logic.
 		return nil
 	},
 }
+
+// ensureLocalExecutable ensures that an executable is present on the local machine
+func ensureLocalExecutable(exe string) (bool, error) {
+	out, err := exec.Command("which", exe).Output()
+	if err != nil {
+		return false, err
+	}
+
+	return string(out) != "", nil
+}
+
+// ensureRemoteExecutable ensures that an executable is present on a remote machine
+func ensureRemoteExecutable(exe string, sshCl *ssh.Client) (bool, error) {
+	out, err := sshCl.Exec(fmt.Sprintf("which %s", exe))
+	if err != nil && string(out) != "" {
+		return false, err
+	}
+
+	return string(out) != "", nil
+}

cli/server/list.go

@@ -3,9 +3,9 @@ package server
 import (
 	"strings"
 
-	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/docker/cli/cli/connhelper/ssh"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"

cli/server/new.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/libcapsul"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/hetznercloud/hcloud-go/hcloud"
@@ -43,13 +43,18 @@ func newHetznerCloudVPS(c *cli.Context) error {
 		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
 	}
 
+	sshKeyIDs := strings.Join(sshKeysRaw, "\n")
+	if sshKeyIDs == "" {
+		sshKeyIDs = "N/A (password auth)"
+	}
+
 	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
 	table := formatter.CreateTable(tableColumns)
 	table.Append([]string{
 		internal.HetznerCloudName,
 		internal.HetznerCloudType,
 		internal.HetznerCloudImage,
-		strings.Join(sshKeysRaw, "\n"),
+		sshKeyIDs,
 		internal.HetznerCloudLocation,
 	})
 	table.Render()
@@ -96,9 +101,21 @@ Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
 not list this server)! You will need to assign a domain name record ("abra
 record new") and add the server to your Abra configuration ("abra server add")
 to have a working server that you can deploy Co-op Cloud apps to.
+
+When setting up domain name records, you probably want to set up the following
+2 A records. This supports deploying apps to your root domain (e.g.
+example.com) and other apps on sub-domains (e.g. foo.example.com,
+bar.example.com).
+
+    @  1800 IN A %s
+    *  1800 IN A %s
+
+"abra record new --auto" can help you do this quickly if you use a supported
+DNS provider.
+
 	`,
 		internal.HetznerCloudName, ip, rootPassword,
-		ip,
+		ip, ip, ip,
 	))
 
 	return nil
@@ -169,6 +186,15 @@ Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
 not list this server)! You will need to assign a domain name record ("abra
 record new") and add the server to your Abra configuration ("abra server add")
 to have a working server that you can deploy Co-op Cloud apps to.
+
+When setting up domain name records, you probably want to set up the following
+2 A records. This supports deploying apps to your root domain (e.g.
+example.com) and other apps on sub-domains (e.g. foo.example.com,
+bar.example.com).
+
+    @  1800 IN A <your-capsul-ip>
+    *  1800 IN A <your-capsul-ip>
+
 	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
 
 	return nil
@@ -196,7 +222,6 @@ API tokens are read from the environment if specified, e.g.
 
 Where "$provider_TOKEN" is the expected env var format.
 `,
-	ArgsUsage: "<provider>",
 	Flags: []cli.Flag{
 		internal.ServerProviderFlag,
 

cli/server/remove.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
@@ -102,7 +102,7 @@ destroyed.
 var serverRemoveCommand = &cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
-	ArgsUsage: "<server>",
+	ArgsUsage: "[<server>]",
 	Usage:     "Remove a managed server",
 	Description: `
 This command removes a server from Abra management.
@@ -117,15 +117,36 @@ like tears in rain.
 `,
 	Flags: []cli.Flag{
 		rmServerFlag,
+		internal.ServerProviderFlag,
 
 		// Hetzner
 		internal.HetznerCloudNameFlag,
 		internal.HetznerCloudAPITokenFlag,
 	},
 	Action: func(c *cli.Context) error {
-		serverName, err := internal.ValidateServer(c)
-		if err != nil {
-			logrus.Fatal(err)
+		serverName := c.Args().Get(1)
+		if serverName != "" {
+			var err error
+			serverName, err = internal.ValidateServer(c)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if !rmServer {
+			logrus.Warn("did not pass -s/--server for actual server deletion, prompting")
+
+			response := false
+			prompt := &survey.Confirm{
+				Message: "prompt to actual server deletion?",
+			}
+			if err := survey.AskOne(prompt, &response); err != nil {
+				logrus.Fatal(err)
+			}
+			if response {
+				logrus.Info("setting -s/--server and attempting to remove actual server")
+				rmServer = true
+			}
 		}
 
 		if rmServer {
@@ -144,15 +165,17 @@ like tears in rain.
 
 		}
 
-		if err := client.DeleteContext(serverName); err != nil {
-			logrus.Fatal(err)
-		}
+		if serverName != "" {
+			if err := client.DeleteContext(serverName); err != nil {
+				logrus.Fatal(err)
+			}
 
-		if err := os.RemoveAll(filepath.Join(config.ABRA_SERVER_FOLDER, serverName)); err != nil {
-			logrus.Fatal(err)
-		}
+			if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, serverName)); err != nil {
+				logrus.Fatal(err)
+			}
 
-		logrus.Infof("server at '%s' has been lost in time, like tears in rain", serverName)
+			logrus.Infof("server at %s has been lost in time, like tears in rain", serverName)
+		}
 
 		return nil
 	},

cli/server/server.go

@@ -8,7 +8,7 @@ import (
 var ServerCommand = &cli.Command{
 	Name:    "server",
 	Aliases: []string{"s"},
-	Usage:   "Manage servers via 3rd party providers",
+	Usage:   "Manage servers",
 	Description: `
 These commands support creating, managing and removing servers using 3rd party
 integrations. 

cli/upgrade.go

@@ -1,23 +0,0 @@
-package cli
-
-import (
-	"os/exec"
-
-	"coopcloud.tech/abra/cli/internal"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-// UpgradeCommand upgrades abra in-place.
-var UpgradeCommand = &cli.Command{
-	Name:  "upgrade",
-	Usage: "Upgrade abra",
-	Action: func(c *cli.Context) error {
-		cmd := exec.Command("bash", "-c", "curl -s https://install.abra.coopcloud.tech | bash")
-		logrus.Debugf("attempting to run '%s'", cmd)
-		if err := internal.RunCmd(cmd); err != nil {
-			logrus.Fatal(err)
-		}
-		return nil
-	},
-}

cmd/abra/main.go

@@ -5,14 +5,13 @@ import (
 	"coopcloud.tech/abra/cli"
 )
 
-// Version is the current version of abra.
+// Version is the current version of Abra
 var Version string
 
-// Commit is the current commit of abra.
+// Commit is the current git commit of Abra
 var Commit string
 
 func main() {
-	// If not set in the ld-flags
 	if Version == "" {
 		Version = "dev"
 	}

go.mod

@@ -4,20 +4,20 @@ go 1.16
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
-	github.com/AlecAivazis/survey/v2 v2.3.1
+	github.com/AlecAivazis/survey/v2 v2.3.2
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.8+incompatible
+	github.com/docker/cli v20.10.12+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/docker v20.10.12+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/go-git/go-git/v5 v5.4.2
-	github.com/hetznercloud/hcloud-go v1.32.0
-	github.com/moby/sys/signal v0.5.0
+	github.com/hetznercloud/hcloud-go v1.33.1
+	github.com/moby/sys/signal v0.6.0
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.8.3
+	github.com/schollz/progressbar/v3 v3.8.5
 	github.com/schultz-is/passgen v1.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
@@ -27,14 +27,16 @@ require (
 require (
 	coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
 	github.com/Microsoft/hcsshim v0.8.21 // indirect
+	github.com/buger/goterm v1.0.3
 	github.com/containerd/containerd v1.5.5 // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/gliderlabs/ssh v0.2.2
+	github.com/gliderlabs/ssh v0.3.3
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351
+	github.com/hashicorp/go-retryablehttp v0.7.0
+	github.com/kevinburke/ssh_config v1.1.0
 	github.com/libdns/gandi v1.0.2
 	github.com/libdns/libdns v0.2.1
 	github.com/moby/sys/mount v0.2.0 // indirect
@@ -42,6 +44,6 @@ require (
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
-	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0
+	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
 )

go.sum

@@ -26,8 +26,8 @@ coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e/go.mod h1:HEQ9pSJRsD
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52 h1:cyFFOl0tKe+dVHt8saejG8xoff33eQiHxFCVzRpPUjM=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/AlecAivazis/survey/v2 v2.3.1 h1:lzkuHA60pER7L4eYL8qQJor4bUWlJe4V0gqAT19tdOA=
-github.com/AlecAivazis/survey/v2 v2.3.1/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg=
+github.com/AlecAivazis/survey/v2 v2.3.2 h1:TqTB+aDDCLYhf9/bD2TwSO8u8jDSmMUd2SUVO4gCnU8=
+github.com/AlecAivazis/survey/v2 v2.3.2/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg=
 github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4 h1:aYUdiI42a4fWfPoUr25XlaJrFEICv24+o/gWhqYS/jk=
 github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -88,8 +88,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
@@ -109,6 +110,8 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
+github.com/buger/goterm v1.0.3 h1:7V/HeAQHrzPk/U4BvyH2g9u+xbUW9nr4yRPyG59W4fM=
+github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0 h1:s7+5BfS4WFJoVF9pnB8kBk03S7pZXRdKamnV0FOl5Sc=
@@ -259,14 +262,14 @@ github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/cli v20.10.8+incompatible h1:/zO/6y9IOpcehE49yMRTV9ea0nBpb8OeqSskXLNfH1E=
-github.com/docker/cli v20.10.8+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.12+incompatible h1:lZlz0uzG+GH+c0plStMUdF/qk3ppmgnswpR5EbqzVGA=
+github.com/docker/cli v20.10.12+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.8+incompatible h1:RVqD337BgQicVCzYrrlhLDWhq6OAD2PJDUg2LsEUvKM=
-github.com/docker/docker v20.10.8+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.12+incompatible h1:CEeNmFM0QZIsJCZKMkZx0ZcahTiewkrgiwfYD+dfl1U=
+github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
@@ -302,7 +305,6 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -315,8 +317,9 @@ github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui72
 github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/gliderlabs/ssh v0.3.3 h1:mBQ8NiOgDkINJrZtoizkC3nDNYgSaWtxyem6S2XHBtA=
+github.com/gliderlabs/ssh v0.3.3/go.mod h1:ZSS+CUoKHDrqVakTfTWUlKSr9MtMFkC4UvtQKD7O914=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
@@ -443,14 +446,20 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
+github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hetznercloud/hcloud-go v1.32.0 h1:7zyN2V7hMlhm3HZdxOarmOtvzKvkcYKjM0hcwYMQZz0=
-github.com/hetznercloud/hcloud-go v1.32.0/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
+github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ=
+github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
 github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ=
 github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -489,8 +498,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kevinburke/ssh_config v1.1.0 h1:pH/t1WS9NzT8go394IqZeJTMHVm6Cr6ZJ6AQ+mdNo/o=
+github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -563,8 +573,8 @@ github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7s
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/sys/signal v0.5.0 h1:MzpEFrMxugDynb1gkTIThU1O3wEmrAkOY+G9dHcHnCc=
-github.com/moby/sys/signal v0.5.0/go.mod h1:JwObcMnOrUy2VTP5swPKWwywH0Mbgk8Y5qua9iwtIRM=
+github.com/moby/sys/signal v0.6.0 h1:aDpY94H8VlhTGa9sNYUFCFsMZIUh5wm0B6XkIoJj/iY=
+github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
@@ -687,8 +697,8 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8=
-github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
+github.com/schollz/progressbar/v3 v3.8.5 h1:VcmmNRO+eFN3B0m5dta6FXYXY+MEJmXdWoIS+jjssQM=
+github.com/schollz/progressbar/v3 v3.8.5/go.mod h1:ewO25kD7ZlaJFTvMeOItkOZa8kXu1UvFs379htE8HMQ=
 github.com/schultz-is/passgen v1.0.1 h1:wUINzqW1Xmmy3yREHR6YTj+83VlFYjj2DIDMHzIi5TQ=
 github.com/schultz-is/passgen v1.0.1/go.mod h1:NnqzT2aSfvyheNQvBtlLUa0YlPFLDj60Jw2DZVwqiJk=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
@@ -820,8 +830,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -887,8 +898,9 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -966,26 +978,30 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
-golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/app/app.go

@@ -23,7 +23,7 @@ func Get(appName string) (config.App, error) {
 		return config.App{}, err
 	}
 
-	logrus.Debugf("retrieved '%s' for '%s'", app, appName)
+	logrus.Debugf("retrieved %s for %s", app, appName)
 
 	return app, nil
 }
@@ -57,9 +57,9 @@ func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App)
 	deployed := len(services) > 0
 
 	if deployed {
-		logrus.Debugf("detected '%s' as deployed versions of '%s'", appSpec, app.Name)
+		logrus.Debugf("detected %s as deployed versions of %s", appSpec, app.Name)
 	} else {
-		logrus.Debugf("detected '%s' as not deployed", app.Name)
+		logrus.Debugf("detected %s as not deployed", app.Name)
 	}
 
 	return appSpec, len(services) > 0, nil
@@ -71,15 +71,15 @@ func ParseVersionLabel(label string) (string, string) {
 	idx := strings.LastIndex(label, "-")
 	version := label[:idx]
 	digest := label[idx+1:]
-	logrus.Debugf("parsed '%s' as version from '%s'", version, label)
-	logrus.Debugf("parsed '%s' as digest from '%s'", digest, label)
+	logrus.Debugf("parsed %s as version from %s", version, label)
+	logrus.Debugf("parsed %s as digest from %s", digest, label)
 	return version, digest
 }
 
-// ParseVersionName parses a $STACK_NAME_$SERVICE_NAME service label.
+// ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label.
 func ParseServiceName(label string) string {
 	idx := strings.LastIndex(label, "_")
 	serviceName := label[idx+1:]
-	logrus.Debugf("parsed '%s' as service name from '%s'", serviceName, label)
+	logrus.Debugf("parsed %s as service name from %s", serviceName, label)
 	return serviceName
 }

pkg/autocomplete/autocomplete.go

@@ -0,0 +1,42 @@
+package autocomplete
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// AppNameComplete copletes app names
+func AppNameComplete(c *cli.Context) {
+	appNames, err := config.GetAppNames()
+	if err != nil {
+		logrus.Warn(err)
+	}
+
+	if c.NArg() > 0 {
+		return
+	}
+
+	for _, a := range appNames {
+		fmt.Println(a)
+	}
+}
+
+// RecipeNameComplete completes recipe names
+func RecipeNameComplete(c *cli.Context) {
+	catl, err := recipe.ReadRecipeCatalogue()
+	if err != nil {
+		logrus.Warn(err)
+	}
+
+	if c.NArg() > 0 {
+		return
+	}
+
+	for name := range catl {
+		fmt.Println(name)
+	}
+}

pkg/catalogue/catalogue.go

@@ -1,500 +0,0 @@
-// Package catalogue provides ways of interacting with recipe catalogues which
-// are JSON data structures which contain meta information about recipes (e.g.
-// what versions of the Nextcloud recipe are available?).
-package catalogue
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/web"
-	"github.com/docker/distribution/reference"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/sirupsen/logrus"
-)
-
-// RecipeCatalogueURL is the only current recipe catalogue available.
-const RecipeCatalogueURL = "https://apps.coopcloud.tech"
-
-// ReposMetadataURL is the recipe repository metadata
-const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
-
-// image represents a recipe container image.
-type image struct {
-	Image  string `json:"image"`
-	Rating string `json:"rating"`
-	Source string `json:"source"`
-	URL    string `json:"url"`
-}
-
-// features represent what top-level features a recipe supports (e.g. does this
-// recipe support backups?).
-type features struct {
-	Backups     string `json:"backups"`
-	Email       string `json:"email"`
-	Healthcheck string `json:"healthcheck"`
-	Image       image  `json:"image"`
-	Status      int    `json:"status"`
-	Tests       string `json:"tests"`
-}
-
-// tag represents a git tag.
-type tag = string
-
-// service represents a service within a recipe.
-type service = string
-
-// ServiceMeta represents meta info associated with a service.
-type ServiceMeta struct {
-	Digest string `json:"digest"`
-	Image  string `json:"image"`
-	Tag    string `json:"tag"`
-}
-
-// RecipeVersions are the versions associated with a recipe.
-type RecipeVersions []map[tag]map[service]ServiceMeta
-
-// RecipeMeta represents metadata for a recipe in the abra catalogue.
-type RecipeMeta struct {
-	Category      string         `json:"category"`
-	DefaultBranch string         `json:"default_branch"`
-	Description   string         `json:"description"`
-	Features      features       `json:"features"`
-	Icon          string         `json:"icon"`
-	Name          string         `json:"name"`
-	Repository    string         `json:"repository"`
-	Versions      RecipeVersions `json:"versions"`
-	Website       string         `json:"website"`
-}
-
-// LatestVersion returns the latest version of a recipe.
-func (r RecipeMeta) LatestVersion() string {
-	var version string
-
-	// apps.json versions are sorted so the last key is latest
-	latest := r.Versions[len(r.Versions)-1]
-
-	for tag := range latest {
-		version = tag
-	}
-
-	logrus.Debugf("choosing '%s' as latest version of '%s'", version, r.Name)
-
-	return version
-}
-
-// Name represents a recipe name.
-type Name = string
-
-// RecipeCatalogue represents the entire recipe catalogue.
-type RecipeCatalogue map[Name]RecipeMeta
-
-// Flatten converts AppCatalogue to slice
-func (r RecipeCatalogue) Flatten() []RecipeMeta {
-	recipes := make([]RecipeMeta, 0, len(r))
-
-	for name := range r {
-		recipes = append(recipes, r[name])
-	}
-
-	return recipes
-}
-
-// ByRecipeName sorts recipes by name.
-type ByRecipeName []RecipeMeta
-
-func (r ByRecipeName) Len() int      { return len(r) }
-func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
-func (r ByRecipeName) Less(i, j int) bool {
-	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
-}
-
-// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
-// is up to date.
-func recipeCatalogueFSIsLatest() (bool, error) {
-	httpClient := &http.Client{Timeout: web.Timeout}
-	res, err := httpClient.Head(RecipeCatalogueURL)
-	if err != nil {
-		return false, err
-	}
-
-	lastModified := res.Header["Last-Modified"][0]
-	parsed, err := time.Parse(time.RFC1123, lastModified)
-	if err != nil {
-		return false, err
-	}
-
-	info, err := os.Stat(config.APPS_JSON)
-	if err != nil {
-		if os.IsNotExist(err) {
-			logrus.Debugf("no recipe catalogue found in file system cache")
-			return false, nil
-		}
-		return false, err
-	}
-
-	localModifiedTime := info.ModTime().Unix()
-	remoteModifiedTime := parsed.Unix()
-
-	if localModifiedTime < remoteModifiedTime {
-		logrus.Debug("file system cached recipe catalogue is out-of-date")
-		return false, nil
-	}
-
-	logrus.Debug("file system cached recipe catalogue is up-to-date")
-
-	return true, nil
-}
-
-// ReadRecipeCatalogue reads the recipe catalogue.
-func ReadRecipeCatalogue() (RecipeCatalogue, error) {
-	recipes := make(RecipeCatalogue)
-
-	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
-	if err != nil {
-		return nil, err
-	}
-
-	if !recipeFSIsLatest {
-		logrus.Debugf("reading recipe catalogue from web to get latest")
-		if err := readRecipeCatalogueWeb(&recipes); err != nil {
-			return nil, err
-		}
-		return recipes, nil
-	}
-
-	logrus.Debugf("reading recipe catalogue from file system cache to get latest")
-	if err := readRecipeCatalogueFS(&recipes); err != nil {
-		return nil, err
-	}
-
-	return recipes, nil
-}
-
-// readRecipeCatalogueFS reads the catalogue from the file system.
-func readRecipeCatalogueFS(target interface{}) error {
-	recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
-	if err != nil {
-		return err
-	}
-
-	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
-		return err
-	}
-
-	logrus.Debugf("read recipe catalogue from file system cache in '%s'", config.APPS_JSON)
-
-	return nil
-}
-
-// readRecipeCatalogueWeb reads the catalogue from the web.
-func readRecipeCatalogueWeb(target interface{}) error {
-	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
-		return err
-	}
-
-	recipesJSON, err := json.MarshalIndent(target, "", "    ")
-	if err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
-		return err
-	}
-
-	logrus.Debugf("read recipe catalogue from web at '%s'", RecipeCatalogueURL)
-
-	return nil
-}
-
-// VersionsOfService lists the version of a service.
-func VersionsOfService(recipe, serviceName string) ([]string, error) {
-	catalogue, err := ReadRecipeCatalogue()
-	if err != nil {
-		return nil, err
-	}
-
-	rec, ok := catalogue[recipe]
-	if !ok {
-		return nil, fmt.Errorf("recipe '%s' does not exist?", recipe)
-	}
-
-	versions := []string{}
-	alreadySeen := make(map[string]bool)
-	for _, serviceVersion := range rec.Versions {
-		for tag := range serviceVersion {
-			if _, ok := alreadySeen[tag]; !ok {
-				alreadySeen[tag] = true
-				versions = append(versions, tag)
-			}
-		}
-	}
-
-	logrus.Debugf("detected versions '%s' for '%s'", strings.Join(versions, ", "), recipe)
-
-	return versions, nil
-}
-
-// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
-func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
-	catl, err := ReadRecipeCatalogue()
-	if err != nil {
-		return RecipeMeta{}, err
-	}
-
-	recipeMeta, ok := catl[recipeName]
-	if !ok {
-		err := fmt.Errorf("recipe '%s' does not exist?", recipeName)
-		return RecipeMeta{}, err
-	}
-
-	if err := recipe.EnsureExists(recipeName); err != nil {
-		return RecipeMeta{}, err
-	}
-
-	logrus.Debugf("recipe metadata retrieved for '%s'", recipeName)
-
-	return recipeMeta, nil
-}
-
-// RepoMeta is a single recipe repo metadata.
-type RepoMeta struct {
-	ID                        int `json:"id"`
-	Owner                     Owner
-	Name                      string      `json:"name"`
-	FullName                  string      `json:"full_name"`
-	Description               string      `json:"description"`
-	Empty                     bool        `json:"empty"`
-	Private                   bool        `json:"private"`
-	Fork                      bool        `json:"fork"`
-	Template                  bool        `json:"template"`
-	Parent                    interface{} `json:"parent"`
-	Mirror                    bool        `json:"mirror"`
-	Size                      int         `json:"size"`
-	HTMLURL                   string      `json:"html_url"`
-	SSHURL                    string      `json:"ssh_url"`
-	CloneURL                  string      `json:"clone_url"`
-	OriginalURL               string      `json:"original_url"`
-	Website                   string      `json:"website"`
-	StarsCount                int         `json:"stars_count"`
-	ForksCount                int         `json:"forks_count"`
-	WatchersCount             int         `json:"watchers_count"`
-	OpenIssuesCount           int         `json:"open_issues_count"`
-	OpenPRCount               int         `json:"open_pr_counter"`
-	ReleaseCounter            int         `json:"release_counter"`
-	DefaultBranch             string      `json:"default_branch"`
-	Archived                  bool        `json:"archived"`
-	CreatedAt                 string      `json:"created_at"`
-	UpdatedAt                 string      `json:"updated_at"`
-	Permissions               Permissions
-	HasIssues                 bool `json:"has_issues"`
-	InternalTracker           InternalTracker
-	HasWiki                   bool   `json:"has_wiki"`
-	HasPullRequests           bool   `json:"has_pull_requests"`
-	HasProjects               bool   `json:"has_projects"`
-	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
-	AllowMergeCommits         bool   `json:"allow_merge_commits"`
-	AllowRebase               bool   `json:"allow_rebase"`
-	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
-	AllowSquashMerge          bool   `json:"allow_squash_merge"`
-	AvatarURL                 string `json:"avatar_url"`
-	Internal                  bool   `json:"internal"`
-	MirrorInterval            string `json:"mirror_interval"`
-}
-
-// Owner is the repo organisation owner metadata.
-type Owner struct {
-	ID         int    `json:"id"`
-	Login      string `json:"login"`
-	FullName   string `json:"full_name"`
-	Email      string `json:"email"`
-	AvatarURL  string `json:"avatar_url"`
-	Language   string `json:"language"`
-	IsAdmin    bool   `json:"is_admin"`
-	LastLogin  string `json:"last_login"`
-	Created    string `json:"created"`
-	Restricted bool   `json:"restricted"`
-	Username   string `json:"username"`
-}
-
-// Permissions is perms metadata for a repo.
-type Permissions struct {
-	Admin bool `json:"admin"`
-	Push  bool `json:"push"`
-	Pull  bool `json:"pull"`
-}
-
-// InternalTracker is issue tracker metadata for a repo.
-type InternalTracker struct {
-	EnableTimeTracker                bool `json:"enable_time_tracker"`
-	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
-	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
-}
-
-// RepoCatalogue represents all the recipe repo metadata.
-type RepoCatalogue map[string]RepoMeta
-
-// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
-func ReadReposMetadata() (RepoCatalogue, error) {
-	reposMeta := make(RepoCatalogue)
-
-	pageIdx := 1
-	for {
-		var reposList []RepoMeta
-
-		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
-
-		logrus.Debugf("fetching repo metadata from '%s'", pagedURL)
-
-		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
-			return reposMeta, err
-		}
-
-		if len(reposList) == 0 {
-			break
-		}
-
-		for idx, repo := range reposList {
-			reposMeta[repo.Name] = reposList[idx]
-		}
-
-		pageIdx++
-	}
-
-	return reposMeta, nil
-}
-
-// GetRecipeVersions retrieves all recipe versions.
-func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
-	versions := RecipeVersions{}
-
-	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
-
-	logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
-
-	repo, err := git.PlainOpen(recipeDir)
-	if err != nil {
-		return versions, err
-	}
-
-	worktree, err := repo.Worktree()
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	gitTags, err := repo.Tags()
-	if err != nil {
-		return versions, err
-	}
-
-	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
-		tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
-
-		logrus.Debugf("processing '%s' for '%s'", tag, recipeName)
-
-		checkOutOpts := &git.CheckoutOptions{
-			Create: false,
-			Force:  true,
-			Branch: plumbing.ReferenceName(ref.Name()),
-		}
-		if err := worktree.Checkout(checkOutOpts); err != nil {
-			logrus.Debugf("failed to check out '%s' in '%s'", tag, recipeDir)
-			return err
-		}
-
-		logrus.Debugf("successfully checked out '%s' in '%s'", ref.Name(), recipeDir)
-
-		recipe, err := recipe.Get(recipeName)
-		if err != nil {
-			return err
-		}
-
-		versionMeta := make(map[string]ServiceMeta)
-		for _, service := range recipe.Config.Services {
-
-			img, err := reference.ParseNormalizedNamed(service.Image)
-			if err != nil {
-				return err
-			}
-
-			path := reference.Path(img)
-			if strings.Contains(path, "library") {
-				path = strings.Split(path, "/")[1]
-			}
-
-			digest, err := client.GetTagDigest(img)
-			if err != nil {
-				return err
-			}
-
-			versionMeta[service.Name] = ServiceMeta{
-				Digest: digest,
-				Image:  path,
-				Tag:    img.(reference.NamedTagged).Tag(),
-			}
-
-			logrus.Debugf("collecting digest: '%s', image: '%s', tag: '%s'", digest, path, tag)
-		}
-
-		versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
-
-		return nil
-	}); err != nil {
-		return versions, err
-	}
-
-	branch := "master"
-	if _, err := repo.Branch("master"); err != nil {
-		if _, err := repo.Branch("main"); err != nil {
-			logrus.Debugf("failed to select branch in '%s'", recipeDir)
-			logrus.Fatal(err)
-		}
-		branch = "main"
-	}
-
-	refName := fmt.Sprintf("refs/heads/%s", branch)
-	checkOutOpts := &git.CheckoutOptions{
-		Create: false,
-		Force:  true,
-		Branch: plumbing.ReferenceName(refName),
-	}
-	if err := worktree.Checkout(checkOutOpts); err != nil {
-		logrus.Debugf("failed to check out '%s' in '%s'", branch, recipeDir)
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("switched back to '%s' in '%s'", branch, recipeDir)
-	logrus.Debugf("collected '%s' for '%s'", versions, recipeName)
-
-	return versions, nil
-}
-
-// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
-func GetRecipeCatalogueVersions(recipeName string) ([]string, error) {
-	var versions []string
-
-	catl, err := ReadRecipeCatalogue()
-	if err != nil {
-		return versions, err
-	}
-
-	if recipeMeta, exists := catl[recipeName]; exists {
-		for _, versionMeta := range recipeMeta.Versions {
-			for tag := range versionMeta {
-				versions = append(versions, tag)
-			}
-		}
-	}
-
-	return versions, nil
-}

pkg/client/client.go

@@ -55,7 +55,7 @@ func New(contextName string) (*client.Client, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("created client for '%s'", contextName)
+	logrus.Debugf("created client for %s", contextName)
 
 	return cl, nil
 }

pkg/client/context.go

@@ -26,7 +26,7 @@ func CreateContext(contextName string, user string, port string) error {
 	if err := createContext(contextName, host); err != nil {
 		return err
 	}
-	logrus.Debugf("created the '%s' context", contextName)
+	logrus.Debugf("created the %s context", contextName)
 	return nil
 }
 
@@ -72,8 +72,6 @@ func DeleteContext(name string) error {
 		return err
 	}
 
-	// remove any context that might be loaded
-	// TODO: Check if the context we are removing is the active one rather than doing it all the time
 	cfg := dConfig.LoadDefaultConfigFile(nil)
 	cfg.CurrentContext = ""
 	if err := cfg.Save(); err != nil {

pkg/client/registry.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -9,6 +10,9 @@ import (
 
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/client"
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/sirupsen/logrus"
 )
 
 type RawTag struct {
@@ -31,16 +35,30 @@ func GetRegistryTags(image string) (RawTags, error) {
 	return tags, nil
 }
 
+func basicAuth(username, password string) string {
+	auth := username + ":" + password
+	return base64.StdEncoding.EncodeToString([]byte(auth))
+}
+
 // getRegv2Token retrieves a registry v2 authentication token.
-func getRegv2Token(image reference.Named) (string, error) {
+func getRegv2Token(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
 	img := reference.Path(image)
-	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
-	req, err := http.NewRequest("GET", authTokenURL, nil)
+	tokenURL := "https://auth.docker.io/token"
+	values := fmt.Sprintf("service=registry.docker.io&scope=repository:%s:pull", img)
+
+	fullURL := fmt.Sprintf("%s?%s", tokenURL, values)
+	req, err := retryablehttp.NewRequest("GET", fullURL, nil)
 	if err != nil {
 		return "", err
 	}
 
-	client := &http.Client{Timeout: web.Timeout}
+	if registryUsername != "" && registryPassword != "" {
+		logrus.Debugf("using registry log in credentials for token request")
+		auth := basicAuth(registryUsername, registryPassword)
+		req.Header.Add("Authorization", fmt.Sprintf("Basic %s", auth))
+	}
+
+	client := web.NewHTTPRetryClient()
 	res, err := client.Do(req)
 	if err != nil {
 		return "", err
@@ -60,9 +78,10 @@ func getRegv2Token(image reference.Named) (string, error) {
 	}
 
 	tokenRes := struct {
-		Token  string
-		Expiry string
-		Issued string
+		AccessToken string `json:"access_token"`
+		Expiry      int    `json:"expires_in"`
+		Issued      string `json:"issued_at"`
+		Token       string `json:"token"`
 	}{}
 
 	if err := json.Unmarshal(body, &tokenRes); err != nil {
@@ -73,21 +92,25 @@ func getRegv2Token(image reference.Named) (string, error) {
 }
 
 // GetTagDigest retrieves an image digest from a v2 registry
-func GetTagDigest(image reference.Named) (string, error) {
+func GetTagDigest(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
 	img := reference.Path(image)
 	tag := image.(reference.NamedTagged).Tag()
 	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
 
-	req, err := http.NewRequest("GET", manifestURL, nil)
+	req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
 	if err != nil {
 		return "", err
 	}
 
-	token, err := getRegv2Token(image)
+	token, err := getRegv2Token(cl, image, registryUsername, registryPassword)
 	if err != nil {
 		return "", err
 	}
 
+	if token == "" {
+		return "", fmt.Errorf("unable to retrieve registry token?")
+	}
+
 	req.Header = http.Header{
 		"Accept": []string{
 			"application/vnd.docker.distribution.manifest.v2+json",
@@ -96,7 +119,7 @@ func GetTagDigest(image reference.Named) (string, error) {
 		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
 	}
 
-	client := &http.Client{Timeout: web.Timeout}
+	client := web.NewHTTPRetryClient()
 	res, err := client.Do(req)
 	if err != nil {
 		return "", err
@@ -163,7 +186,7 @@ func GetTagDigest(image reference.Named) (string, error) {
 	}
 
 	if digest == "" {
-		return "", fmt.Errorf("Unable to retrieve amd64 digest for '%s'", image)
+		return "", fmt.Errorf("Unable to retrieve amd64 digest for %s", image)
 	}
 
 	return digest, nil

pkg/compose/compose.go

@@ -16,26 +16,26 @@ import (
 )
 
 // UpdateTag updates an image tag in-place on file system local compose files.
-func UpdateTag(pattern, image, tag, recipeName string) error {
+func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 	composeFiles, err := filepath.Glob(pattern)
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	logrus.Debugf("considering '%s' config(s) for tag update", strings.Join(composeFiles, ", "))
+	logrus.Debugf("considering %s config(s) for tag update", strings.Join(composeFiles, ", "))
 
 	for _, composeFile := range composeFiles {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
-		envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
+		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
 		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
-			return err
+			return false, err
 		}
 
 		compose, err := loader.LoadComposefile(opts, sampleEnv)
 		if err != nil {
-			return err
+			return false, err
 		}
 
 		for _, service := range compose.Services {
@@ -45,40 +45,42 @@ func UpdateTag(pattern, image, tag, recipeName string) error {
 
 			img, _ := reference.ParseNormalizedNamed(service.Image)
 			if err != nil {
-				return err
+				return false, err
+			}
+
+			var composeTag string
+			switch img.(type) {
+			case reference.NamedTagged:
+				composeTag = img.(reference.NamedTagged).Tag()
+			default:
+				// unable to parse, typically image missing tag
+				return false, nil
 			}
 
 			composeImage := reference.Path(img)
-			if strings.Contains(composeImage, "library") {
-				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
-				// postgres:<tag>, i.e. images which do not have a username in the
-				// first position of the string
-				composeImage = strings.Split(composeImage, "/")[1]
-			}
-			composeTag := img.(reference.NamedTagged).Tag()
 
-			logrus.Debugf("parsed '%s' from '%s'", composeTag, service.Image)
+			logrus.Debugf("parsed %s from %s", composeTag, service.Image)
 
 			if image == composeImage {
 				bytes, err := ioutil.ReadFile(composeFile)
 				if err != nil {
-					return err
+					return false, err
 				}
 
 				old := fmt.Sprintf("%s:%s", composeImage, composeTag)
 				new := fmt.Sprintf("%s:%s", composeImage, tag)
 				replacedBytes := strings.Replace(string(bytes), old, new, -1)
 
-				logrus.Debugf("updating '%s' to '%s' in '%s'", old, new, compose.Filename)
+				logrus.Debugf("updating %s to %s in %s", old, new, compose.Filename)
 
-				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
-					return err
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0764); err != nil {
+					return true, err
 				}
 			}
 		}
 	}
 
-	return nil
+	return false, nil
 }
 
 // UpdateLabel updates a label in-place on file system local compose files.
@@ -88,12 +90,12 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		return err
 	}
 
-	logrus.Debugf("considering '%s' config(s) for label update", strings.Join(composeFiles, ", "))
+	logrus.Debugf("considering %s config(s) for label update", strings.Join(composeFiles, ", "))
 
 	for _, composeFile := range composeFiles {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
-		envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
+		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
 		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
 			return err
@@ -130,19 +132,25 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 				old := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", value)
 				replacedBytes := strings.Replace(string(bytes), old, label, -1)
 
+				if old == label {
+					logrus.Warnf("%s is already set, nothing to do?", label)
+					return nil
+				}
+
 				logrus.Debugf("updating %s to %s in %s", old, label, compose.Filename)
 
-				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0764); err != nil {
 					return err
 				}
+
+				logrus.Infof("synced label %s to service %s", label, serviceName)
 			}
 		}
 
 		if !discovered {
-			logrus.Warn("no existing label found, cannot continue...")
-			logrus.Fatalf("add '%s' manually, automagic insertion not supported yet", label)
+			logrus.Warn("no existing label found, automagic insertion not supported yet")
+			logrus.Fatalf("add '- \"%s\"' manually to the 'app' service in %s", label, composeFile)
 		}
-
 	}
 
 	return nil

pkg/config/app.go

@@ -1,15 +1,14 @@
 package config
 
 import (
-	"errors"
 	"fmt"
+	"html/template"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
 
-	"coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/pkg/ssh"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
@@ -44,13 +43,17 @@ type App struct {
 	Path   string
 }
 
-// StackName gets what the docker safe stack name is for the app
+// StackName gets what the docker safe stack name is for the app. This should
+// not not shown to the user, use a.Name for that. Give the output of this
+// command to Docker only.
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
 	}
+
 	stackName := SanitiseAppName(a.Name)
 	a.Env["STACK_NAME"] = stackName
+
 	return stackName
 }
 
@@ -98,14 +101,14 @@ func (a ByName) Less(i, j int) bool {
 func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	env, err := ReadEnv(appFile.Path)
 	if err != nil {
-		return App{}, fmt.Errorf("env file for '%s' couldn't be read: %s", name, err.Error())
+		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
 
-	logrus.Debugf("read env '%s' from '%s'", env, appFile.Path)
+	logrus.Debugf("read env %s from %s", env, appFile.Path)
 
 	app, err := newApp(env, name, appFile)
 	if err != nil {
-		return App{}, fmt.Errorf("env file for '%s' has issues: %s", name, err.Error())
+		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
 	}
 
 	return app, nil
@@ -113,17 +116,17 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 
 // newApp creates new App object
 func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
-	// Checking for type as it is required - apps wont work without it
 	domain := env["DOMAIN"]
-	apptype, ok := env["TYPE"]
-	if !ok {
-		return App{}, errors.New("missing TYPE variable")
+
+	appType, exists := env["TYPE"]
+	if !exists {
+		return App{}, fmt.Errorf("%s is missing the TYPE env var", name)
 	}
 
 	return App{
 		Name:   name,
 		Domain: domain,
-		Type:   apptype,
+		Type:   appType,
 		Env:    env,
 		Server: appFile.Server,
 		Path:   appFile.Path,
@@ -137,28 +140,24 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 		if servers[0] == "" {
 			// Empty servers flag, one string will always be passed
 			var err error
-			servers, err = getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+			servers, err = GetAllFoldersInDirectory(SERVERS_DIR)
 			if err != nil {
 				return nil, err
 			}
 		}
 	}
 
-	logrus.Debugf("collecting metadata from '%v' servers: '%s'", len(servers), strings.Join(servers, ", "))
-
-	if err := EnsureHostKeysAllServers(servers...); err != nil {
-		return nil, err
-	}
+	logrus.Debugf("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", "))
 
 	for _, server := range servers {
-		serverDir := path.Join(ABRA_SERVER_FOLDER, server)
+		serverDir := path.Join(SERVERS_DIR, server)
 		files, err := getAllFilesInDirectory(serverDir)
 		if err != nil {
 			return nil, err
 		}
 		for _, file := range files {
 			appName := strings.TrimSuffix(file.Name(), ".env")
-			appFilePath := path.Join(ABRA_SERVER_FOLDER, server, file.Name())
+			appFilePath := path.Join(SERVERS_DIR, server, file.Name())
 			appFiles[appName] = AppFile{
 				Path:   appFilePath,
 				Server: server,
@@ -174,7 +173,7 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 func GetApp(apps AppFiles, name AppName) (App, error) {
 	appFile, exists := apps[name]
 	if !exists {
-		return App{}, fmt.Errorf("cannot find app with name '%s'", name)
+		return App{}, fmt.Errorf("cannot find app with name %s", name)
 	}
 
 	app, err := readAppEnvFile(appFile, name)
@@ -254,27 +253,39 @@ func GetAppNames() ([]string, error) {
 }
 
 // TemplateAppEnvSample copies the example env file for the app into the users env files
-func TemplateAppEnvSample(appType, appName, server, domain, recipe string) error {
-	envSamplePath := path.Join(ABRA_DIR, "apps", appType, ".env.sample")
+func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
+	envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample")
 	envSample, err := ioutil.ReadFile(envSamplePath)
 	if err != nil {
 		return err
 	}
 
 	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
-	if _, err := os.Stat(appEnvPath); err == nil {
+	if _, err := os.Stat(appEnvPath); os.IsExist(err) {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
-	envSample = []byte(strings.Replace(string(envSample), fmt.Sprintf("%s.example.com", recipe), domain, -1))
-	envSample = []byte(strings.Replace(string(envSample), "example.com", domain, -1))
-
-	err = ioutil.WriteFile(appEnvPath, envSample, 0755)
+	err = ioutil.WriteFile(appEnvPath, envSample, 0664)
 	if err != nil {
 		return err
 	}
 
-	logrus.Debugf("copied '%s' to '%s'", envSamplePath, appEnvPath)
+	file, err := os.OpenFile(appEnvPath, os.O_RDWR, 0664)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	tpl, err := template.ParseFiles(appEnvPath)
+	if err != nil {
+		return err
+	}
+
+	if err := tpl.Execute(file, struct{ Name string }{recipeName}); err != nil {
+		return err
+	}
+
+	logrus.Debugf("copied & templated %s to %s", envSamplePath, appEnvPath)
 
 	return nil
 }
@@ -320,9 +331,6 @@ func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
 			if version, ok := service.Spec.Labels[labelKey]; ok {
 				result["version"] = version
 			} else {
-				//FIXME: we only need to check containers with the version label not
-				//       every single container and then skip when we see no label perf gains
-				//       to be had here
 				continue
 			}
 
@@ -330,7 +338,7 @@ func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
 		}
 	}
 
-	logrus.Debugf("retrieved app statuses: '%s'", statuses)
+	logrus.Debugf("retrieved app statuses: %s", statuses)
 
 	return statuses, nil
 }
@@ -342,20 +350,20 @@ func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
 
 	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
 		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
-		path := fmt.Sprintf("%s/%s/compose.yml", APPS_DIR, recipe)
+		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
 		composeFiles = append(composeFiles, path)
 		return composeFiles, nil
 	}
 
 	composeFileEnvVar := appEnv["COMPOSE_FILE"]
 	envVars := strings.Split(composeFileEnvVar, ":")
-	logrus.Debugf("COMPOSE_FILE detected ('%s'), loading '%s'", composeFileEnvVar, strings.Join(envVars, ", "))
+	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
 	for _, file := range strings.Split(composeFileEnvVar, ":") {
-		path := fmt.Sprintf("%s/%s/%s", APPS_DIR, recipe, file)
+		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
 		composeFiles = append(composeFiles, path)
 	}
 
-	logrus.Debugf("retrieved '%s' configs for '%s'", strings.Join(composeFiles, ", "), recipe)
+	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
 
 	return composeFiles, nil
 }
@@ -369,19 +377,7 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp
 		return &composetypes.Config{}, err
 	}
 
-	logrus.Debugf("retrieved '%s' for '%s'", compose.Filename, recipe)
+	logrus.Debugf("retrieved %s for %s", compose.Filename, recipe)
 
 	return compose, nil
 }
-
-// EnsureHostKeysAllServers ensures all configured servers have server SSH host keys validated
-func EnsureHostKeysAllServers(servers ...string) error {
-	for _, serverName := range servers {
-		logrus.Debugf("ensuring server SSH host key available for %s", serverName)
-		if err := ssh.EnsureHostKey(serverName); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/config/app_test.go

@@ -26,7 +26,6 @@ func TestReadAppEnvFile(t *testing.T) {
 }
 
 func TestGetApp(t *testing.T) {
-	// TODO: Test failures as well as successes
 	app, err := GetApp(expectedAppFiles, appName)
 	if err != nil {
 		t.Fatal(err)

pkg/config/env.go

@@ -15,21 +15,23 @@ import (
 )
 
 var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
-var ABRA_SERVER_FOLDER = path.Join(ABRA_DIR, "servers")
-var APPS_JSON = path.Join(ABRA_DIR, "apps.json")
-var APPS_DIR = path.Join(ABRA_DIR, "apps")
+var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
+var RECIPES_DIR = path.Join(ABRA_DIR, "apps")
+var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
+var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json")
 var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
+var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
 // GetServers retrieves all servers.
 func GetServers() ([]string, error) {
 	var servers []string
 
-	servers, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+	servers, err := GetAllFoldersInDirectory(SERVERS_DIR)
 	if err != nil {
 		return servers, err
 	}
 
-	logrus.Debugf("retrieved '%v' servers: '%s'", len(servers), servers)
+	logrus.Debugf("retrieved %v servers: %s", len(servers), servers)
 
 	return servers, nil
 }
@@ -43,20 +45,20 @@ func ReadEnv(filePath string) (AppEnv, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("read '%s' from '%s'", envFile, filePath)
+	logrus.Debugf("read %s from %s", envFile, filePath)
 
 	return envFile, nil
 }
 
 // ReadServerNames retrieves all server names.
 func ReadServerNames() ([]string, error) {
-	serverNames, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
 
 	if err != nil {
 		return nil, err
 	}
 
-	logrus.Debugf("read '%s' from '%s'", strings.Join(serverNames, ","), ABRA_SERVER_FOLDER)
+	logrus.Debugf("read %s from %s", strings.Join(serverNames, ","), SERVERS_DIR)
 
 	return serverNames, nil
 }
@@ -80,7 +82,7 @@ func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
 
 		realPath, err := filepath.EvalSymlinks(filePath)
 		if err != nil {
-			logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
+			logrus.Warningf("broken symlink in your abra config folders: %s", filePath)
 		} else {
 			realFile, err := os.Stat(realPath)
 			if err != nil {
@@ -95,8 +97,8 @@ func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
 	return realFiles, nil
 }
 
-// getAllFoldersInDirectory returns both folder and symlink paths
-func getAllFoldersInDirectory(directory string) ([]string, error) {
+// GetAllFoldersInDirectory returns both folder and symlink paths
+func GetAllFoldersInDirectory(directory string) ([]string, error) {
 	var folders []string
 
 	files, err := ioutil.ReadDir(directory)
@@ -104,7 +106,7 @@ func getAllFoldersInDirectory(directory string) ([]string, error) {
 		return nil, err
 	}
 	if len(files) == 0 {
-		return nil, fmt.Errorf("directory is empty: '%s'", directory)
+		return nil, fmt.Errorf("directory is empty: %s", directory)
 	}
 
 	for _, file := range files {
@@ -113,7 +115,7 @@ func getAllFoldersInDirectory(directory string) ([]string, error) {
 			filePath := path.Join(directory, file.Name())
 			realDir, err := filepath.EvalSymlinks(filePath)
 			if err != nil {
-				logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
+				logrus.Warningf("broken symlink in your abra config folders: %s", filePath)
 			} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
 				// path is a directory
 				folders = append(folders, file.Name())
@@ -124,17 +126,6 @@ func getAllFoldersInDirectory(directory string) ([]string, error) {
 	return folders, nil
 }
 
-// EnsureAbraDirExists checks for the abra config folder and throws error if not
-func EnsureAbraDirExists() error {
-	if _, err := os.Stat(ABRA_DIR); os.IsNotExist(err) {
-		logrus.Debugf("'%s' does not exist, creating it", ABRA_DIR)
-		if err := os.Mkdir(ABRA_DIR, 0777); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // ReadAbraShEnvVars reads env vars from an abra.sh recipe file.
 func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 	envVars := make(map[string]string)
@@ -161,7 +152,7 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 		}
 	}
 
-	logrus.Debugf("read '%s' from '%s'", envVars, abraSh)
+	logrus.Debugf("read %s from %s", envVars, abraSh)
 
 	return envVars, nil
 }

pkg/config/env_test.go

@@ -44,7 +44,7 @@ var expectedAppFiles = map[string]AppFile{
 // var expectedServerNames = []string{"evil.corp"}
 
 func TestGetAllFoldersInDirectory(t *testing.T) {
-	folders, err := getAllFoldersInDirectory(testFolder)
+	folders, err := GetAllFoldersInDirectory(testFolder)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/container/container.go

@@ -0,0 +1,70 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// GetContainer retrieves a container. If prompt is true and the retrievd count
+// of containers does not match 1, then a prompt is presented to let the user
+// choose. A count of 0 is handled gracefully.
+func GetContainer(c context.Context, cl *client.Client, filters filters.Args, prompt bool) (types.Container, error) {
+	containerOpts := types.ContainerListOptions{Filters: filters}
+	containers, err := cl.ContainerList(c, containerOpts)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	if len(containers) == 0 {
+		filter := filters.Get("name")[0]
+		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
+	}
+
+	if len(containers) != 1 {
+		var containersRaw []string
+		for _, container := range containers {
+			containerName := strings.Join(container.Names, " ")
+			trimmed := strings.TrimPrefix(containerName, "/")
+			created := formatter.HumanDuration(container.Created)
+			containersRaw = append(containersRaw, fmt.Sprintf("%s (created %v)", trimmed, created))
+		}
+
+		if !prompt {
+			err := fmt.Errorf("expected 1 container but found %v: %s", len(containers), strings.Join(containersRaw, " "))
+			return types.Container{}, err
+		}
+
+		logrus.Warnf("ambiguous container list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which container are you looking for?",
+			Options: containersRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return types.Container{}, err
+		}
+
+		chosenContainer := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, container := range containers {
+			containerName := strings.TrimSpace(strings.Join(container.Names, " "))
+			trimmed := strings.TrimPrefix(containerName, "/")
+			if trimmed == chosenContainer {
+				return container, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen container")
+	}
+
+	return containers[0], nil
+}

pkg/dns/common.go

@@ -1,8 +1,11 @@
 package dns
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"os"
+	"time"
 
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
@@ -26,3 +29,77 @@ func NewToken(provider, providerTokenEnvVar string) (string, error) {
 
 	return token, nil
 }
+
+// EnsureIPv4 ensures that an ipv4 address is set for a domain name
+func EnsureIPv4(domainName string) (string, error) {
+	var ipv4 string
+
+	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
+	freifunkDNS := "5.1.66.255:53"
+
+	resolver := &net.Resolver{
+		PreferGo: false,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: time.Millisecond * time.Duration(10000),
+			}
+			return d.DialContext(ctx, "udp", freifunkDNS)
+		},
+	}
+
+	logrus.Debugf("created DNS resolver via %s", freifunkDNS)
+
+	ctx := context.Background()
+	ips, err := resolver.LookupIPAddr(ctx, domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if len(ips) == 0 {
+		return ipv4, fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
+	}
+
+	ipv4 = ips[0].IP.To4().String()
+	logrus.Debugf("discovered the following ipv4 addr: %s", ipv4)
+
+	return ipv4, nil
+}
+
+// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address
+func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
+	var ipv4 string
+
+	domainIPv4, err := EnsureIPv4(domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if domainIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", domainName)
+	}
+
+	serverIPv4, err := EnsureIPv4(server)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if serverIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", server)
+	}
+
+	if domainIPv4 != serverIPv4 {
+		err := "app domain %s (%s) does not appear to resolve to app server %s (%s)?"
+		return ipv4, fmt.Errorf(err, domainName, domainIPv4, server, serverIPv4)
+	}
+
+	return ipv4, nil
+}
+
+// GetTTL parses a ttl string into a duration
+func GetTTL(ttl string) (time.Duration, error) {
+	val, err := time.ParseDuration(ttl)
+	if err != nil {
+		return val, err
+	}
+	return val, nil
+}

pkg/formatter/formatter.go

@@ -1,12 +1,10 @@
 package formatter
 
 import (
-	"fmt"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/docker/cli/cli/command/formatter"
 	"github.com/docker/go-units"
 	"github.com/olekukonko/tablewriter"
 	"github.com/schollz/progressbar/v3"
@@ -16,10 +14,6 @@ func ShortenID(str string) string {
 	return str[:12]
 }
 
-func Truncate(str string) string {
-	return fmt.Sprintf(`"%s"`, formatter.Ellipsis(str, 19))
-}
-
 func SmallSHA(hash string) string {
 	return hash[:8]
 }
@@ -39,6 +33,7 @@ func HumanDuration(timestamp int64) string {
 // CreateTable prepares a table layout for output.
 func CreateTable(columns []string) *tablewriter.Table {
 	table := tablewriter.NewWriter(os.Stdout)
+	table.SetAutoWrapText(false)
 	table.SetHeader(columns)
 	return table
 }

pkg/git/branch.go

@@ -0,0 +1,35 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// GetCurrentBranch retrieves the current branch of a repository
+func GetCurrentBranch(repository *git.Repository) (string, error) {
+	branchRefs, err := repository.Branches()
+	if err != nil {
+		return "", err
+	}
+
+	headRef, err := repository.Head()
+	if err != nil {
+		return "", err
+	}
+
+	var currentBranchName string
+	err = branchRefs.ForEach(func(branchRef *plumbing.Reference) error {
+		if branchRef.Hash() == headRef.Hash() {
+			currentBranchName = branchRef.Name().String()
+
+			return nil
+		}
+
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return currentBranchName, nil
+}

pkg/git/clone.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
@@ -15,10 +14,10 @@ import (
 // Clone runs a git clone which accounts for different default branches.
 func Clone(dir, url string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		logrus.Debugf("'%s' does not exist, attempting to git clone from '%s'", dir, url)
+		logrus.Debugf("%s does not exist, attempting to git clone from %s", dir, url)
 		_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
 		if err != nil {
-			logrus.Debugf("cloning '%s' default branch failed, attempting from main branch", url)
+			logrus.Debugf("cloning %s default branch failed, attempting from main branch", url)
 			_, err := git.PlainClone(dir, false, &git.CloneOptions{
 				URL:           url,
 				Tags:          git.AllTags,
@@ -32,67 +31,10 @@ func Clone(dir, url string) error {
 				return err
 			}
 		}
-		logrus.Debugf("'%s' has been git cloned successfully", dir)
+		logrus.Debugf("%s has been git cloned successfully", dir)
 	} else {
-		logrus.Debugf("'%s' already exists, doing nothing", dir)
+		logrus.Debugf("%s already exists", dir)
 	}
 
 	return nil
 }
-
-// EnsureUpToDate ensures that a git repo on disk has the latest changes (git-fetch).
-func EnsureUpToDate(dir string) error {
-	repo, err := git.PlainOpen(dir)
-	if err != nil {
-		return err
-	}
-
-	branch := "master"
-	if _, err := repo.Branch("master"); err != nil {
-		if _, err := repo.Branch("main"); err != nil {
-			logrus.Debugf("failed to select branch in '%s'", dir)
-			return err
-		}
-		branch = "main"
-	}
-
-	logrus.Debugf("choosing '%s' as main git branch in '%s'", branch, dir)
-
-	worktree, err := repo.Worktree()
-	if err != nil {
-		return err
-	}
-
-	refName := fmt.Sprintf("refs/heads/%s", branch)
-	checkOutOpts := &git.CheckoutOptions{
-		Create: false,
-		Force:  true,
-		Branch: plumbing.ReferenceName(refName),
-	}
-	if err := worktree.Checkout(checkOutOpts); err != nil {
-		logrus.Debugf("failed to check out '%s' in '%s'", refName, dir)
-		return err
-	}
-
-	logrus.Debugf("successfully checked out '%s' in '%s'", branch, dir)
-
-	remote, err := repo.Remote("origin")
-	if err != nil {
-		return err
-	}
-
-	fetchOpts := &git.FetchOptions{
-		RemoteName: "origin",
-		RefSpecs:   []config.RefSpec{"refs/heads/*:refs/remotes/origin/*"},
-		Force:      true,
-	}
-	if err := remote.Fetch(fetchOpts); err != nil {
-		if !strings.Contains(err.Error(), "already up-to-date") {
-			return err
-		}
-	}
-
-	logrus.Debugf("successfully fetched all changes in '%s'", dir)
-
-	return nil
-}

pkg/git/commit.go

@@ -0,0 +1,56 @@
+package git
+
+import (
+	"fmt"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// Commit runs a git commit
+func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
+	if commitMessage == "" {
+		return fmt.Errorf("no commit message specified?")
+	}
+
+	commitRepo, err := git.PlainOpen(repoPath)
+	if err != nil {
+		return err
+	}
+
+	commitWorktree, err := commitRepo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return err
+	}
+
+	if len(patterns) > 0 {
+		commitWorktree.Excludes = append(patterns, commitWorktree.Excludes...)
+	}
+
+	if !dryRun {
+		err = commitWorktree.AddGlob(glob)
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("staged %s for commit", glob)
+	} else {
+		logrus.Debugf("dry run: did not stage %s for commit", glob)
+	}
+
+	if !dryRun {
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
+		if err != nil {
+			return err
+		}
+		logrus.Debug("git changes commited")
+	} else {
+		logrus.Debug("dry run: no changes commited")
+	}
+
+	return nil
+}

pkg/git/common.go

@@ -0,0 +1,14 @@
+package git
+
+import (
+	"fmt"
+	"os"
+)
+
+// EnsureGitRepo ensures a git repo .git folder exists
+func EnsureGitRepo(repoPath string) error {
+	if _, err := os.Stat(repoPath); os.IsNotExist(err) {
+		return fmt.Errorf("no .git directory in %s?", repoPath)
+	}
+	return nil
+}

pkg/git/init.go

@@ -0,0 +1,38 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	gitPkg "github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// Init inits a new repo and commits all the stuff if you want
+func Init(repoPath string, commit bool) error {
+	if _, err := gitPkg.PlainInit(repoPath, false); err != nil {
+		logrus.Fatal(err)
+	}
+	logrus.Debugf("initialised new git repo in %s", repoPath)
+
+	if commit {
+		commitRepo, err := git.PlainOpen(repoPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		commitWorktree, err := commitRepo.Worktree()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := commitWorktree.AddWithOptions(&git.AddOptions{All: true}); err != nil {
+			return err
+		}
+
+		if _, err = commitWorktree.Commit("init", &git.CommitOptions{}); err != nil {
+			return err
+		}
+		logrus.Debugf("init committed all files for new git repo in %s", repoPath)
+	}
+
+	return nil
+}

pkg/git/push.go

@@ -0,0 +1,43 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/sirupsen/logrus"
+)
+
+// Push pushes the latest changes & optionally tags to the default remote
+func Push(repoDir string, remote string, tags bool, dryRun bool) error {
+	if dryRun {
+		logrus.Debugf("dry run: no git changes pushed in %s", repoDir)
+		return nil
+	}
+
+	commitRepo, err := git.PlainOpen(repoDir)
+	if err != nil {
+		return err
+	}
+
+	opts := &git.PushOptions{}
+	if remote != "" {
+		opts.RemoteName = remote
+	}
+
+	if err := commitRepo.Push(opts); err != nil {
+		return err
+	}
+
+	logrus.Debugf("git changes pushed")
+
+	if tags {
+		opts.RefSpecs = append(opts.RefSpecs, config.RefSpec("+refs/tags/*:refs/tags/*"))
+
+		if err := commitRepo.Push(opts); err != nil {
+			return err
+		}
+
+		logrus.Debugf("git tags pushed")
+	}
+
+	return nil
+}

pkg/git/read.go

@@ -1,17 +1,24 @@
 package git
 
 import (
+	"io/ioutil"
+	"os"
+	"os/user"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/go-git/go-git/v5"
+	gitConfigPkg "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
 	"github.com/sirupsen/logrus"
 )
 
 // GetRecipeHead retrieves latest HEAD metadata.
 func GetRecipeHead(recipeName string) (*plumbing.Reference, error) {
-	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
 
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
@@ -27,10 +34,8 @@ func GetRecipeHead(recipeName string) (*plumbing.Reference, error) {
 }
 
 // IsClean checks if a repo has unstaged changes
-func IsClean(recipeName string) (bool, error) {
-	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
-
-	repo, err := git.PlainOpen(recipeDir)
+func IsClean(repoPath string) (bool, error) {
+	repo, err := git.PlainOpen(repoPath)
 	if err != nil {
 		return false, err
 	}
@@ -40,16 +45,143 @@ func IsClean(recipeName string) (bool, error) {
 		return false, err
 	}
 
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return false, err
+	}
+
+	if len(patterns) > 0 {
+		worktree.Excludes = append(patterns, worktree.Excludes...)
+	}
+
 	status, err := worktree.Status()
 	if err != nil {
 		return false, err
 	}
 
 	if status.String() != "" {
-		logrus.Debugf("discovered git status for %s repository: %s", recipeName, status.String())
+		logrus.Debugf("discovered git status in %s: %s", repoPath, status.String())
 	} else {
-		logrus.Debugf("discovered clean git status for %s repository", recipeName)
+		logrus.Debugf("discovered clean git status in %s", repoPath)
 	}
 
 	return status.IsClean(), nil
 }
+
+// GetExcludesFiles reads the exlude files from a global gitignore
+func GetExcludesFiles() ([]gitignore.Pattern, error) {
+	var err error
+	var patterns []gitignore.Pattern
+
+	cfg, err := parseGitConfig()
+	if err != nil {
+		return patterns, err
+	}
+
+	excludesfile := getExcludesFile(cfg)
+	patterns, err = parseExcludesFile(excludesfile)
+	if err != nil {
+		return patterns, err
+	}
+
+	return patterns, nil
+}
+
+func parseGitConfig() (*gitConfigPkg.Config, error) {
+	cfg := gitConfigPkg.NewConfig()
+
+	usr, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	globalGitConfig := filepath.Join(usr.HomeDir, ".gitconfig")
+	if _, err := os.Stat(globalGitConfig); err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no %s exists, not reading any global gitignore config", globalGitConfig)
+			return cfg, nil
+		}
+		return cfg, err
+	}
+
+	b, err := ioutil.ReadFile(globalGitConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cfg.Unmarshal(b); err != nil {
+		return nil, err
+	}
+
+	return cfg, err
+}
+
+func getExcludesFile(cfg *gitConfigPkg.Config) string {
+	for _, sec := range cfg.Raw.Sections {
+		if sec.Name == "core" {
+			for _, opt := range sec.Options {
+				if opt.Key == "excludesfile" {
+					return opt.Value
+				}
+			}
+		}
+	}
+
+	return "~/.gitignore"
+}
+
+func parseExcludesFile(excludesfile string) ([]gitignore.Pattern, error) {
+	var ps []gitignore.Pattern
+
+	excludesfile, err := expandTilde(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := os.Stat(excludesfile); err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no %s exists, skipping reading gitignore paths", excludesfile)
+			return ps, nil
+		}
+		return ps, err
+	}
+
+	data, err := ioutil.ReadFile(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	var pathsRaw []string
+	for _, s := range strings.Split(string(data), "\n") {
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			pathsRaw = append(pathsRaw, s)
+			ps = append(ps, gitignore.ParsePattern(s, nil))
+		}
+	}
+
+	logrus.Debugf("read global ignore paths: %s", strings.Join(pathsRaw, " "))
+
+	return ps, nil
+}
+
+func expandTilde(path string) (string, error) {
+	if !strings.HasPrefix(path, "~") {
+		return path, nil
+	}
+
+	var paths []string
+	u, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	for _, p := range strings.Split(path, string(filepath.Separator)) {
+		if p == "~" {
+			paths = append(paths, u.HomeDir)
+		} else {
+			paths = append(paths, p)
+		}
+	}
+
+	return filepath.Join(paths...), nil
+}

pkg/git/remote.go

@@ -0,0 +1,28 @@
+package git
+
+import (
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/sirupsen/logrus"
+)
+
+// CreateRemote creates a new git remote in a repository
+func CreateRemote(repo *git.Repository, name, url string, dryRun bool) error {
+	if dryRun {
+		logrus.Debugf("dry run: remote %s (%s) not created", name, url)
+		return nil
+	}
+
+	if _, err := repo.CreateRemote(&config.RemoteConfig{
+		Name: name,
+		URLs: []string{url},
+	}); err != nil {
+		if !strings.Contains(err.Error(), "remote already exists") {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/integration/integration.go

@@ -0,0 +1,12 @@
+package integration
+
+import (
+	"os"
+	"testing"
+)
+
+func skipIfNotIntegration(t *testing.T) {
+	if os.Getenv("ABRA_INTEGRATION") == "" {
+		t.Skip("missing 'ABRA_INTEGRATION', not running integration tests")
+	}
+}

pkg/lint/recipe.go

@@ -0,0 +1,338 @@
+package lint
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+)
+
+var Warn = "warn"
+var Critical = "critical"
+
+type LintFunction func(recipe.Recipe) (bool, error)
+
+type LintRule struct {
+	Ref          string
+	Level        string
+	Description  string
+	HowToResolve string
+	Function     LintFunction
+}
+
+var LintRules = map[string][]LintRule{
+	"warn": {
+		{
+			Ref:          "R001",
+			Level:        "warn",
+			Description:  "compose config has expected version",
+			HowToResolve: "ensure 'version: \"3.8\"' in compose configs",
+			Function:     LintComposeVersion,
+		},
+		{
+			Ref:          "R002",
+			Level:        "warn",
+			Description:  "healthcheck enabled for all services",
+			HowToResolve: "wire up healthchecks",
+			Function:     LintHealthchecks,
+		},
+		{
+			Ref:          "R003",
+			Level:        "warn",
+			Description:  "all images use a tag",
+			HowToResolve: "use a tag for all images",
+			Function:     LintAllImagesTagged,
+		},
+		{
+			Ref:          "R004",
+			Level:        "warn",
+			Description:  "no unstable tags",
+			HowToResolve: "tag all images with stable tags",
+			Function:     LintNoUnstableTags,
+		},
+		{
+			Ref:          "R005",
+			Level:        "warn",
+			Description:  "tags use semver-like format",
+			HowToResolve: "use semver-like tags",
+			Function:     LintSemverLikeTags,
+		},
+		{
+			Ref:          "R006",
+			Level:        "warn",
+			Description:  "has published catalogue version",
+			HowToResolve: "publish a recipe version to the catalogue",
+			Function:     LintHasPublishedVersion,
+		},
+		{
+			Ref:          "R007",
+			Level:        "warn",
+			Description:  "README.md metadata filled in",
+			HowToResolve: "fill out all the metadata",
+			Function:     LintMetadataFilledIn,
+		},
+	},
+	"error": {
+		{
+			Ref:          "R008",
+			Level:        "error",
+			Description:  ".env.sample provided",
+			HowToResolve: "create an example .env.sample",
+			Function:     LintEnvConfigPresent,
+		},
+		{
+			Ref:          "R009",
+			Level:        "error",
+			Description:  "one service named 'app'",
+			HowToResolve: "name a servce 'app'",
+			Function:     LintAppService,
+		},
+		{
+			Ref:          "R010",
+			Level:        "error",
+			Description:  "traefik routing enabled",
+			HowToResolve: "include \"traefik.enable=true\" deploy label",
+			Function:     LintTraefikEnabled,
+		},
+		{
+			Ref:          "R011",
+			Level:        "error",
+			Description:  "all services have images",
+			HowToResolve: "ensure \"image: ...\" set on all services",
+			Function:     LintImagePresent,
+		},
+		{
+			Ref:          "R012",
+			Level:        "error",
+			Description:  "config version are vendored",
+			HowToResolve: "vendor config versions in an abra.sh",
+			Function:     LintAbraShVendors,
+		},
+		{
+			Ref:          "R013",
+			Level:        "error",
+			Description:  "git.coopcloud.tech repo exists",
+			HowToResolve: "upload your recipe to git.coopcloud.tech/coop-cloud/...",
+			Function:     LintHasRecipeRepo,
+		},
+	},
+}
+
+func LintForErrors(recipe recipe.Recipe) error {
+	logrus.Debugf("linting for critical errors in %s configs", recipe.Name)
+
+	for level := range LintRules {
+		if level != "error" {
+			continue
+		}
+		for _, rule := range LintRules[level] {
+			ok, err := rule.Function(recipe)
+			if err != nil {
+				return err
+			}
+			if !ok {
+				return fmt.Errorf("lint error in %s configs: \"%s\" failed lint checks (%s)", recipe.Name, rule.Description, rule.Ref)
+			}
+		}
+	}
+
+	logrus.Debugf("linting successful, %s is well configured", recipe.Name)
+
+	return nil
+}
+
+func LintComposeVersion(recipe recipe.Recipe) (bool, error) {
+	if recipe.Config.Version == "3.8" {
+		return true, nil
+	}
+
+	return true, nil
+}
+
+func LintEnvConfigPresent(recipe recipe.Recipe) (bool, error) {
+	envSample := fmt.Sprintf("%s/%s/.env.sample", config.RECIPES_DIR, recipe.Name)
+	if _, err := os.Stat(envSample); !os.IsNotExist(err) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func LintAppService(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if service.Name == "app" {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func LintTraefikEnabled(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		for label := range service.Deploy.Labels {
+			if label == "traefik.enable" {
+				if service.Deploy.Labels[label] == "true" {
+					return true, nil
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
+func LintHealthchecks(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if service.HealthCheck == nil {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return false, err
+		}
+		if reference.IsNameOnly(img) {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return false, err
+		}
+
+		var tag string
+		switch img.(type) {
+		case reference.NamedTagged:
+			tag = img.(reference.NamedTagged).Tag()
+		case reference.Named:
+			return false, nil
+		}
+
+		if tag == "latest" {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintSemverLikeTags(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return false, err
+		}
+
+		var tag string
+		switch img.(type) {
+		case reference.NamedTagged:
+			tag = img.(reference.NamedTagged).Tag()
+		case reference.Named:
+			return false, nil
+		}
+
+		if !tagcmp.IsParsable(tag) {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintImagePresent(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if service.Image == "" {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
+	catl, err := recipePkg.ReadRecipeCatalogue()
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	versions, err := recipePkg.GetRecipeCatalogueVersions(recipe.Name, catl)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if len(versions) == 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func LintMetadataFilledIn(r recipe.Recipe) (bool, error) {
+	features, category, err := recipe.GetRecipeFeaturesAndCategory(r.Name)
+	if err != nil {
+		return false, err
+	}
+
+	if category == "" {
+		return false, nil
+	}
+
+	if features.Backups == "" ||
+		features.Email == "" ||
+		features.Healthcheck == "" ||
+		features.Image.Image == "" ||
+		features.SSO == "" {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func LintAbraShVendors(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if len(service.Configs) > 0 {
+			abraSh := path.Join(config.RECIPES_DIR, recipe.Name, "abra.sh")
+			if _, err := os.Stat(abraSh); err != nil {
+				if os.IsNotExist(err) {
+					return false, err
+				}
+				return false, err
+			}
+		}
+	}
+	return true, nil
+}
+
+func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
+	url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe.Name)
+
+	res, err := http.Get(url)
+	if err != nil {
+		return false, err
+	}
+
+	if res.StatusCode != 200 {
+		return false, err
+	}
+
+	return true, nil
+}

pkg/secret/pass.go

@@ -19,13 +19,13 @@ func PassInsertSecret(secretValue, secretName, appName, server string) error {
 		secretValue, server, appName, secretName,
 	)
 
-	logrus.Debugf("attempting to run '%s'", cmd)
+	logrus.Debugf("attempting to run %s", cmd)
 
 	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
 		return err
 	}
 
-	logrus.Infof("'%s' inserted into pass store", secretName)
+	logrus.Infof("%s inserted into pass store", secretName)
 
 	return nil
 }
@@ -41,13 +41,13 @@ func PassRmSecret(secretName, appName, server string) error {
 		server, appName, secretName,
 	)
 
-	logrus.Debugf("attempting to run '%s'", cmd)
+	logrus.Debugf("attempting to run %s", cmd)
 
 	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
 		return err
 	}
 
-	logrus.Infof("'%s' removed from pass store", secretName)
+	logrus.Infof("%s removed from pass store", secretName)
 
 	return nil
 }

pkg/secret/secret.go

@@ -34,7 +34,7 @@ func GeneratePasswords(count, length uint) ([]string, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("generated '%s'", strings.Join(passwords, ", "))
+	logrus.Debugf("generated %s", strings.Join(passwords, ", "))
 
 	return passwords, nil
 }
@@ -53,7 +53,7 @@ func GeneratePassphrases(count uint) ([]string, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("generated '%s'", strings.Join(passphrases, ", "))
+	logrus.Debugf("generated %s", strings.Join(passphrases, ", "))
 
 	return passphrases, nil
 }
@@ -69,35 +69,32 @@ func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
 		}
 	}
 
-	logrus.Debugf("read '%s' as secrets from '%s'", secretEnvVars, appEnv)
+	logrus.Debugf("read %s as secrets from %s", secretEnvVars, appEnv)
 
 	return secretEnvVars
 }
 
-// TODO: should probably go in the config/app package?
 func ParseSecretEnvVarName(secretEnvVar string) string {
 	withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_")
 	withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION")
 	name := strings.ToLower(withoutSuffix)
-	logrus.Debugf("parsed '%s' as name from '%s'", name, secretEnvVar)
+	logrus.Debugf("parsed %s as name from %s", name, secretEnvVar)
 	return name
 }
 
-// TODO: should probably go in the config/app package?
 func ParseGeneratedSecretName(secret string, appEnv config.App) string {
 	name := fmt.Sprintf("%s_", appEnv.StackName())
 	withoutAppName := strings.TrimPrefix(secret, name)
 	idx := strings.LastIndex(withoutAppName, "_")
 	parsed := withoutAppName[:idx]
-	logrus.Debugf("parsed '%s' as name from '%s'", parsed, secret)
+	logrus.Debugf("parsed %s as name from %s", parsed, secret)
 	return parsed
 }
 
-// TODO: should probably go in the config/app package?
 func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 	values := strings.Split(secret, "#")
 	if len(values) == 0 {
-		return secretValue{}, fmt.Errorf("unable to parse '%s'", secret)
+		return secretValue{}, fmt.Errorf("unable to parse %s", secret)
 	}
 
 	if len(values) == 1 {
@@ -113,7 +110,7 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 	}
 	version := strings.ReplaceAll(values[0], " ", "")
 
-	logrus.Debugf("parsed version '%s' and length '%v' from '%s'", version, length, secret)
+	logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
 
 	return secretValue{Version: version, Length: length}, nil
 }
@@ -132,7 +129,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 				return
 			}
 			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version)
-			logrus.Debugf("attempting to generate and store '%s' on '%s'", secretRemoteName, server)
+			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
 			if secretValue.Length > 0 {
 				passwords, err := GeneratePasswords(1, uint(secretValue.Length))
 				if err != nil {
@@ -140,7 +137,12 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					return
 				}
 				if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
-					ch <- err
+					if strings.Contains(err.Error(), "AlreadyExists") {
+						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
+						ch <- nil
+					} else {
+						ch <- err
+					}
 					return
 				}
 				secrets[secretName] = passwords[0]
@@ -151,7 +153,13 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					return
 				}
 				if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
-					ch <- err
+					if strings.Contains(err.Error(), "AlreadyExists") {
+						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
+						ch <- nil
+					} else {
+						ch <- err
+					}
+					return
 				}
 				secrets[secretName] = passphrases[0]
 			}
@@ -166,7 +174,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 		}
 	}
 
-	logrus.Debugf("generated and stored '%s' on '%s'", secrets, server)
+	logrus.Debugf("generated and stored %s on %s", secrets, server)
 
 	return secrets, nil
 }

pkg/server/server.go

@@ -12,7 +12,7 @@ import (
 func CreateServerDir(serverName string) error {
 	serverPath := path.Join(config.ABRA_DIR, "servers", serverName)
 
-	if err := os.Mkdir(serverPath, 0755); err != nil {
+	if err := os.Mkdir(serverPath, 0764); err != nil {
 		if !os.IsExist(err) {
 			return err
 		}

pkg/service/service.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// GetService retrieves a service container. If prompt is true and the retrievd
+// count of service containers does not match 1, then a prompt is presented to
+// let the user choose. A count of 0 is handled gracefully.
+func GetService(c context.Context, cl *client.Client, filters filters.Args, prompt bool) (swarm.Service, error) {
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := cl.ServiceList(c, serviceOpts)
+	if err != nil {
+		return swarm.Service{}, err
+	}
+
+	if len(services) == 0 {
+		filter := filters.Get("name")[0]
+		return swarm.Service{}, fmt.Errorf("no services matching the %v filter found?", filter)
+	}
+
+	if len(services) != 1 {
+		var servicesRaw []string
+		for _, service := range services {
+			serviceName := service.Spec.Name
+			created := formatter.HumanDuration(service.CreatedAt.Unix())
+			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
+		}
+
+		if !prompt {
+			err := fmt.Errorf("expected 1 service but found %v: %s", len(services), strings.Join(servicesRaw, " "))
+			return swarm.Service{}, err
+		}
+
+		logrus.Warnf("ambiguous service list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which service are you looking for?",
+			Options: servicesRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return swarm.Service{}, err
+		}
+
+		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, service := range services {
+			serviceName := strings.ToLower(service.Spec.Name)
+			if serviceName == chosenService {
+				return service, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen service")
+	}
+
+	return services[0], nil
+}
+
+// ContainerToServiceName converts a container name to a service name.
+func ContainerToServiceName(containerNames []string, stackName string) string {
+	containerName := strings.Join(containerNames, "")
+	trimmed := strings.TrimPrefix(containerName, "/")
+	stackNameServiceName := strings.Split(trimmed, ".")[0]
+	splitter := fmt.Sprintf("%s_", stackName)
+	return strings.Split(stackNameServiceName, splitter)[1]
+}

pkg/ssh/ssh.go

@@ -111,7 +111,7 @@ type sudoWriter struct {
 
 // Write satisfies the write interface for sudoWriter
 func (w *sudoWriter) Write(p []byte) (int, error) {
-	if string(p) == "sudo_password" {
+	if strings.Contains(string(p), "sudo_password") {
 		w.stdin.Write([]byte(w.pw + "\n"))
 		w.pw = ""
 		return len(p), nil
@@ -131,11 +131,9 @@ func RunSudoCmd(cmd, passwd string, cl *Client) error {
 	}
 	defer session.Close()
 
-	cmd = "sudo -p " + "sudo_password" + " -S " + cmd
+	sudoCmd := fmt.Sprintf("SSH_ASKPASS=/usr/bin/ssh-askpass; sudo -p sudo_password -S %s", cmd)
 
-	w := &sudoWriter{
-		pw: passwd,
-	}
+	w := &sudoWriter{pw: passwd}
 	w.stdin, err = session.StdinPipe()
 	if err != nil {
 		return err
@@ -144,79 +142,19 @@ func RunSudoCmd(cmd, passwd string, cl *Client) error {
 	session.Stdout = w
 	session.Stderr = w
 
-	done := make(chan struct{})
-	scanner := bufio.NewScanner(session.Stdin)
-
-	go func() {
-		for scanner.Scan() {
-			line := scanner.Text()
-			fmt.Println(line)
-		}
-		done <- struct{}{}
-	}()
-
-	if err := session.Start(cmd); err != nil {
-		return err
+	modes := ssh.TerminalModes{
+		ssh.ECHO:          0,
+		ssh.TTY_OP_ISPEED: 14400,
+		ssh.TTY_OP_OSPEED: 14400,
 	}
 
-	<-done
-
-	if err := session.Wait(); err != nil {
-		return err
-	}
-
-	return err
-}
-
-// Exec runs a command on a remote and streams output
-func Exec(cmd string, cl *Client) error {
-	session, err := cl.SSHClient.NewSession()
-	if err != nil {
-		return err
-	}
-	defer session.Close()
-
-	stdout, err := session.StdoutPipe()
+	err = session.RequestPty("xterm", 80, 40, modes)
 	if err != nil {
 		return err
 	}
 
-	stderr, err := session.StdoutPipe()
-	if err != nil {
-		return err
-	}
-
-	stdoutDone := make(chan struct{})
-	stdoutScanner := bufio.NewScanner(stdout)
-
-	go func() {
-		for stdoutScanner.Scan() {
-			line := stdoutScanner.Text()
-			fmt.Println(line)
-		}
-		stdoutDone <- struct{}{}
-	}()
-
-	stderrDone := make(chan struct{})
-	stderrScanner := bufio.NewScanner(stderr)
-
-	go func() {
-		for stderrScanner.Scan() {
-			line := stderrScanner.Text()
-			fmt.Println(line)
-		}
-		stderrDone <- struct{}{}
-	}()
-
-	if err := session.Start(cmd); err != nil {
-		return err
-	}
-
-	<-stdoutDone
-	<-stderrDone
-
-	if err := session.Wait(); err != nil {
-		return err
+	if err := session.Run(sudoCmd); err != nil {
+		return fmt.Errorf("%s", string(w.b.Bytes()))
 	}
 
 	return nil
@@ -320,7 +258,7 @@ func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.Publ
 
 	if exists {
 		hostname := strings.Split(hostnameAndPort, ":")[0]
-		logrus.Debugf("server SSH host key found for %s, moving on", hostname)
+		logrus.Debugf("server SSH host key found for %s", hostname)
 		return nil
 	}
 
@@ -330,9 +268,9 @@ func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.Publ
 
 		fmt.Printf(fmt.Sprintf(`
 You are attempting to make an SSH connection to a server but there is no entry
-in your ~/.ssh/known_hosts file which confirms that this is indeed the server
-you want to connect to. Please take a moment to validate the following SSH host
-key, it is important.
+in your ~/.ssh/known_hosts file which confirms that you have already validated
+that this is indeed the server you want to connect to. Please take a moment to
+validate the following SSH host key, it is important.
 
     Host:        %s
     Fingerprint: %s
@@ -409,12 +347,31 @@ func connect(username, host, port string, authMethod ssh.AuthMethod, timeout tim
 }
 
 func connectWithAgentTimeout(host, username, port string, timeout time.Duration) (*Client, error) {
+	logrus.Debugf("using ssh-agent to make an SSH connection for %s", host)
+
 	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
 	if err != nil {
 		return nil, err
 	}
 
-	authMethod := ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
+	agentCl := agent.NewClient(sshAgent)
+	authMethod := ssh.PublicKeysCallback(agentCl.Signers)
+
+	loadedKeys, err := agentCl.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var convertedKeys []string
+	for _, key := range loadedKeys {
+		convertedKeys = append(convertedKeys, key.String())
+	}
+
+	if len(convertedKeys) > 0 {
+		logrus.Debugf("ssh-agent has these keys loaded: %s", strings.Join(convertedKeys, ","))
+	} else {
+		logrus.Debug("ssh-agent has no keys loaded")
+	}
 
 	return connect(username, host, port, authMethod, timeout)
 }
@@ -427,6 +384,11 @@ func connectWithPasswordTimeout(host, username, port, pass string, timeout time.
 
 // EnsureHostKey ensures that a host key trusted and added to the ~/.ssh/known_hosts file
 func EnsureHostKey(hostname string) error {
+	if hostname == "default" || hostname == "local" {
+		logrus.Debugf("not checking server SSH host key against local/default target")
+		return nil
+	}
+
 	exists, _, err := GetHostKey(hostname)
 	if err != nil {
 		return err
@@ -513,11 +475,10 @@ func GetContextConnDetails(serverName string) (*dockerSSHPkg.Spec, error) {
 func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 	var hostConfig HostConfig
 
-	var host, idf string
-
-	if host = ssh_config.Get(hostname, "Hostname"); host == "" {
-		logrus.Debugf("no hostname found in SSH config, assuming %s", hostname)
-		host = hostname
+	if hostname == "" {
+		if hostname = ssh_config.Get(hostname, "Hostname"); hostname == "" {
+			logrus.Debugf("no hostname found in SSH config, assuming %s", hostname)
+		}
 	}
 
 	if username == "" {
@@ -538,12 +499,19 @@ func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 		}
 	}
 
-	idf = ssh_config.Get(hostname, "IdentityFile")
-
-	hostConfig.Host = host
-	if idf != "" {
+	if idf := ssh_config.Get(hostname, "IdentityFile"); idf != "" && idf != "~/.ssh/identity" {
+		var err error
+		idf, err = identityFileAbsPath(idf)
+		if err != nil {
+			return hostConfig, err
+		}
 		hostConfig.IdentityFile = idf
+	} else {
+		logrus.Debugf("no identity file found in SSH config for %s", hostname)
+		hostConfig.IdentityFile = ""
 	}
+
+	hostConfig.Host = hostname
 	hostConfig.Port = port
 	hostConfig.User = username
 
@@ -551,3 +519,25 @@ func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 
 	return hostConfig, nil
 }
+
+func identityFileAbsPath(relPath string) (string, error) {
+	var err error
+	var absPath string
+
+	if strings.HasPrefix(relPath, "~/") {
+		systemUser, err := user.Current()
+		if err != nil {
+			return absPath, err
+		}
+		absPath = filepath.Join(systemUser.HomeDir, relPath[2:])
+	} else {
+		absPath, err = filepath.Abs(relPath)
+		if err != nil {
+			return absPath, err
+		}
+	}
+
+	logrus.Debugf("resolved %s to %s to read the ssh identity file", relPath, absPath)
+
+	return absPath, nil
+}

pkg/upstream/commandconn/commandconn.go

@@ -188,14 +188,14 @@ func ignorableCloseError(err error) bool {
 func (c *commandConn) CloseRead() error {
 	// NOTE: maybe already closed here
 	if err := c.stdout.Close(); err != nil && !ignorableCloseError(err) {
-		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseRead: %v", err)
 	}
 	c.stdioClosedMu.Lock()
 	c.stdoutClosed = true
 	c.stdioClosedMu.Unlock()
 	if err := c.killIfStdioClosed(); err != nil {
-		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseRead: %v", err)
 	}
 	return nil
@@ -212,14 +212,14 @@ func (c *commandConn) Read(p []byte) (int, error) {
 func (c *commandConn) CloseWrite() error {
 	// NOTE: maybe already closed here
 	if err := c.stdin.Close(); err != nil && !ignorableCloseError(err) {
-		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseWrite: %v", err)
 	}
 	c.stdioClosedMu.Lock()
 	c.stdinClosed = true
 	c.stdioClosedMu.Unlock()
 	if err := c.killIfStdioClosed(); err != nil {
-		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseWrite: %v", err)
 	}
 	return nil
@@ -239,7 +239,7 @@ func (c *commandConn) Close() error {
 		logrus.Warnf("commandConn.Close: CloseRead: %v", err)
 	}
 	if err = c.CloseWrite(); err != nil {
-		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.Close: CloseWrite: %v", err)
 	}
 	return err

pkg/upstream/commandconn/connection.go

@@ -2,6 +2,7 @@ package commandconn
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/url"
 
@@ -34,9 +35,25 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 		if err != nil {
 			return nil, errors.Wrap(err, "ssh host connection is not valid")
 		}
+
 		if err := sshPkg.EnsureHostKey(ctxConnDetails.Host); err != nil {
 			return nil, err
 		}
+
+		hostConfig, err := sshPkg.GetHostConfig(
+			ctxConnDetails.Host,
+			ctxConnDetails.User,
+			ctxConnDetails.Port,
+		)
+		if err != nil {
+			return nil, err
+		}
+		if hostConfig.IdentityFile != "" {
+			msg := "discovered %s as identity file for %s, using for ssh connection"
+			logrus.Debugf(msg, hostConfig.IdentityFile, ctxConnDetails.Host)
+			sshFlags = append(sshFlags, fmt.Sprintf("-o IdentityFile=%s", hostConfig.IdentityFile))
+		}
+
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)

pkg/upstream/container/hijack.go

@@ -16,7 +16,6 @@ import (
 )
 
 // The default escape key sequence: ctrl-p, ctrl-q
-// TODO: This could be moved to `pkg/term`.
 var defaultEscapeKeys = []byte{16, 17}
 
 // A hijackedIOStreamer handles copying input to and output from streams to the

pkg/upstream/convert/service.go

@@ -399,7 +399,6 @@ func convertServiceNetworks(
 	return nets, nil
 }
 
-// TODO: fix secrets API so that SecretAPIClient is not required here
 func convertServiceSecrets(
 	client client.SecretAPIClient,
 	namespace Namespace,
@@ -442,8 +441,6 @@ func convertServiceSecrets(
 // required by the serivce. Unlike convertServiceSecrets, this takes the whole
 // ServiceConfig, because some Configs may be needed as a result of other
 // fields (like CredentialSpecs).
-//
-// TODO: fix configs API so that ConfigsAPIClient is not required here
 func convertServiceConfigObjs(
 	client client.ConfigAPIClient,
 	namespace Namespace,
@@ -626,7 +623,6 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 }
 
 func convertRestartPolicy(restart string, source *composetypes.RestartPolicy) (*swarm.RestartPolicy, error) {
-	// TODO: log if restart is being ignored
 	if source == nil {
 		policy, err := opts.ParseRestartPolicy(restart)
 		if err != nil {

pkg/upstream/service/scale.go

@@ -0,0 +1,44 @@
+package upstream
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// RunServiceScale scales a service (useful for restart action)
+func RunServiceScale(ctx context.Context, cl *client.Client, serviceID string, scale uint64) error {
+	service, _, err := cl.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
+	if err != nil {
+		return err
+	}
+
+	serviceMode := &service.Spec.Mode
+	if serviceMode.Replicated != nil {
+		serviceMode.Replicated.Replicas = &scale
+	} else if serviceMode.ReplicatedJob != nil {
+		serviceMode.ReplicatedJob.TotalCompletions = &scale
+	} else {
+		return fmt.Errorf("scale can only be used with replicated or replicated-job mode")
+	}
+
+	response, err := cl.ServiceUpdate(
+		ctx,
+		service.ID,
+		service.Version,
+		service.Spec,
+		types.ServiceUpdateOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	for _, warning := range response.Warnings {
+		logrus.Warn(warning)
+	}
+
+	return nil
+}

pkg/upstream/stack/loader.go

@@ -13,6 +13,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// DontSkipValidation ensures validation is done for compose file loading
+func DontSkipValidation(opts *loader.Options) {
+	opts.SkipValidation = false
+}
+
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
 func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
@@ -21,26 +26,25 @@ func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Confi
 	}
 
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails)
+	config, err := loader.Load(configDetails, DontSkipValidation)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
-			return nil, fmt.Errorf("compose file contains unsupported options:\n\n%s",
+			return nil, fmt.Errorf("compose file contains unsupported options: %s",
 				propertyWarnings(fpe.Properties))
 		}
-
 		return nil, err
 	}
 
 	unsupportedProperties := loader.GetUnsupportedProperties(dicts...)
 	if len(unsupportedProperties) > 0 {
-		logrus.Warnf("Ignoring unsupported options: %s\n\n",
-			strings.Join(unsupportedProperties, ", "))
+		logrus.Warnf("%s: ignoring unsupported options: %s",
+			appEnv["TYPE"], strings.Join(unsupportedProperties, ", "))
 	}
 
 	deprecatedProperties := loader.GetDeprecatedProperties(dicts...)
 	if len(deprecatedProperties) > 0 {
-		logrus.Warnf("Ignoring deprecated options:\n\n%s\n\n",
-			propertyWarnings(deprecatedProperties))
+		logrus.Warnf("%s: ignoring deprecated options: %s",
+			appEnv["TYPE"], propertyWarnings(deprecatedProperties))
 	}
 	return config, nil
 }

pkg/upstream/stack/stack.go

@@ -3,10 +3,14 @@ package stack // https://github.com/docker/cli/blob/master/cli/command/stack/swa
 import (
 	"context"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strings"
+	"time"
 
 	abraClient "coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/upstream/convert"
+	"github.com/docker/cli/cli/command/service/progress"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -108,7 +112,7 @@ func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, sta
 
 // IsDeployed chekcks whether an appp is deployed or not.
 func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string) (bool, string, error) {
-	version := ""
+	version := "unknown"
 	isDeployed := false
 
 	filter := filters.NewArgs()
@@ -128,12 +132,12 @@ func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string)
 			}
 		}
 
-		logrus.Debugf("'%s' has been detected as deployed with version '%s'", stackName, version)
+		logrus.Debugf("%s has been detected as deployed with version %s", stackName, version)
 
 		return true, version, nil
 	}
 
-	logrus.Debugf("'%s' has been detected as not deployed", stackName)
+	logrus.Debugf("%s has been detected as not deployed", stackName)
 	return isDeployed, version, nil
 }
 
@@ -154,7 +158,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve
 }
 
 // RunDeploy is the swarm implementation of docker stack deploy
-func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config) error {
+func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
 	ctx := context.Background()
 
 	if err := validateResolveImageFlag(&opts); err != nil {
@@ -166,7 +170,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config) e
 		opts.ResolveImage = ResolveImageNever
 	}
 
-	return deployCompose(ctx, cl, opts, cfg)
+	return deployCompose(ctx, cl, opts, cfg, appName, dontWait)
 }
 
 // validateResolveImageFlag validates the opts.resolveImage command line option
@@ -179,7 +183,7 @@ func validateResolveImageFlag(opts *Deploy) error {
 	}
 }
 
-func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config) error {
+func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
 	namespace := convert.NewNamespace(opts.Namespace)
 
 	if opts.Prune {
@@ -220,7 +224,7 @@ func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, co
 		return err
 	}
 
-	return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage)
+	return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, appName, dontWait)
 }
 
 func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
@@ -335,7 +339,9 @@ func deployServices(
 	services map[string]swarm.ServiceSpec,
 	namespace convert.Namespace,
 	sendAuth bool,
-	resolveImage string) error {
+	resolveImage string,
+	appName string,
+	dontWait bool) error {
 	existingServices, err := GetStackServices(ctx, cl, namespace.Name())
 	if err != nil {
 		return err
@@ -346,6 +352,7 @@ func deployServices(
 		existingServiceMap[service.Spec.Name] = service
 	}
 
+	serviceIDs := make(map[string]string)
 	for internalName, serviceSpec := range services {
 		var (
 			name        = namespace.Scope(internalName)
@@ -353,18 +360,6 @@ func deployServices(
 			encodedAuth string
 		)
 
-		// FIXME: disable for now as not sure how to avoid having a `dockerCli`
-		// instance here and would rather not copy/pasta that entire module in
-		// right now for something that we don't even support right now. Will skip
-		// this for now.
-		if sendAuth {
-			// Retrieve encoded auth token from the image reference
-			// encodedAuth, err = command.RetrieveAuthTokenFromImage(ctx, dockerCli, image)
-			// if err != nil {
-			// 	return err
-			// }
-		}
-
 		if service, exists := existingServiceMap[name]; exists {
 			logrus.Infof("Updating service %s (id: %s)\n", name, service.ID)
 
@@ -397,7 +392,6 @@ func deployServices(
 
 			// Stack deploy does not have a `--force` option. Preserve existing
 			// ForceUpdate value so that tasks are not re-deployed if not updated.
-			// TODO move this to API client?
 			serviceSpec.TaskTemplate.ForceUpdate = service.Spec.TaskTemplate.ForceUpdate
 
 			response, err := cl.ServiceUpdate(ctx, service.ID, service.Version, serviceSpec, updateOpts)
@@ -405,6 +399,8 @@ func deployServices(
 				return errors.Wrapf(err, "failed to update service %s", name)
 			}
 
+			serviceIDs[service.ID] = name
+
 			for _, warning := range response.Warnings {
 				logrus.Warn(warning)
 			}
@@ -418,11 +414,44 @@ func deployServices(
 				createOpts.QueryRegistry = true
 			}
 
-			if _, err := cl.ServiceCreate(ctx, serviceSpec, createOpts); err != nil {
+			serviceCreateResponse, err := cl.ServiceCreate(ctx, serviceSpec, createOpts)
+			if err != nil {
 				return errors.Wrapf(err, "failed to create service %s", name)
 			}
+
+			serviceIDs[serviceCreateResponse.ID] = name
 		}
 	}
+
+	var serviceNames []string
+	for _, serviceName := range serviceIDs {
+		serviceNames = append(serviceNames, serviceName)
+	}
+
+	if dontWait {
+		logrus.Warn("skipping converge logic checks")
+		return nil
+	}
+
+	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
+	ch := make(chan error, len(serviceIDs))
+	for serviceID, serviceName := range serviceIDs {
+		logrus.Debugf("waiting on %s to converge", serviceName)
+		go func(sID, sName, aName string) {
+			ch <- WaitOnService(ctx, cl, sID, aName)
+		}(serviceID, serviceName, appName)
+	}
+
+	for _, serviceID := range serviceIDs {
+		err := <-ch
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("assuming %s converged successfully", serviceID)
+	}
+
+	logrus.Info("services converged 👌")
+
 	return nil
 }
 
@@ -437,3 +466,44 @@ func getStackSecrets(ctx context.Context, dockerclient client.APIClient, namespa
 func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespace string) ([]swarm.Config, error) {
 	return dockerclient.ConfigList(ctx, types.ConfigListOptions{Filters: getStackFilter(namespace)})
 }
+
+// https://github.com/docker/cli/blob/master/cli/command/service/helpers.go
+// https://github.com/docker/cli/blob/master/cli/command/service/progress/progress.go
+func WaitOnService(ctx context.Context, cl *dockerclient.Client, serviceID, appName string) error {
+	errChan := make(chan error, 1)
+	pipeReader, pipeWriter := io.Pipe()
+
+	go func() {
+		errChan <- progress.ServiceProgress(ctx, cl, serviceID, pipeWriter)
+	}()
+
+	go io.Copy(ioutil.Discard, pipeReader)
+
+	timeout := 50 * time.Second
+
+	select {
+	case err := <-errChan:
+		return err
+	case <-time.After(timeout):
+		return fmt.Errorf(fmt.Sprintf(`
+%s has not converged (%s second timeout reached).
+
+This does not necessarily mean your deployment has failed, it may just be that
+the app is taking longer to deploy based on your server resources or network
+latency.
+
+You can track latest deployment status with:
+
+    abra app ps --watch %s
+
+And inspect the logs with:
+
+    abra app logs %s
+
+If a service is failing to even start, try smoke out the error with:
+
+    abra app errors --watch %s
+
+`, appName, timeout, appName, appName, appName))
+	}
+}

pkg/web/client.go

@@ -0,0 +1,25 @@
+package web
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/sirupsen/logrus"
+)
+
+// customLeveledLogger is custom logger with logrus baked in
+type customLeveledLogger struct {
+	retryablehttp.Logger
+}
+
+// Printf wires up logrus into the custom retryablehttp logger
+func (l customLeveledLogger) Printf(msg string, args ...interface{}) {
+	logrus.Debugf(fmt.Sprintf(msg, args...))
+}
+
+// NewHTTPRetryClient instantiates a new http client with retries baked in
+func NewHTTPRetryClient() *retryablehttp.Client {
+	retryClient := retryablehttp.NewClient()
+	retryClient.Logger = customLeveledLogger{}
+	return retryClient
+}

pkg/web/web.go

@@ -3,7 +3,10 @@ package web
 
 import (
 	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
+	"os"
 	"time"
 )
 
@@ -13,7 +16,7 @@ const Timeout = 10 * time.Second
 
 // ReadJSON reads JSON and parses it into your chosen interface pointer
 func ReadJSON(url string, target interface{}) error {
-	httpClient := &http.Client{Timeout: Timeout}
+	httpClient := NewHTTPRetryClient()
 	res, err := httpClient.Get(url)
 	if err != nil {
 		return err
@@ -21,3 +24,29 @@ func ReadJSON(url string, target interface{}) error {
 	defer res.Body.Close()
 	return json.NewDecoder(res.Body).Decode(target)
 }
+
+// GetFile downloads a file and saves it to a filepath
+func GetFile(filepath string, url string) (err error) {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

renovate.json

@@ -0,0 +1,3 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json"
+}