NOT WORKING: WIP on adding server-wide logs functionality

.drone.yml

@@ -3,12 +3,12 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.18
+    image: golang:1.17
     commands:
       - make check
 
   - name: make static
-    image: golang:1.18
+    image: golang:1.17
     ignore: true # until we decide we all want this check
     environment:
       STATIC_CHECK_URL: honnef.co/go/tools/cmd/staticcheck
@@ -18,12 +18,12 @@ steps:
       - make static
 
   - name: make build
-    image: golang:1.18
+    image: golang:1.17
     commands:
       - make build
 
   - name: make test
-    image: golang:1.18
+    image: golang:1.17
     commands:
       - make test
 
@@ -55,7 +55,7 @@ steps:
       event: tag
 
   - name: release
-    image: golang:1.18
+    image: golang:1.17
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token

.e2e.env.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

.gitignore

@@ -1,8 +1,6 @@
-*fmtcoverage.html
-.e2e.env
-.envrc
-.vscode/
 abra
-dist/
+.vscode/
-tests/integration/.abra/catalogue
 vendor/
+.envrc
+dist/
+*fmtcoverage.html

AUTHORS.md

@@ -1,10 +0,0 @@
-# authors
-
-> If you're looking at this and you hack on Abra and you're not listed here,
-> please do add yourself! This is a community project, let's show
-
-- 3wordchant
-- decentral1se
-- kawaiipunk
-- knoflook
-- roxxers

Makefile

@@ -5,7 +5,7 @@ LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 export GOPRIVATE=coopcloud.tech
 
-all: format check static build test
+all: run test install build clean format check static
 
 run:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)

README.md

@@ -7,6 +7,69 @@
 
 The Co-op Cloud utility belt 🎩🐇
 
-`abra` is our flagship client & command-line tool which has been developed specifically in the context of the Co-op Cloud project for the purpose of making day-to-day operations for [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) as convenient as possible. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community ❤
+`abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create apps, deploy them, run backup and restore operations and a whole lot of other things. Please see [docs.coopcloud.tech](https://docs.coopcloud.tech) for more extensive documentation.
 
-Please see [docs.coopcloud.tech/abra/](https://docs.coopcloud.tech/abra/) for help on install, upgrade, hacking, troubleshooting & more!
+## Quick install
+
+```bash
+curl https://install.abra.autonomic.zone | bash
+```
+
+Or using the latest release candidate (extra experimental!):
+
+```bash
+curl https://install.abra.autonomic.zone | bash -s -- --rc
+```
+
+Source for this script is in [scripts/installer/installer](./scripts/installer/installer).
+
+## Hacking
+
+### Getting started
+
+Install [direnv](https://direnv.net), run `cp .envrc.sample .envrc`, then run `direnv allow` in this directory. This will set coopcloud repos as private due to [this bug.](https://git.coopcloud.tech/coop-cloud/coopcloud.tech/issues/20#issuecomment-8201). Or you can run `go env -w GOPRIVATE=coopcloud.tech` but I'm not sure how persistent this is.
+
+Install [Go >= 1.16](https://golang.org/doc/install) and then:
+
+- `make build` to build
+- `./abra` to run commands
+- `make test` will run tests
+- `make install` will install it to `$GOPATH/bin`
+- `go get <package>` and `go mod tidy` to add a new dependency
+
+Our [Drone CI configuration](.drone.yml) runs a number of sanity on each pushed commit. See the [Makefile](./Makefile) for more handy targets.
+
+Please use the [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/) for your commits so we can automate our change log.
+
+### Versioning
+
+We use [goreleaser](https://goreleaser.com) to help us automate releases. We use [semver](https://semver.org) for versioning all releases of the tool. While we are still in the public alpha release phase, we will maintain a `0.y.z-alpha` format. Change logs are generated from our commit logs. We are still working this out and aim to refine our release praxis as we go.
+
+For developers, while using this `-alpha` format, the `y` part is the "major" version part. So, if you make breaking changes, you increment that and _not_ the `x` part. So, if you're on `0.1.0-alpha`, then you'd go to `0.1.1-alpha` for a backwards compatible change and `0.2.0-alpha` for a backwards incompatible change.
+
+### Making a new release
+
+- Change `ABRA_VERSION` to match the new tag in [`scripts`](./scripts/installer/installer) (use [semver](https://semver.org))
+- Commit that change (e.g. `git commit -m 'chore: publish next tag x.y.z-alpha'`)
+- Make a new tag (e.g. `git tag -a x.y.z-alpha`)
+- Push the new tag (e.g. `git push && git push --tags`)
+- Wait until the build finishes on [build.coopcloud.tech](https://build.coopcloud.tech/coop-cloud/abra)
+- Deploy the new installer script (e.g. `cd ./scripts/installer && make`)
+- Check the release worked, (e.g. `abra upgrade; abra -v`)
+
+### Fork maintenance
+
+#### `godotenv`
+
+We maintain a fork of [godotenv](https://github.com/Autonomic-Cooperative/godotenv) for two features:
+
+1. multi-line env var support
+2. inline comment parsing
+
+You can upgrade the version here by running `go get github.com/Autonomic-Cooperative/godotenv@<commit>` where `<commit>` is the
+latest commit you want to pin to. We are aiming to migrate to YAML format for the environment configuration, so this should only
+be a temporary thing.
+
+#### `docker/client`
+
+A number of modules in [pkg/upstream](./pkg/upstream) are copy/pasta'd from the upstream [docker/docker/client](https://pkg.go.dev/github.com/docker/docker/client). We had to do this because upstream are not exposing their API as public.

cli/app/app.go

@@ -1,22 +1,29 @@
 package app
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var AppCommand = cli.Command{
+// AppCommand defines the `abra app` command and ets subcommands
+var AppCommand = &cli.Command{
 	Name:      "app",
+	Usage:     "Manage deployed apps",
 	Aliases:   []string{"a"},
-	Usage:       "Manage apps",
+	ArgsUsage: "<app>",
-	ArgsUsage:   "<domain>",
+	Description: `
-	Description: "This command provides functionality for managing the life cycle of your apps",
+This command provides all the functionality you need to manage the life cycle
-	Subcommands: []cli.Command{
+of your apps. From initial deployment, day-2 operations (e.g. backup/restore)
+to scaling apps up and spinning them down.
+`,
+	Subcommands: []*cli.Command{
 		appNewCommand,
 		appConfigCommand,
 		appRestartCommand,
 		appDeployCommand,
 		appUpgradeCommand,
 		appUndeployCommand,
+		appBackupCommand,
+		appRestoreCommand,
 		appRemoveCommand,
 		appCheckCommand,
 		appListCommand,
@@ -28,6 +35,5 @@ var AppCommand = cli.Command{
 		appSecretCommand,
 		appVolumeCommand,
 		appVersionCommand,
-		appErrorsCommand,
 	},
 }

cli/app/backup.go

@@ -0,0 +1,87 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var backupAllServices bool
+var backupAllServicesFlag = &cli.BoolFlag{
+	Name:        "all",
+	Value:       false,
+	Destination: &backupAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Backup all services",
+}
+
+var appBackupCommand = &cli.Command{
+	Name:      "backup",
+	Usage:     "Backup an app",
+	Aliases:   []string{"b"},
+	Flags:     []cli.Flag{backupAllServicesFlag},
+	ArgsUsage: "<service>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Get(1) != "" && backupAllServices {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<service>' and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
+			logrus.Fatal(err)
+		}
+
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+
+		execCmd := "abra_backup"
+		if !backupAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_backup_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/check.go

@@ -1,33 +1,29 @@
 package app
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appCheckCommand = cli.Command{
+var appCheckCommand = &cli.Command{
 	Name:      "check",
-	Aliases:   []string{"c"},
 	Usage:     "Check if app is configured correctly",
-	ArgsUsage: "<domain>",
+	Aliases:   []string{"c"},
-	Flags: []cli.Flag{
+	ArgsUsage: "<service>",
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", app.Type, ".env.sample")
 		if _, err := os.Stat(envSamplePath); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
+				logrus.Fatalf("'%s' does not exist?", envSamplePath)
 			}
 			logrus.Fatal(err)
 		}
@@ -49,9 +45,20 @@ var appCheckCommand = cli.Command{
 			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
 		}
 
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+		logrus.Infof("all necessary environment variables defined for '%s'", app.Name)
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/config.go

@@ -2,26 +2,21 @@ package app
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"os/exec"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appConfigCommand = cli.Command{
+var appConfigCommand = &cli.Command{
 	Name:    "config",
 	Aliases: []string{"c"},
 	Usage:   "Edit app config",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		appName := c.Args().First()
 
@@ -36,7 +31,7 @@ var appConfigCommand = cli.Command{
 
 		appFile, exists := files[appName]
 		if !exists {
-			logrus.Fatalf("cannot find app with name %s", appName)
+			logrus.Fatalf("cannot find app with name '%s'", appName)
 		}
 
 		ed, ok := os.LookupEnv("EDITOR")
@@ -60,5 +55,16 @@ var appConfigCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/cp.go

@@ -1,44 +1,32 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appCpCommand = cli.Command{
+var appCpCommand = &cli.Command{
 	Name:      "cp",
 	Aliases:   []string{"c"},
-	ArgsUsage: "<domain> <src> <dst>",
+	ArgsUsage: "<src> <dst>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Usage:     "Copy files to/from a running app service",
 	Description: `
 This command supports copying files to and from any app service file system.
 
 If you want to copy a myfile.txt to the root of the app service:
 
-    abra app cp <domain> myfile.txt app:/
+    abra app cp <app> myfile.txt app:/
 
 And if you want to copy that file back to your current working directory locally:
 
-		abra app cp <domain> app:/myfile.txt .
+		abra app cp <app> app:/myfile.txt .
+
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
@@ -88,64 +76,23 @@ And if you want to copy that file back to your current working directory locally
 				logrus.Fatalf("%s does not exist locally?", dstPath)
 			}
 		}
-		err := configureAndCp(c, app, srcPath, dstPath, service, isToContainer)
+		err := internal.ConfigureAndCp(c, app, srcPath, dstPath, service, isToContainer)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		return nil
 
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
-}
+		appNames, err := config.GetAppNames()
-
+		if err != nil {
-func configureAndCp(
+			logrus.Warn(err)
-	c *cli.Context,
+		}
-	app config.App,
+		if c.NArg() > 0 {
-	srcPath string,
+			return
-	dstPath string,
+		}
-	service string,
+		for _, a := range appNames {
-	isToContainer bool) error {
+			fmt.Println(a)
-	cl, err := client.New(app.Server)
+		}
-	if err != nil {
+	},
-		logrus.Fatal(err)
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), service))
-
-	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
-
-	if isToContainer {
-		if _, err := os.Stat(srcPath); err != nil {
-			logrus.Fatalf("%s does not exist?", srcPath)
-		}
-
-		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		content, err := archive.TarWithOptions(srcPath, toTarOpts)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	} else {
-		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		defer content.Close()
-		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	return nil
 }

cli/app/deploy.go

@@ -1,29 +1,26 @@
 package app
 
 import (
+	"fmt"
+
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
-	"github.com/urfave/cli"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
 )
 
-var appDeployCommand = cli.Command{
+var appDeployCommand = &cli.Command{
 	Name:    "deploy",
 	Aliases: []string{"d"},
 	Usage:   "Deploy an app",
-	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
-This command deploys an app. It does not support incrementing the version of a
+This command deploys a new instance of an app. It does not support changing the
-deployed app, for this you need to look at the "abra app upgrade <domain>"
+version of an existing deployed app, for this you need to look at the "abra app
-command.
+upgrade <app>" command.
 
 You may pass "--force" to re-deploy the same version again. This can be useful
 if the container runtime has gotten into a weird state.
@@ -33,5 +30,16 @@ including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
 	Action: internal.DeployAction,
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/errors.go

@@ -1,141 +0,0 @@
-package app
-
-import (
-	"context"
-	"strconv"
-	"strings"
-	"time"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var appErrorsCommand = cli.Command{
-	Name:      "errors",
-	Usage:     "List errors for a deployed app",
-	ArgsUsage: "<domain>",
-	Description: `
-This command lists errors for a deployed app.
-
-This is a best-effort implementation and an attempt to gather a number of tips
-& tricks for finding errors together into one convenient command. When an app
-is failing to deploy or having issues, it could be a lot of things.
-
-This command currently takes into account:
-
-    Is the service deployed?
-    Is the service killed by an OOM error?
-    Is the service reporting an error (like in "ps --no-trunc" output)
-    Is the service healthcheck failing? what are the healthcheck logs?
-
-Got any more ideas? Please let us know:
-
-    https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose
-
-This command is best accompanied by "abra app logs <domain>" which may reveal
-further information which can help you debug the cause of an app failure via
-the logs.
-`,
-	Aliases: []string{"e"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.WatchFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
-		}
-
-		if !internal.Watch {
-			if err := checkErrors(c, cl, app); err != nil {
-				logrus.Fatal(err)
-			}
-			return nil
-		}
-
-		for {
-			if err := checkErrors(c, cl, app); err != nil {
-				logrus.Fatal(err)
-			}
-			time.Sleep(2 * time.Second)
-		}
-
-		return nil
-	},
-}
-
-func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
-	recipe, err := recipe.Get(app.Recipe)
-	if err != nil {
-		return err
-	}
-
-	for _, service := range recipe.Config.Services {
-		filters := filters.NewArgs()
-		filters.Add("name", service.Name)
-		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
-		if err != nil {
-			return err
-		}
-
-		if len(containers) == 0 {
-			logrus.Warnf("%s is not up, something seems wrong", service.Name)
-			continue
-		}
-
-		container := containers[0]
-		containerState, err := cl.ContainerInspect(context.Background(), container.ID)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if containerState.State.OOMKilled {
-			logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
-		}
-
-		if containerState.State.Error != "" {
-			logrus.Warnf("%s reports this error: %s", service.Name, containerState.State.Error)
-		}
-
-		if containerState.State.Health != nil {
-			if containerState.State.Health.Status != "healthy" {
-				logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
-				logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
-				for _, log := range containerState.State.Health.Log {
-					logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func getServiceName(names []string) string {
-	containerName := strings.Join(names, " ")
-	trimmed := strings.TrimPrefix(containerName, "/")
-	return strings.Split(trimmed, ".")[0]
-}

cli/app/list.go

@@ -5,61 +5,45 @@ import (
 	"sort"
 	"strings"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/ssh"
 	"coopcloud.tech/tagcmp"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var status bool
 var statusFlag = &cli.BoolFlag{
-	Name:        "status, S",
+	Name:        "status",
+	Aliases:     []string{"S"},
+	Value:       false,
 	Usage:       "Show app deployment status",
 	Destination: &status,
 }
 
-var appRecipe string
+var appType string
-var recipeFlag = &cli.StringFlag{
+var typeFlag = &cli.StringFlag{
-	Name:        "recipe, r",
+	Name:        "type",
+	Aliases:     []string{"t"},
 	Value:       "",
-	Usage:       "Show apps of a specific recipe",
+	Usage:       "Show apps of a specific type",
-	Destination: &appRecipe,
+	Destination: &appType,
 }
 
 var listAppServer string
 var listAppServerFlag = &cli.StringFlag{
-	Name:        "server, s",
+	Name:        "server",
+	Aliases:     []string{"s"},
 	Value:       "",
 	Usage:       "Show apps of a specific server",
 	Destination: &listAppServer,
 }
 
-type appStatus struct {
+var appListCommand = &cli.Command{
-	server  string
-	recipe  string
-	appName string
-	domain  string
-	status  string
-	version string
-	upgrade string
-}
-
-type serverStatus struct {
-	apps             []appStatus
-	appCount         int
-	versionCount     int
-	unversionedCount int
-	latestCount      int
-	upgradeCount     int
-}
-
-var appListCommand = cli.Command{
 	Name:  "list",
-	Aliases: []string{"ls"},
 	Usage: "List all managed apps",
 	Description: `
 This command looks at your local file system listing of apps and servers (e.g.
@@ -68,14 +52,13 @@ in ~/.abra/) to generate a report of all your apps.
 By passing the "--status/-S" flag, you can query all your servers for the
 actual live deployment status. Depending on how many servers you manage, this
 can take some time.
-`,
+	`,
+	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
 		statusFlag,
 		listAppServerFlag,
-		recipeFlag,
+		typeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		appFiles, err := config.LoadAppFiles(listAppServer)
 		if err != nil {
@@ -86,12 +69,8 @@ can take some time.
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		sort.Sort(config.ByServerAndType(apps))
 
-		sort.Sort(config.ByServerAndRecipe(apps))
-
-		statuses := make(map[string]map[string]string)
-		var catl recipe.RecipeCatalogue
-		if status {
 		alreadySeen := make(map[string]bool)
 		for _, app := range apps {
 			if _, ok := alreadySeen[app.Server]; !ok {
@@ -102,65 +81,61 @@ can take some time.
 			}
 		}
 
+		statuses := make(map[string]map[string]string)
+		tableCol := []string{"Server", "Type", "App Name", "Domain"}
+		if status {
+			tableCol = append(tableCol, "Status", "Version", "Updates")
 			statuses, err = config.GetAppStatuses(appFiles)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+		}
 
-			var err error
+		table := abraFormatter.CreateTable(tableCol)
-			catl, err = recipe.ReadRecipeCatalogue()
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+
+		var (
+			versionedAppsCount   int
+			unversionedAppsCount int
+			onLatestCount        int
+			canUpgradeCount      int
+		)
+
+		catl, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		}
 
-		var totalServersCount int
+		var appsCount int
-		var totalAppsCount int
-		allStats := make(map[string]serverStatus)
 		for _, app := range apps {
-			var stats serverStatus
+			var tableRow []string
-			var ok bool
+			if app.Type == appType || appType == "" {
-			if stats, ok = allStats[app.Server]; !ok {
+				appsCount++
-				stats = serverStatus{}
-				if appRecipe == "" {
-					// count server, no filtering
-					totalServersCount++
-				}
-			}
-
-			if app.Recipe == appRecipe || appRecipe == "" {
-				if appRecipe != "" {
-					// only count server if matches filter
-					totalServersCount++
-				}
-
-				appStats := appStatus{}
-				stats.appCount++
-				totalAppsCount++
 
+				// If type flag is set, check for it, if not, Type == ""
+				tableRow = []string{app.Server, app.Type, app.StackName(), app.Domain}
 				if status {
+					stackName := app.StackName()
 					status := "unknown"
 					version := "unknown"
-					if statusMeta, ok := statuses[app.StackName()]; ok {
+					if statusMeta, ok := statuses[stackName]; ok {
 						if currentVersion, exists := statusMeta["version"]; exists {
-							if currentVersion != "" {
 							version = currentVersion
 						}
-						}
 						if statusMeta["status"] != "" {
 							status = statusMeta["status"]
 						}
-						stats.versionCount++
+						tableRow = append(tableRow, status, version)
+						versionedAppsCount++
 					} else {
-						stats.unversionedCount++
+						tableRow = append(tableRow, status, version)
+						unversionedAppsCount++
 					}
 
-					appStats.status = status
-					appStats.version = version
-
 					var newUpdates []string
 					if version != "unknown" {
-						updates, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+
+						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 						if err != nil {
 							logrus.Fatal(err)
 						}
@@ -184,80 +159,40 @@ can take some time.
 
 					if len(newUpdates) == 0 {
 						if version == "unknown" {
-							appStats.upgrade = "unknown"
+							tableRow = append(tableRow, "unknown")
 						} else {
-							appStats.upgrade = "latest"
+							tableRow = append(tableRow, "on latest")
-							stats.latestCount++
+							onLatestCount++
 						}
 					} else {
-						newUpdates = internal.ReverseStringList(newUpdates)
+						// FIXME: jeezus golang why do you not have a list reverse function
-						appStats.upgrade = strings.Join(newUpdates, "\n")
+						for i, j := 0, len(newUpdates)-1; i < j; i, j = i+1, j-1 {
-						stats.upgradeCount++
+							newUpdates[i], newUpdates[j] = newUpdates[j], newUpdates[i]
+						}
+						tableRow = append(tableRow, strings.Join(newUpdates, "\n"))
+						canUpgradeCount++
 					}
 				}
-
-				appStats.server = app.Server
-				appStats.recipe = app.Recipe
-				appStats.appName = app.Name
-				appStats.domain = app.Domain
-
-				stats.apps = append(stats.apps, appStats)
-			}
-
-			allStats[app.Server] = stats
-		}
-
-		alreadySeen := make(map[string]bool)
-		for _, app := range apps {
-			if _, ok := alreadySeen[app.Server]; ok {
-				continue
-			}
-
-			serverStat := allStats[app.Server]
-
-			tableCol := []string{"recipe", "domain"}
-			if status {
-				tableCol = append(tableCol, []string{"status", "version", "upgrade"}...)
-			}
-
-			table := formatter.CreateTable(tableCol)
-
-			for _, appStat := range serverStat.apps {
-				tableRow := []string{appStat.recipe, appStat.domain}
-				if status {
-					tableRow = append(tableRow, []string{appStat.status, appStat.version, appStat.upgrade}...)
 			}
 			table.Append(tableRow)
 		}
 
-			if table.NumLines() > 0 {
+		var stats string
-				table.Render()
-
 		if status {
-					fmt.Println(fmt.Sprintf(
+			stats = fmt.Sprintf(
-						"server: %s | total apps: %v | versioned: %v | unversioned: %v | latest: %v | upgrade: %v",
+				"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
-						app.Server,
+				appsCount,
-						serverStat.appCount,
+				versionedAppsCount,
-						serverStat.versionCount,
+				unversionedAppsCount,
-						serverStat.unversionedCount,
+				onLatestCount,
-						serverStat.latestCount,
+				canUpgradeCount,
-						serverStat.upgradeCount,
+			)
-					))
 		} else {
-					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.appCount))
+			stats = fmt.Sprintf("Total apps: %v", appsCount)
-				}
 		}
 
-			if len(allStats) > 1 && table.NumLines() > 0 {
+		table.SetCaption(true, stats)
-				fmt.Println() // newline separator for multiple servers
+		table.Render()
-			}
-
-			alreadySeen[app.Server] = true
-		}
-
-		if len(allStats) > 1 {
-			fmt.Println(fmt.Sprintf("total servers: %v | total apps: %v ", totalServersCount, totalAppsCount))
-		}
 
 		return nil
 	},

cli/app/logs.go

@@ -1,119 +1,77 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
-	"sync"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/service"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var logOpts = types.ContainerLogsOptions{
+var appLogsCommand = &cli.Command{
-	Details:    false,
-	Follow:     true,
-	ShowStderr: true,
-	ShowStdout: true,
-	Tail:       "20",
-	Timestamps: true,
-}
-
-// stackLogs lists logs for all stack services
-func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
-	filters := filters.NewArgs()
-	filters.Add("name", stackName)
-	serviceOpts := types.ServiceListOptions{Filters: filters}
-	services, err := client.ServiceList(context.Background(), serviceOpts)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	for _, service := range services {
-		wg.Add(1)
-		go func(s string) {
-			if internal.StdErrOnly {
-				logOpts.ShowStdout = false
-			}
-
-			logs, err := client.ServiceLogs(context.Background(), s, logOpts)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			defer logs.Close()
-
-			_, err = io.Copy(os.Stdout, logs)
-			if err != nil && err != io.EOF {
-				logrus.Fatal(err)
-			}
-		}(service.ID)
-	}
-
-	wg.Wait()
-
-	os.Exit(0)
-}
-
-var appLogsCommand = cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
-	ArgsUsage: "<domain> [<service>]",
+	ArgsUsage: "[<service>]",
 	Usage:     "Tail app logs",
 	Flags: []cli.Flag{
-		internal.StdErrOnlyFlag,
+		internal.StderrFlag,
-		internal.DebugFlag,
+		internal.StdoutFlag,
+		internal.HealthcheckFlag,
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		if !internal.Stderr && !internal.Stdout && !internal.Healthcheck {
+			internal.Stderr = true
+			internal.Stdout = true
+			internal.Healthcheck = true
+		}
+
+		logrus.Debugf("flags parsed. --stderr: %t, --stdout: %t, --healthcheck: %t", internal.Stderr, internal.Stdout, internal.Healthcheck)
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		logOpts := types.ContainerLogsOptions{
+			Details:    false,
+			Follow:     true,
+			ShowStderr: internal.Stderr,
+			ShowStdout: internal.Stdout,
+			Tail:       "20",
+			Timestamps: true,
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
-			logrus.Debugf("tailing logs for all %s services", app.Recipe)
+			logrus.Debug("tailing logs for all app services")
-			stackLogs(c, app.StackName(), cl)
+			internal.StackLogs(c, app.StackName(), logOpts, cl)
-		} else {
-			logrus.Debugf("tailing logs for %s", serviceName)
-			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
-				logrus.Fatal(err)
-			}
 		}
+		logrus.Debugf("tailing logs for '%s'", serviceName)
 
-		return nil
+		service := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	},
-}
-
-func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
 		filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
+		filters.Add("name", service)
-	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
+		serviceOpts := types.ServiceListOptions{Filters: filters}
+		services, err := cl.ServiceList(c.Context, serviceOpts)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
+		if len(services) != 1 {
-	if internal.StdErrOnly {
+			logrus.Fatalf("expected 1 service but got %v", len(services))
-		logOpts.ShowStdout = false
 		}
 
-	logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts)
+		logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		// defer after err check as any err returns a nil io.ReadCloser
 		defer logs.Close()
 
 		_, err = io.Copy(os.Stdout, logs)
@@ -122,4 +80,17 @@ func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, se
 		}
 
 		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/new.go

@@ -1,9 +1,12 @@
 package app
 
 import (
+	"fmt"
+
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"github.com/urfave/cli"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
 )
 
 var appNewDescription = `
@@ -11,7 +14,7 @@ This command takes a recipe and uses it to create a new app. This new app
 configuration is stored in your ~/.abra directory under the appropriate server.
 
 This command does not deploy your app for you. You will need to run "abra app
-deploy <domain>" to do so.
+deploy <app>" to do so.
 
 You can see what recipes are available (i.e. values for the <recipe> argument)
 by running "abra recipe ls".
@@ -26,21 +29,30 @@ pass store (see passwordstore.org for more). The pass command must be available
 on your $PATH.
 `
 
-var appNewCommand = cli.Command{
+var appNewCommand = &cli.Command{
 	Name:        "new",
-	Aliases:     []string{"n"},
 	Usage:       "Create a new app",
+	Aliases:     []string{"n"},
 	Description: appNewDescription,
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.NewAppServerFlag,
 		internal.DomainFlag,
+		internal.NewAppNameFlag,
 		internal.PassFlag,
 		internal.SecretsFlag,
 	},
-	Before:       internal.SubCommandBefore,
+	ArgsUsage: "<recipe>",
-	ArgsUsage:    "[<recipe>]",
 	Action:    internal.NewAction,
-	BashComplete: autocomplete.RecipeNameComplete,
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
 }

cli/app/ps.go

@@ -1,39 +1,66 @@
 package app
 
 import (
-	"context"
+	"fmt"
 	"strings"
 	"time"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/docker/cli/cli/command/formatter"
-	"coopcloud.tech/abra/pkg/service"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/buger/goterm"
-	dockerFormatter "github.com/docker/cli/cli/command/formatter"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appPsCommand = cli.Command{
+var watch bool
+var watchFlag = &cli.BoolFlag{
+	Name:        "watch",
+	Aliases:     []string{"w"},
+	Value:       false,
+	Usage:       "Watch status by polling repeatedly",
+	Destination: &watch,
+}
+
+var appPsCommand = &cli.Command{
 	Name:        "ps",
-	Aliases:     []string{"p"},
 	Usage:       "Check app status",
-	ArgsUsage:   "<domain>",
 	Description: "This command shows a more detailed status output of a specific deployed app.",
+	Aliases:     []string{"p"},
 	Flags: []cli.Flag{
-		internal.WatchFlag,
+		watchFlag,
-		internal.DebugFlag,
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
+		if !watch {
+			showPSOutput(c)
+			return nil
+		}
+
+		// TODO: how do we make this update in-place in an x-platform way?
+		for {
+			showPSOutput(c)
+			time.Sleep(2 * time.Second)
+		}
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}
+
+// showPSOutput renders ps output.
+func showPSOutput(c *cli.Context) {
 	app := internal.ValidateApp(c)
 
 	cl, err := client.New(app.Server)
@@ -41,42 +68,16 @@ var appPsCommand = cli.Command{
 		logrus.Fatal(err)
 	}
 
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
-		}
-
-		if !internal.Watch {
-			showPSOutput(c, app, cl)
-			return nil
-		}
-
-		goterm.Clear()
-		for {
-			goterm.MoveCursor(1, 1)
-			showPSOutput(c, app, cl)
-			goterm.Flush()
-			time.Sleep(2 * time.Second)
-		}
-	},
-}
-
-// showPSOutput renders ps output.
-func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
 	filters := filters.NewArgs()
 	filters.Add("name", app.StackName())
 
-	containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+	containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
+	tableCol := []string{"image", "created", "status", "ports", "app name", "services"}
-	table := formatter.CreateTable(tableCol)
+	table := abraFormatter.CreateTable(tableCol)
 
 	for _, container := range containers {
 		var containerNames []string
@@ -86,12 +87,12 @@ func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
 		}
 
 		tableRow := []string{
-			service.ContainerToServiceName(container.Names, app.StackName()),
+			abraFormatter.RemoveSha(container.Image),
-			formatter.RemoveSha(container.Image),
+			abraFormatter.HumanDuration(container.Created),
-			formatter.HumanDuration(container.Created),
 			container.Status,
-			container.State,
+			formatter.DisplayablePorts(container.Ports),
-			dockerFormatter.DisplayablePorts(container.Ports),
+			app.StackName(),
+			strings.Join(containerNames, "\n"),
 		}
 		table.Append(tableRow)
 	}

cli/app/remove.go

@@ -1,19 +1,18 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"os"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // Volumes stores the variable from VolumesFlag
@@ -21,35 +20,32 @@ var Volumes bool
 
 // VolumesFlag is used to specify if volumes should be deleted when deleting an app
 var VolumesFlag = &cli.BoolFlag{
-	Name:        "volumes, V",
+	Name:        "volumes",
+	Value:       false,
 	Destination: &Volumes,
 }
 
-var appRemoveCommand = cli.Command{
+var appRemoveCommand = &cli.Command{
 	Name:    "remove",
-	Aliases:   []string{"rm"},
-	ArgsUsage: "<domain>",
 	Usage:   "Remove an already undeployed app",
+	Aliases: []string{"rm"},
 	Flags: []cli.Flag{
 		VolumesFlag,
 		internal.ForceFlag,
-		internal.DebugFlag,
-		internal.NoInputFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if !internal.Force && !internal.NoInput {
+		if !internal.Force {
 			response := false
 			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("about to remove %s, are you sure?", app.Name),
+				Message: fmt.Sprintf("about to delete %s, are you sure?", app.Name),
 			}
 			if err := survey.AskOne(prompt, &response); err != nil {
 				logrus.Fatal(err)
 			}
 			if !response {
-				logrus.Fatal("aborting as requested")
+				logrus.Fatal("user aborted app removal")
 			}
 		}
 
@@ -58,17 +54,19 @@ var appRemoveCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if !internal.Force {
+			isDeployed, _, err := stack.IsDeployed(c.Context, cl, app.StackName())
 			if err != nil {
 				logrus.Fatal(err)
 			}
 			if isDeployed {
-			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+				logrus.Fatalf("'%s' is still deployed. Run \"abra app undeploy %s \" or pass --force", app.Name, app.Name)
+			}
 		}
 
 		fs := filters.NewArgs()
-		fs.Add("name", app.StackName())
+		fs.Add("name", app.Name)
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: fs})
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: fs})
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -83,12 +81,9 @@ var appRemoveCommand = cli.Command{
 
 		if len(secrets) > 0 {
 			var secretNamesToRemove []string
-
+			if !internal.Force {
-			if !internal.Force && !internal.NoInput {
 				secretsPrompt := &survey.MultiSelect{
 					Message: "which secrets do you want to remove?",
-					Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-					VimMode: true,
 					Options: secretNames,
 					Default: secretNames,
 				}
@@ -97,12 +92,8 @@ var appRemoveCommand = cli.Command{
 				}
 			}
 
-			if internal.Force || internal.NoInput {
-				secretNamesToRemove = secretNames
-			}
-
 			for _, name := range secretNamesToRemove {
-				err := cl.SecretRemove(context.Background(), secrets[name])
+				err := cl.SecretRemove(c.Context, secrets[name])
 				if err != nil {
 					logrus.Fatal(err)
 				}
@@ -112,7 +103,7 @@ var appRemoveCommand = cli.Command{
 			logrus.Info("no secrets to remove")
 		}
 
-		volumeListOKBody, err := cl.VolumeList(context.Background(), fs)
+		volumeListOKBody, err := cl.VolumeList(c.Context, fs)
 		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
@@ -126,11 +117,9 @@ var appRemoveCommand = cli.Command{
 		if len(vols) > 0 {
 			if Volumes {
 				var removeVols []string
-				if !internal.Force && !internal.NoInput {
+				if !internal.Force {
 					volumesPrompt := &survey.MultiSelect{
 						Message: "which volumes do you want to remove?",
-						Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-						VimMode: true,
 						Options: vols,
 						Default: vols,
 					}
@@ -138,9 +127,8 @@ var appRemoveCommand = cli.Command{
 						logrus.Fatal(err)
 					}
 				}
-
 				for _, vol := range removeVols {
-					err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+					err := cl.VolumeRemove(c.Context, vol, internal.Force) // last argument is for force removing
 					if err != nil {
 						logrus.Fatal(err)
 					}
@@ -150,10 +138,8 @@ var appRemoveCommand = cli.Command{
 				logrus.Info("no volumes were removed")
 			}
 		} else {
-			if Volumes {
 			logrus.Info("no volumes to remove")
 		}
-		}
 
 		err = os.Remove(app.Path)
 		if err != nil {
@@ -163,5 +149,16 @@ var appRemoveCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/restart.go

@@ -1,35 +1,30 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	upstream "coopcloud.tech/abra/pkg/upstream/service"
+	"coopcloud.tech/abra/pkg/config"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appRestartCommand = cli.Command{
+var appRestartCommand = &cli.Command{
 	Name:        "restart",
-	Aliases:   []string{"re"},
 	Usage:       "Restart an app",
-	ArgsUsage: "<domain>",
+	Aliases:     []string{"R"},
-	Flags: []cli.Flag{
+	ArgsUsage:   "<service>",
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
 	Description: `This command restarts a service within a deployed app.`,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		serviceNameShort := c.Args().Get(1)
+		serviceName := c.Args().Get(1)
-		if serviceNameShort == "" {
+		if serviceName == "" {
 			err := errors.New("missing service?")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
@@ -39,32 +34,39 @@ var appRestartCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
+		serviceFilter := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+		filters := filters.NewArgs()
+		filters.Add("name", serviceFilter)
+		containerOpts := types.ContainerListOptions{Filters: filters}
+		containers, err := cl.ContainerList(c.Context, containerOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if len(containers) != 1 {
+			logrus.Fatalf("expected 1 service but got %v", len(containers))
+		}
 
-		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)
+		logrus.Debugf("attempting to restart %s", serviceFilter)
-		if err := upstream.RunServiceScale(context.Background(), cl, serviceName, 0); err != nil {
+
+		timeout := 30 * time.Second
+		if err := cl.ContainerRestart(c.Context, containers[0].ID, &timeout); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := stack.WaitOnService(context.Background(), cl, serviceName, app.Name); err != nil {
+		logrus.Infof("%s service restarted", serviceFilter)
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("%s has been scaled to 0 (restart logic)", serviceName)
-
-		logrus.Debugf("attempting to scale %s to 1 (restart logic)", serviceName)
-		if err := upstream.RunServiceScale(context.Background(), cl, serviceName, 1); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := stack.WaitOnService(context.Background(), cl, serviceName, app.Name); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("%s has been scaled to 1 (restart logic)", serviceName)
-
-		logrus.Infof("%s service successfully restarted", serviceNameShort)
 
 		return nil
 	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/restore.go

@@ -0,0 +1,79 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var restoreAllServices bool
+var restoreAllServicesFlag = &cli.BoolFlag{
+	Name:        "all",
+	Value:       false,
+	Destination: &restoreAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Restore all services",
+}
+
+var appRestoreCommand = &cli.Command{
+	Name:      "restore",
+	Usage:     "Restore an app from a backup",
+	Aliases:   []string{"r"},
+	Flags:     []cli.Flag{restoreAllServicesFlag},
+	ArgsUsage: "<service> [<backup file>]",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() > 1 && restoreAllServices {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use <service>/<backup file> and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
+			logrus.Fatal(err)
+		}
+
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+		execCmd := "abra_restore"
+		if !restoreAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_restore_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+		}
+
+		backupFile := c.Args().Get(2)
+		if backupFile != "" {
+			execCmd = fmt.Sprintf("%s %s", execCmd, backupFile)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/rollback.go

@@ -1,12 +1,10 @@
 package app
 
 import (
-	"context"
 	"fmt"
 
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
@@ -15,23 +13,18 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appRollbackCommand = cli.Command{
+var appRollbackCommand = &cli.Command{
 	Name:      "rollback",
-	Aliases:   []string{"rl"},
 	Usage:     "Roll an app back to a previous version",
-	ArgsUsage: "<domain>",
+	Aliases:   []string{"r", "downgrade"},
+	ArgsUsage: "<app>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command rolls an app back to a previous version if one exists.
 
@@ -39,66 +32,59 @@ You may pass "--force/-f" to downgrade to the same version again. This can be
 useful if the container runtime has gotten into a weird state.
 
 This action could be destructive, please ensure you have a copy of your app
-data beforehand.
+data beforehand - see "abra app backup <app>" for more.
 
 Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
-		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		r, err := recipe.Get(app.Recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
-		}
-
 		var availableDowngrades []string
-		if deployedVersion == "unknown" {
+		if deployedVersion == "" {
+			deployedVersion = "unknown"
 			availableDowngrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
 		}
 
 		if deployedVersion != "unknown" && !internal.Chaos {
@@ -117,21 +103,23 @@ recipes.
 			}
 
 			if len(availableDowngrades) == 0 {
-				logrus.Info("no available downgrades, you're on oldest ✌️")
+				logrus.Fatal("no available downgrades, you're on latest")
-				return nil
 			}
 		}
 
-		availableDowngrades = internal.ReverseStringList(availableDowngrades)
+		// FIXME: jeezus golang why do you not have a list reverse function
+		for i, j := 0, len(availableDowngrades)-1; i < j; i, j = i+1, j-1 {
+			availableDowngrades[i], availableDowngrades[j] = availableDowngrades[j], availableDowngrades[i]
+		}
 
 		var chosenDowngrade string
 		if !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force {
 				chosenDowngrade = availableDowngrades[0]
-				logrus.Debugf("choosing %s as version to downgrade to (--force)", chosenDowngrade)
+				logrus.Debugf("choosing '%s' as version to downgrade to (--force)", chosenDowngrade)
 			} else {
 				prompt := &survey.Select{
-					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
+					Message: fmt.Sprintf("Please select a downgrade (current version: '%s'):", deployedVersion),
 					Options: availableDowngrades,
 				}
 				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
@@ -141,7 +129,7 @@ recipes.
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenDowngrade); err != nil {
+			if err := recipe.EnsureVersion(app.Type, chosenDowngrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -149,13 +137,13 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenDowngrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenDowngrade, err = recipe.ChaosVersion(app.Type)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
 		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
@@ -164,7 +152,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -180,12 +168,12 @@ recipes.
 		}
 
 		if !internal.Force {
-			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
+			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/run.go

@@ -1,55 +1,51 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var user string
 var userFlag = &cli.StringFlag{
-	Name:        "user, u",
+	Name:        "user",
 	Value:       "",
 	Destination: &user,
 }
 
 var noTTY bool
 var noTTYFlag = &cli.BoolFlag{
-	Name:        "no-tty, t",
+	Name:        "no-tty",
+	Value:       false,
 	Destination: &noTTY,
 }
 
-var appRunCommand = cli.Command{
+var appRunCommand = &cli.Command{
 	Name: "run",
-	Aliases: []string{"r"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
 		noTTYFlag,
 		userFlag,
 	},
-	Before:       internal.SubCommandBefore,
+	Aliases:   []string{"r"},
-	ArgsUsage:    "<domain> <service> <args>...",
+	ArgsUsage: "<service> <args>...",
 	Usage:     "Run a command in a service container",
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if len(c.Args()) < 2 {
+		if c.Args().Len() < 2 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no <service> provided?"))
 		}
 
-		if len(c.Args()) < 3 {
+		if c.Args().Len() < 3 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no <args> provided?"))
 		}
 
@@ -63,12 +59,19 @@ var appRunCommand = cli.Command{
 		filters := filters.NewArgs()
 		filters.Add("name", stackAndServiceName)
 
-		targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		cmd := c.Args()[2:]
+		if len(containers) == 0 {
+			logrus.Fatalf("no containers matching '%s' found?", stackAndServiceName)
+		}
+		if len(containers) > 1 {
+			logrus.Fatalf("expected 1 container matching '%s' but got %d", stackAndServiceName, len(containers))
+		}
+
+		cmd := c.Args().Slice()[2:]
 		execCreateOpts := types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
@@ -85,16 +88,41 @@ var appRunCommand = cli.Command{
 			execCreateOpts.Tty = false
 		}
 
-		// FIXME: avoid instantiating a new CLI
+		// FIXME: an absolutely monumental hack to instantiate another command-line
+		// client withing our command-line client so that we pass something down
+		// the tubes that satisfies the necessary interface requirements. We should
+		// refactor our vendored container code to not require all this cruft.  For
+		// now, It Works.
 		dcli, err := command.NewDockerCli()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if err := container.RunExec(dcli, cl, containers[0].ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
+	BashComplete: func(c *cli.Context) {
+		switch c.NArg() {
+		case 0:
+			appNames, err := config.GetAppNames()
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, a := range appNames {
+				fmt.Println(a)
+			}
+		case 1:
+			appName := c.Args().First()
+			serviceNames, err := config.GetAppServiceNames(appName)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, s := range serviceNames {
+				fmt.Println(s)
+			}
+		}
+	},
 }

cli/app/secret.go

@@ -1,55 +1,41 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"os"
 	"strconv"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var allSecrets bool
 var allSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
+	Name:        "all",
+	Aliases:     []string{"A"},
+	Value:       false,
 	Destination: &allSecrets,
 	Usage:       "Generate all secrets",
 }
 
-var rmAllSecrets bool
+var appSecretGenerateCommand = &cli.Command{
-var rmAllSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &rmAllSecrets,
-	Usage:       "Remove all secrets",
-}
-
-var appSecretGenerateCommand = cli.Command{
 	Name:      "generate",
 	Aliases:   []string{"g"},
 	Usage:     "Generate secrets",
-	ArgsUsage: "<domain> <secret> <version>",
+	ArgsUsage: "<secret> <version>",
-	Flags: []cli.Flag{
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
-		internal.DebugFlag,
-		allSecretsFlag,
-		internal.PassFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if len(c.Args()) == 1 && !allSecrets {
+		if c.Args().Len() == 1 && !allSecrets {
 			err := errors.New("missing arguments <secret>/<version> or '--all'")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
@@ -71,12 +57,10 @@ var appSecretGenerateCommand = cli.Command{
 				parsed := secret.ParseSecretEnvVarName(sec)
 				if secretName == parsed {
 					secretsToCreate[sec] = secretVersion
-					matches = true
 				}
 			}
-
 			if !matches {
-				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
+				logrus.Fatalf("'%s' doesn't exist in the env config?", secretName)
 			}
 		}
 
@@ -87,7 +71,7 @@ var appSecretGenerateCommand = cli.Command{
 
 		if internal.Pass {
 			for name, data := range secretVals {
-				if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+				if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
 					logrus.Fatal(err)
 				}
 			}
@@ -99,7 +83,7 @@ var appSecretGenerateCommand = cli.Command{
 		}
 
 		tableCol := []string{"name", "value"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 		for name, val := range secretVals {
 			table.Append([]string{name, val})
 		}
@@ -110,17 +94,12 @@ var appSecretGenerateCommand = cli.Command{
 	},
 }
 
-var appSecretInsertCommand = cli.Command{
+var appSecretInsertCommand = &cli.Command{
 	Name:      "insert",
 	Aliases:   []string{"i"},
 	Usage:     "Insert secret",
-	Flags: []cli.Flag{
+	Flags:     []cli.Flag{internal.PassFlag},
-		internal.DebugFlag,
+	ArgsUsage: "<app> <secret-name> <version> <data>",
-		internal.PassFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	ArgsUsage:    "<domain> <secret-name> <version> <data>",
-	BashComplete: autocomplete.AppNameComplete,
 	Description: `
 This command inserts a secret into an app environment.
 
@@ -136,7 +115,7 @@ Example:
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if len(c.Args()) != 4 {
+		if c.Args().Len() != 4 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
 		}
 
@@ -149,10 +128,8 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		logrus.Infof("%s successfully stored on server", secretName)
-
 		if internal.Pass {
-			if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+			if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -161,54 +138,28 @@ Example:
 	},
 }
 
-// secretRm removes a secret.
+var appSecretRmCommand = &cli.Command{
-func secretRm(cl *dockerClient.Client, app config.App, secretName, parsed string) error {
-	if err := cl.SecretRemove(context.Background(), secretName); err != nil {
-		return err
-	}
-
-	logrus.Infof("deleted %s successfully from server", secretName)
-
-	if internal.PassRemove {
-		if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
-			return err
-		}
-
-		logrus.Infof("deleted %s successfully from local pass store", secretName)
-	}
-
-	return nil
-}
-
-var appSecretRmCommand = cli.Command{
 	Name:      "remove",
-	Aliases: []string{"rm"},
 	Usage:     "Remove a secret",
-	Flags: []cli.Flag{
+	Aliases:   []string{"rm"},
-		internal.DebugFlag,
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
-		internal.NoInputFlag,
+	ArgsUsage: "<app> <secret-name>",
-		rmAllSecretsFlag,
-		internal.PassRemoveFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	ArgsUsage:    "<domain> [<secret-name>]",
-	BashComplete: autocomplete.AppNameComplete,
 	Description: `
-This command removes app secrets.
+This command removes a secret from an app environment.
 
 Example:
 
     abra app secret remove myapp db_pass
+
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		secrets := secret.ReadSecretEnvVars(app.Env)
 
-		if c.Args().Get(1) != "" && rmAllSecrets {
+		if c.Args().Get(1) != "" && allSecrets {
 			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret-name>' and '--all' together"))
 		}
 
-		if c.Args().Get(1) == "" && !rmAllSecrets {
+		if c.Args().Get(1) == "" && !allSecrets {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no secret(s) specified?"))
 		}
 
@@ -219,72 +170,52 @@ Example:
 
 		filters := filters.NewArgs()
 		filters.Add("name", app.StackName())
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		remoteSecretNames := make(map[string]bool)
-		for _, cont := range secretList {
-			remoteSecretNames[cont.Spec.Annotations.Name] = true
-		}
-
-		match := false
 		secretToRm := c.Args().Get(1)
-		for sec := range secrets {
+		for _, cont := range secretList {
-			secretName := secret.ParseSecretEnvVarName(sec)
+			secretName := cont.Spec.Annotations.Name
-
+			parsed := secret.ParseGeneratedSecretName(secretName, app)
-			secVal, err := secret.ParseSecretEnvVarValue(secrets[sec])
+			if allSecrets {
-			if err != nil {
+				if err := cl.SecretRemove(c.Context, secretName); err != nil {
 					logrus.Fatal(err)
 				}
-
+				if internal.Pass {
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version)
+					if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
-			if _, ok := remoteSecretNames[secretRemoteName]; ok {
-				if secretToRm != "" {
-					if secretName == secretToRm {
-						if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
 						logrus.Fatal(err)
 					}
-
-						return nil
 				}
 			} else {
-					match = true
+				if parsed == secretToRm {
-
+					if err := cl.SecretRemove(c.Context, secretName); err != nil {
-					if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
+						logrus.Fatal(err)
+					}
+					if internal.Pass {
+						if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
 							logrus.Fatal(err)
 						}
 					}
 				}
 			}
-
-		if !match && secretToRm != "" {
-			logrus.Fatalf("%s doesn't exist on server?", secretToRm)
-		}
-
-		if !match {
-			logrus.Fatal("no secrets to remove?")
 		}
 
 		return nil
 	},
 }
 
-var appSecretLsCommand = cli.Command{
+var appSecretLsCommand = &cli.Command{
 	Name:    "list",
-	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Usage:   "List all secrets",
+	Aliases: []string{"ls"},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		secrets := secret.ReadSecretEnvVars(app.Env)
 
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
@@ -293,7 +224,7 @@ var appSecretLsCommand = cli.Command{
 
 		filters := filters.NewArgs()
 		filters.Add("name", app.StackName())
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -318,23 +249,29 @@ var appSecretLsCommand = cli.Command{
 			table.Append(tableRow)
 		}
 
-		if table.NumLines() > 0 {
 		table.Render()
-		} else {
-			logrus.Warnf("no secrets stored for %s", app.Name)
-		}
-
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }
 
-var appSecretCommand = cli.Command{
+var appSecretCommand = &cli.Command{
 	Name:      "secret",
 	Aliases:   []string{"s"},
 	Usage:     "Manage app secrets",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<command>",
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		appSecretGenerateCommand,
 		appSecretInsertCommand,
 		appSecretRmCommand,

cli/app/undeploy.go

@@ -1,25 +1,19 @@
 package app
 
 import (
-	"context"
+	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appUndeployCommand = cli.Command{
+var appUndeployCommand = &cli.Command{
 	Name:    "undeploy",
-	Aliases:   []string{"un"},
+	Aliases: []string{"u"},
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Usage:   "Undeploy an app",
 	Description: `
 This does not destroy any of the application data. However, you should remain
@@ -35,15 +29,15 @@ volumes as eligiblef or pruning once undeployed.
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", stackName)
 		}
 
 		if err := internal.DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
@@ -51,11 +45,22 @@ volumes as eligiblef or pruning once undeployed.
 		}
 
 		rmOpts := stack.Remove{Namespaces: []string{app.StackName()}}
-		if err := stack.RunRemove(context.Background(), cl, rmOpts); err != nil {
+		if err := stack.RunRemove(c.Context, cl, rmOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/upgrade.go

@@ -1,49 +1,42 @@
 package app
 
 import (
-	"context"
 	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appUpgradeCommand = cli.Command{
+var appUpgradeCommand = &cli.Command{
 	Name:      "upgrade",
-	Aliases:   []string{"up"},
+	Aliases:   []string{"u"},
 	Usage:     "Upgrade an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<app>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command supports upgrading an app. You can use it to choose and roll out a
 new upgrade to an existing app.
 
-This command specifically supports incrementing the version of running apps, as
+This command specifically supports changing the version of running apps, as
-opposed to "abra app deploy <domain>" which will not change the version of a
+opposed to "abra app deploy <app>" which will not change the version of a
 deployed app.
 
 You may pass "--force/-f" to upgrade to the same version again. This can be
 useful if the container runtime has gotten into a weird state.
 
 This action could be destructive, please ensure you have a copy of your app
-data beforehand.
+data beforehand - see "abra app backup <app>" for more.
 
 Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
@@ -53,55 +46,41 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
-		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		r, err := recipe.Get(app.Recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Fatalf("no versions available '%s' in recipe catalogue?", app.Type)
 		}
 
 		var availableUpgrades []string
-		if deployedVersion == "unknown" {
+		if deployedVersion == "" {
+			deployedVersion = "unknown"
 			availableUpgrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
 		}
 
 		if deployedVersion != "unknown" && !internal.Chaos {
@@ -120,21 +99,19 @@ recipes.
 			}
 
 			if len(availableUpgrades) == 0 && !internal.Force {
-				logrus.Infof("no available upgrades, you're on latest (%s) ✌️", deployedVersion)
+				logrus.Fatal("no available upgrades, you're on latest")
-				return nil
+				availableUpgrades = versions
 			}
 		}
 
-		availableUpgrades = internal.ReverseStringList(availableUpgrades)
-
 		var chosenUpgrade string
 		if len(availableUpgrades) > 0 && !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force {
 				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
-				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
+				logrus.Debugf("choosing '%s' as version to upgrade to", chosenUpgrade)
 			} else {
 				prompt := &survey.Select{
-					Message: fmt.Sprintf("Please select an upgrade (current version: %s):", deployedVersion),
+					Message: fmt.Sprintf("Please select an upgrade (current version: '%s'):", deployedVersion),
 					Options: availableUpgrades,
 				}
 				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
@@ -143,16 +120,8 @@ recipes.
 			}
 		}
 
-		// if release notes written after git tag published, read them before we
-		// check out the tag and then they'll appear to be missing. this covers
-		// when we obviously will forget to write release notes before publishing
-		releaseNotes, err := internal.GetReleaseNotes(app.Recipe, chosenUpgrade)
-		if err != nil {
-			return err
-		}
-
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
+			if err := recipe.EnsureVersion(app.Type, chosenUpgrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -160,13 +129,13 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenUpgrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenUpgrade, err = recipe.ChaosVersion(app.Type)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
 		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
@@ -175,7 +144,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -190,15 +159,26 @@ recipes.
 			logrus.Fatal(err)
 		}
 
-		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/version.go

@@ -1,17 +1,18 @@
 package app
 
 import (
-	"context"
+	"fmt"
+	"strings"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // getImagePath returns the image name
@@ -20,25 +21,17 @@ func getImagePath(image string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-
 	path := reference.Path(img)
-
+	if strings.Contains(path, "library") {
-	path = formatter.StripTagMeta(path)
+		path = strings.Split(path, "/")[1]
-
+	}
-	logrus.Debugf("parsed %s from %s", path, image)
+	logrus.Debugf("parsed '%s' from '%s'", path, image)
-
 	return path, nil
 }
 
-var appVersionCommand = cli.Command{
+var appVersionCommand = &cli.Command{
 	Name:    "version",
 	Aliases: []string{"v"},
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Usage:   "Show app versions",
 	Description: `
 This command shows all information about versioning related to a deployed app.
@@ -54,27 +47,27 @@ Cloud recipe version.
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if deployedVersion == "unknown" {
+		if deployedVersion == "" {
-			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
+			logrus.Fatalf("failed to determine version of deployed '%s'", app.Name)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe)
+		recipeMeta, err := catalogue.GetRecipeMeta(app.Type)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versionsMeta := make(map[string]recipe.ServiceMeta)
+		versionsMeta := make(map[string]catalogue.ServiceMeta)
 		for _, recipeVersion := range recipeMeta.Versions {
 			if currentVersion, exists := recipeVersion[deployedVersion]; exists {
 				versionsMeta = currentVersion
@@ -82,20 +75,30 @@ Cloud recipe version.
 		}
 
 		if len(versionsMeta) == 0 {
-			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
+			logrus.Fatalf("PANIC: could not retrieve deployed version ('%s') from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"version", "service", "image", "digest"}
+		tableCol := []string{"name", "image", "version", "tag", "digest"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
-		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Digest})
+			table.Append([]string{serviceName, versionMeta.Image, deployedVersion, versionMeta.Tag, versionMeta.Digest})
 		}
 
 		table.Render()
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/volume.go

@@ -1,105 +1,77 @@
 package app
 
 import (
-	"context"
+	"fmt"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/config"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appVolumeListCommand = cli.Command{
+var appVolumeListCommand = &cli.Command{
 	Name:    "list",
-	Aliases:   []string{"ls"},
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:       internal.SubCommandBefore,
 	Usage:   "List volumes associated with an app",
-	BashComplete: autocomplete.AppNameComplete,
+	Aliases: []string{"ls"},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		volumeList, err := client.GetVolumes(context.Background(), app.Server, app.StackName())
+		volumeList, err := client.GetVolumes(c.Context, app.Server, app.Name)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		table := formatter.CreateTable([]string{"name", "created", "mounted"})
+		table := abraFormatter.CreateTable([]string{"driver", "volume name"})
 		var volTable [][]string
 		for _, volume := range volumeList {
-			volRow := []string{volume.Name, volume.CreatedAt, volume.Mountpoint}
+			volRow := []string{
+				volume.Driver,
+				volume.Name,
+			}
 			volTable = append(volTable, volRow)
 		}
 
 		table.AppendBulk(volTable)
-
-		if table.NumLines() > 0 {
 		table.Render()
-		} else {
-			logrus.Warnf("no volumes created for %s", app.Name)
-		}
 
 		return nil
 	},
 }
 
-var appVolumeRemoveCommand = cli.Command{
+var appVolumeRemoveCommand = &cli.Command{
 	Name:    "remove",
 	Usage:   "Remove volume(s) associated with an app",
-	Description: `
-This command supports removing volumes associated with an app. The app in
-question must be undeployed before you try to remove volumes. See "abra app
-undeploy <domain>" for more.
-
-The command is interactive and will show a multiple select input which allows
-you to make a seclection. Use the "?" key to see more help on navigating this
-interface.
-
-Passing "--force/-f" will select all volumes for removal. Be careful.
-`,
-	ArgsUsage: "<domain>",
 	Aliases: []string{"rm"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		volumeList, err := client.GetVolumes(context.Background(), app.Server, app.StackName())
+		volumeList, err := client.GetVolumes(c.Context, app.Server, app.Name)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		volumeNames := client.GetVolumeNames(volumeList)
 
 		var volumesToRemove []string
-		if !internal.Force && !internal.NoInput {
+		if !internal.Force {
 			volumesPrompt := &survey.MultiSelect{
 				Message: "which volumes do you want to remove?",
-				Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-				VimMode: true,
 				Options: volumeNames,
 				Default: volumeNames,
 			}
 			if err := survey.AskOne(volumesPrompt, &volumesToRemove); err != nil {
 				logrus.Fatal(err)
 			}
-		}
+		} else {
-
-		if internal.Force || internal.NoInput {
 			volumesToRemove = volumeNames
 		}
 
-		err = client.RemoveVolumes(context.Background(), app.Server, volumesToRemove, internal.Force)
+		err = client.RemoveVolumes(c.Context, app.Server, volumesToRemove, internal.Force)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -108,15 +80,26 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }
 
-var appVolumeCommand = cli.Command{
+var appVolumeCommand = &cli.Command{
 	Name:      "volume",
 	Aliases:   []string{"vl"},
 	Usage:     "Manage app volumes",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<command>",
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		appVolumeListCommand,
 		appVolumeRemoveCommand,
 	},

cli/autocomplete.go

@@ -0,0 +1,121 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// downloadFile downloads a file brah
+func downloadFile(filepath string, url string) (err error) {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AutoCompleteCommand helps people set up auto-complete in their shells
+var AutoCompleteCommand = &cli.Command{
+	Name:    "autocomplete",
+	Usage:   "Help set up shell autocompletion",
+	Aliases: []string{"ac"},
+	Description: `
+This command helps set up autocompletion in your shell by downloading the
+relevant autocompletion files and laying out what additional information must
+be loaded.
+
+Example:
+
+    abra autocomplete bash
+
+Supported shells are as follows:
+
+		fizsh
+		zsh
+    bash
+`,
+	ArgsUsage: "<shell>",
+	Action: func(c *cli.Context) error {
+		shellType := c.Args().First()
+
+		if shellType == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
+		}
+
+		supportedShells := map[string]bool{
+			"bash":  true,
+			"zsh":   true,
+			"fizsh": true,
+		}
+
+		if _, ok := supportedShells[shellType]; !ok {
+			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
+		}
+
+		if shellType == "fizsh" {
+			shellType = "zsh" // handled the same on the autocompletion side
+		}
+
+		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
+		if err := os.Mkdir(autocompletionDir, 0755); err != nil {
+			if !os.IsExist(err) {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("'%s' already created, moving on...", autocompletionDir)
+		}
+
+		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
+		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
+			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
+			logrus.Infof("fetching %s", url)
+			if err := downloadFile(autocompletionFile, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		switch shellType {
+		case "bash":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install autocompletion
+sudo mkdir /etc/bash/completion.d/
+sudo cp %s /etc/bash_completion.d/abra
+echo "source /etc/bash/completion.d/abra" >> ~/.bashrc
+`, autocompletionFile))
+		case "zsh":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install autocompletion
+sudo mkdir /etc/zsh/completion.d/
+sudo cp %s /etc/zsh/completion.d/abra
+echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
+`, autocompletionFile))
+		}
+
+		return nil
+	},
+}

cli/catalogue/catalogue.go

@@ -1,317 +1,17 @@
 package catalogue
 
 import (
-	"encoding/json"
+	"github.com/urfave/cli/v2"
-	"fmt"
-	"io/ioutil"
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	gitPkg "coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/limit"
-	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                   true,
-	"abra-apps":              true,
-	"abra-aur":               true,
-	"abra-bash":              true,
-	"abra-capsul":            true,
-	"abra-gandi":             true,
-	"abra-hetzner":           true,
-	"apps":                   true,
-	"aur-abra-git":           true,
-	"auto-apps-json":         true,
-	"auto-mirror":            true,
-	"backup-bot":             true,
-	"backup-bot-two":         true,
-	"beta.coopcloud.tech":    true,
-	"comrade-renovate-bot":   true,
-	"coopcloud.tech":         true,
-	"coturn":                 true,
-	"docker-cp-deploy":       true,
-	"docker-dind-bats-kcov":  true,
-	"docs.coopcloud.tech":    true,
-	"drone-abra":             true,
-	"example":                true,
-	"gardening":              true,
-	"go-abra":                true,
-	"organising":             true,
-	"outline-with-patch":     true,
-	"pyabra":                 true,
-	"radicle-seed-node":      true,
-	"recipes-catalogue-json": true,
-	"recipes-wishlist":       true,
-	"recipes.coopcloud.tech": true,
-	"stack-ssh-deploy":       true,
-	"swarm-cronjob":          true,
-	"tagcmp":                 true,
-	"traefik-cert-dumper":    true,
-	"tyop":                   true,
-}
-
-var catalogueGenerateCommand = cli.Command{
-	Name:    "generate",
-	Aliases: []string{"g"},
-	Usage:   "Generate the recipe catalogue",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.PublishFlag,
-		internal.DryFlag,
-		internal.SkipUpdatesFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-This command generates a new copy of the recipe catalogue which can be found on:
-
-    https://recipes.coopcloud.tech
-
-It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
-listing, parses README.md and git tags of those repositories to produce recipe
-metadata and produces a recipes JSON file.
-
-It is possible to generate new metadata for a single recipe by passing
-<recipe>. The existing local catalogue will be updated, not overwritten.
-
-It is quite easy to get rate limited by Docker Hub when running this command.
-If you have a Hub account you can have Abra log you in to avoid this. Pass
-"--user" and "--pass".
-
-Push your new release git.coopcloud.tech with "-p/--publish". This requires
-that you have permission to git push to these repositories and have your SSH
-keys configured on your account.
-`,
-	ArgsUsage: "[<recipe>]",
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-		if recipeName != "" {
-			internal.ValidateRecipe(c, true)
-		}
-
-		repos, err := recipe.ReadReposMetadata()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var barLength int
-		var logMsg string
-		if recipeName != "" {
-			barLength = 1
-			logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
-		} else {
-			barLength = len(repos)
-			logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
-		}
-
-		if !internal.SkipUpdates {
-			logrus.Warn(logMsg)
-			if err := updateRepositories(repos, recipeName); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		catl := make(recipe.RecipeCatalogue)
-		catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
-		for _, recipeMeta := range repos {
-			if recipeName != "" && recipeName != recipeMeta.Name {
-				catlBar.Add(1)
-				continue
-			}
-
-			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
-			versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
-			if err != nil {
-				logrus.Warn(err)
-			}
-
-			features, category, err := recipe.GetRecipeFeaturesAndCategory(recipeMeta.Name)
-			if err != nil {
-				logrus.Warn(err)
-			}
-
-			catl[recipeMeta.Name] = recipe.RecipeMeta{
-				Name:          recipeMeta.Name,
-				Repository:    recipeMeta.CloneURL,
-				SSHURL:        recipeMeta.SSHURL,
-				Icon:          recipeMeta.AvatarURL,
-				DefaultBranch: recipeMeta.DefaultBranch,
-				Description:   recipeMeta.Description,
-				Website:       recipeMeta.Website,
-				Versions:      versions,
-				Category:      category,
-				Features:      features,
-			}
-
-			catlBar.Add(1)
-		}
-
-		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if recipeName == "" {
-			if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			catlFS, err := recipe.ReadRecipeCatalogue()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			catlFS[recipeName] = catl[recipeName]
-
-			updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		logrus.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)
-
-		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
-		if internal.Publish {
-
-			isClean, err := gitPkg.IsClean(cataloguePath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if isClean {
-				if !internal.Dry {
-					logrus.Fatalf("no changes discovered in %s, nothing to publish?", cataloguePath)
-				}
-			}
-
-			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
-				logrus.Fatal(err)
-			}
-
-			repo, err := git.PlainOpen(cataloguePath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			sshURL := fmt.Sprintf(config.SSH_URL_TEMPLATE, config.CATALOGUE_JSON_REPO_NAME)
-			if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		repo, err := git.PlainOpen(cataloguePath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		head, err := repo.Head()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Dry && internal.Publish {
-			url := fmt.Sprintf("%s/%s/commit/%s", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME, head.Hash())
-			logrus.Infof("new changes published: %s", url)
-		}
-
-		if internal.Dry {
-			logrus.Info("dry run: no changes published")
-		}
-
-		return nil
-	},
-	BashComplete: autocomplete.RecipeNameComplete,
-}
-
 // CatalogueCommand defines the `abra catalogue` command and sub-commands.
-var CatalogueCommand = cli.Command{
+var CatalogueCommand = &cli.Command{
 	Name:        "catalogue",
-	Usage:       "Manage the recipe catalogue",
+	Usage:       "Manage the recipe catalogue (for maintainers)",
 	Aliases:     []string{"c"},
 	ArgsUsage:   "<recipe>",
 	Description: "This command helps recipe packagers interact with the recipe catalogue",
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		catalogueGenerateCommand,
 	},
 }
-
-func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error {
-	var barLength int
-	if recipeName != "" {
-		barLength = 1
-	} else {
-		barLength = len(repos)
-	}
-
-	cloneLimiter := limit.New(10)
-
-	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
-	ch := make(chan string, barLength)
-	for _, repoMeta := range repos {
-		go func(rm recipe.RepoMeta) {
-			cloneLimiter.Begin()
-			defer cloneLimiter.End()
-
-			if recipeName != "" && recipeName != rm.Name {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
-			if _, exists := CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
-
-			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
-			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
-				logrus.Fatal(err)
-			}
-
-			isClean, err := gitPkg.IsClean(recipeDir)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !isClean {
-				logrus.Fatalf("%s has locally unstaged changes", rm.Name)
-			}
-
-			if err := recipe.EnsureUpToDate(rm.Name); err != nil {
-				logrus.Fatal(err)
-			}
-
-			ch <- rm.Name
-			retrieveBar.Add(1)
-		}(repoMeta)
-	}
-
-	for range repos {
-		<-ch // wait for everything
-	}
-
-	return nil
-}

cli/catalogue/generate.go

@@ -0,0 +1,282 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/limit"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                  true,
+	"abra-apps":             true,
+	"abra-aur":              true,
+	"abra-bash":             true,
+	"abra-capsul":           true,
+	"abra-gandi":            true,
+	"abra-hetzner":          true,
+	"apps":                  true,
+	"aur-abra-git":          true,
+	"auto-apps-json":        true,
+	"auto-mirror":           true,
+	"backup-bot":            true,
+	"backup-bot-two":        true,
+	"comrade-renovate-bot":  true,
+	"coopcloud.tech":        true,
+	"coturn":                true,
+	"docker-cp-deploy":      true,
+	"docker-dind-bats-kcov": true,
+	"docs.coopcloud.tech":   true,
+	"drone-abra":            true,
+	"example":               true,
+	"gardening":             true,
+	"go-abra":               true,
+	"organising":            true,
+	"pyabra":                true,
+	"radicle-seed-node":     true,
+	"recipes":               true,
+	"stack-ssh-deploy":      true,
+	"swarm-cronjob":         true,
+	"tagcmp":                true,
+	"traefik-cert-dumper":   true,
+	"tyop":                  true,
+}
+
+var commit bool
+var commitFlag = &cli.BoolFlag{
+	Name:        "commit",
+	Usage:       "Commits new generated catalogue changes",
+	Value:       false,
+	Aliases:     []string{"c"},
+	Destination: &commit,
+}
+
+var catalogueGenerateCommand = &cli.Command{
+	Name:    "generate",
+	Aliases: []string{"g"},
+	Usage:   "Generate a new copy of the catalogue",
+	Flags: []cli.Flag{
+		internal.PushFlag,
+		commitFlag,
+		internal.CommitMessageFlag,
+	},
+	Description: `
+This command generates a new copy of the recipe catalogue which can be found on:
+
+    https://recipes.coopcloud.tech
+
+It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
+listing, parses README and tags to produce recipe metadata and produces a
+apps.json file which is placed in your ~/.abra/catalogue/recipes.json.
+
+It is possible to generate new metadata for a single recipe by passing
+<recipe>. The existing local catalogue will be updated, not overwritten.
+
+A new catalogue copy can be published to the recipes repository by passing the
+"--commit" and "--push" flags. The recipes repository is available here:
+
+    https://git.coopcloud.tech/coop-cloud/recipes
+
+`,
+	ArgsUsage: "[<recipe>]",
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		repos, err := catalogue.ReadReposMetadata()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
+
+		cloneLimiter := limit.New(10)
+		retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
+		ch := make(chan string, len(repos))
+		for _, repoMeta := range repos {
+			go func(rm catalogue.RepoMeta) {
+				cloneLimiter.Begin()
+				defer cloneLimiter.End()
+
+				if recipeName != "" && recipeName != rm.Name {
+					ch <- rm.Name
+					retrieveBar.Add(1)
+					return
+				}
+				if _, exists := CatalogueSkipList[rm.Name]; exists {
+					ch <- rm.Name
+					retrieveBar.Add(1)
+					return
+				}
+
+				recipeDir := path.Join(config.ABRA_DIR, "apps", rm.Name)
+
+				if err := gitPkg.Clone(recipeDir, rm.SSHURL); err != nil {
+					logrus.Fatal(err)
+				}
+
+				isClean, err := gitPkg.IsClean(rm.Name)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				if !isClean {
+					logrus.Fatalf("'%s' has locally unstaged changes", rm.Name)
+				}
+
+				if err := gitPkg.EnsureUpToDate(recipeDir); err != nil {
+					logrus.Fatal(err)
+				}
+
+				ch <- rm.Name
+				retrieveBar.Add(1)
+			}(repoMeta)
+		}
+
+		for range repos {
+			<-ch // wait for everything
+		}
+
+		catl := make(catalogue.RecipeCatalogue)
+		catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...")
+		for _, recipeMeta := range repos {
+			if recipeName != "" && recipeName != recipeMeta.Name {
+				catlBar.Add(1)
+				continue
+			}
+
+			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
+				catlBar.Add(1)
+				continue
+			}
+
+			versions, err := catalogue.GetRecipeVersions(recipeMeta.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			features, category, err := catalogue.GetRecipeFeaturesAndCategory(recipeMeta.Name)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
+			catl[recipeMeta.Name] = catalogue.RecipeMeta{
+				Name:          recipeMeta.Name,
+				Repository:    recipeMeta.CloneURL,
+				Icon:          recipeMeta.AvatarURL,
+				DefaultBranch: recipeMeta.DefaultBranch,
+				Description:   recipeMeta.Description,
+				Website:       recipeMeta.Website,
+				Versions:      versions,
+				Category:      category,
+				Features:      features,
+			}
+			catlBar.Add(1)
+		}
+
+		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if _, err := os.Stat(config.APPS_JSON); err != nil && os.IsNotExist(err) {
+			if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if recipeName != "" {
+				catlFS, err := catalogue.ReadRecipeCatalogue()
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				catlFS[recipeName] = catl[recipeName]
+
+				updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if err := ioutil.WriteFile(config.APPS_JSON, updatedRecipesJSON, 0644); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		cataloguePath := path.Join(config.ABRA_DIR, "catalogue", "recipes.json")
+		logrus.Infof("generated new recipe catalogue in %s", cataloguePath)
+
+		if commit {
+			repoPath := path.Join(config.ABRA_DIR, "catalogue")
+			commitRepo, err := git.PlainOpen(repoPath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			commitWorktree, err := commitRepo.Worktree()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if internal.CommitMessage == "" {
+				prompt := &survey.Input{
+					Message: "commit message",
+					Default: "chore: publish new catalogue changes",
+				}
+				if err := survey.AskOne(prompt, &internal.CommitMessage); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			err = commitWorktree.AddGlob("**.json")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debug("staged **.json for commit")
+
+			_, err = commitWorktree.Commit(internal.CommitMessage, &git.CommitOptions{})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info("changes commited")
+
+			if err := commitRepo.Push(&git.PushOptions{}); err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info("changes pushed")
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+}

cli/cli.go

@@ -2,10 +2,8 @@
 package cli
 
 import (
-	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path"
 
 	"coopcloud.tech/abra/cli/app"
@@ -14,145 +12,38 @@ import (
 	"coopcloud.tech/abra/cli/recipe"
 	"coopcloud.tech/abra/cli/record"
 	"coopcloud.tech/abra/cli/server"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/web"
+	logrusStack "github.com/Gurpartap/logrus-stack"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// AutoCompleteCommand helps people set up auto-complete in their shells
+// Verbose stores the variable from VerboseFlag.
-var AutoCompleteCommand = cli.Command{
+var Verbose bool
-	Name:    "autocomplete",
-	Aliases: []string{"ac"},
-	Usage:   "Configure shell autocompletion (recommended)",
-	Description: `
-This command helps set up autocompletion in your shell by downloading the
-relevant autocompletion files and laying out what additional information must
-be loaded.
 
-Example:
+// VerboseFlag turns on/off verbose logging down to the INFO level.
-
+var VerboseFlag = &cli.BoolFlag{
-    abra autocomplete bash
+	Name:        "verbose",
-
+	Aliases:     []string{"V"},
-Supported shells are as follows:
+	Value:       false,
-
+	Destination: &Verbose,
-		fizsh
+	Usage:       "Show INFO messages",
-		zsh
-    bash
-`,
-	ArgsUsage: "<shell>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Action: func(c *cli.Context) error {
-		shellType := c.Args().First()
-
-		if shellType == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
-		}
-
-		supportedShells := map[string]bool{
-			"bash":  true,
-			"zsh":   true,
-			"fizsh": true,
-		}
-
-		if _, ok := supportedShells[shellType]; !ok {
-			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
-		}
-
-		if shellType == "fizsh" {
-			shellType = "zsh" // handled the same on the autocompletion side
-		}
-
-		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
-		if err := os.Mkdir(autocompletionDir, 0764); err != nil {
-			if !os.IsExist(err) {
-				logrus.Fatal(err)
-			}
-			logrus.Debugf("%s already created", autocompletionDir)
-		}
-
-		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
-		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
-			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
-			logrus.Infof("fetching %s", url)
-			if err := web.GetFile(autocompletionFile, url); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		switch shellType {
-		case "bash":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install autocompletion
-sudo mkdir /etc/bash_completion.d/
-sudo cp %s /etc/bash_completion.d/abra
-echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
-`, autocompletionFile))
-		case "zsh":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install autocompletion
-sudo mkdir /etc/zsh/completion.d/
-sudo cp %s /etc/zsh/completion.d/abra
-echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
-# And finally run "abra app ps <hit tab key>" to test things are working, you should see app domains listed!
-`, autocompletionFile))
-		}
-
-		return nil
-	},
-}
-
-// UpgradeCommand upgrades abra in-place.
-var UpgradeCommand = cli.Command{
-	Name:    "upgrade",
-	Aliases: []string{"u"},
-	Usage:   "Upgrade Abra itself",
-	Description: `
-This command allows you to upgrade Abra in-place with the latest stable or
-release candidate.
-
-If you would like to install the latest release candidate, please pass the
-"-r/--rc" option. Please bear in mind that the latest release candidate may
-have some catastrophic bugs contained in it. In any case, thank you very much
-for the testing efforts!
-`,
-	Flags: []cli.Flag{internal.RCFlag},
-	Action: func(c *cli.Context) error {
-		mainURL := "https://install.abra.coopcloud.tech"
-		cmd := exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash", mainURL))
-
-		if internal.RC {
-			releaseCandidateURL := "https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer"
-			cmd = exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash -s -- --rc", releaseCandidateURL))
-		}
-
-		logrus.Debugf("attempting to run %s", cmd)
-
-		if err := internal.RunCmd(cmd); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
 }
 
 func newAbraApp(version, commit string) *cli.App {
 	app := &cli.App{
 		Name: "abra",
 		Usage: `The Co-op Cloud command-line utility belt 🎩🐇
+
     ____                           ____ _                 _
    / ___|___         ___  _ __    / ___| | ___  _   _  __| |
   | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
   | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
    \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
                          |_|
-`,
+		`,
 		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
-		Commands: []cli.Command{
+		Commands: []*cli.Command{
 			app.AppCommand,
 			server.ServerCommand,
 			recipe.RecipeCommand,
@@ -161,29 +52,48 @@ func newAbraApp(version, commit string) *cli.App {
 			UpgradeCommand,
 			AutoCompleteCommand,
 		},
-		BashComplete: autocomplete.SubcommandComplete,
+		Flags: []cli.Flag{
+			VerboseFlag,
+			internal.DebugFlag,
+			internal.NoInputFlag,
+		},
+		Authors: []*cli.Author{
+			{Name: "3wordchant"},
+			{Name: "decentral1se"},
+			{Name: "knoflook"},
+			{Name: "roxxers"},
+		},
 	}
 
 	app.EnableBashCompletion = true
 
 	app.Before = func(c *cli.Context) error {
+		if internal.Debug {
+			logrus.SetLevel(logrus.DebugLevel)
+			logrus.SetFormatter(&logrus.TextFormatter{})
+			logrus.SetOutput(os.Stderr)
+			logrus.AddHook(logrusStack.StandardHook())
+		}
+
 		paths := []string{
 			config.ABRA_DIR,
-			path.Join(config.SERVERS_DIR),
+			path.Join(config.ABRA_DIR, "servers"),
-			path.Join(config.RECIPES_DIR),
+			path.Join(config.ABRA_DIR, "apps"),
-			path.Join(config.VENDOR_DIR),
+			path.Join(config.ABRA_DIR, "vendor"),
 		}
 
 		for _, path := range paths {
-			if err := os.Mkdir(path, 0764); err != nil {
+			if err := os.Mkdir(path, 0755); err != nil {
 				if !os.IsExist(err) {
 					logrus.Fatal(err)
 				}
+				logrus.Debugf("'%s' already created, moving on...", path)
 				continue
 			}
+			logrus.Debugf("'%s' is missing, creating...", path)
 		}
 
-		logrus.Debugf("abra version %s, commit %s", version, commit)
+		logrus.Debugf("abra version '%s', commit '%s'", version, commit)
 
 		return nil
 	}

cli/formatter/formatter.go

@@ -1,20 +1,25 @@
 package formatter
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/docker/cli/cli/command/formatter"
 	"github.com/docker/go-units"
 	"github.com/olekukonko/tablewriter"
 	"github.com/schollz/progressbar/v3"
-	"github.com/sirupsen/logrus"
 )
 
 func ShortenID(str string) string {
 	return str[:12]
 }
 
+func Truncate(str string) string {
+	return fmt.Sprintf(`"%s"`, formatter.Ellipsis(str, 19))
+}
+
 func SmallSHA(hash string) string {
 	return hash[:8]
 }
@@ -34,7 +39,6 @@ func HumanDuration(timestamp int64) string {
 // CreateTable prepares a table layout for output.
 func CreateTable(columns []string) *tablewriter.Table {
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetAutoWrapText(false)
 	table.SetHeader(columns)
 	return table
 }
@@ -50,22 +54,3 @@ func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
 		progressbar.OptionSetDescription(title),
 	)
 }
-
-// StripTagMeta strips front-matter image tag data that we don't need for parsing.
-func StripTagMeta(image string) string {
-	originalImage := image
-
-	if strings.Contains(image, "docker.io") {
-		image = strings.Split(image, "/")[1]
-	}
-
-	if strings.Contains(image, "library") {
-		image = strings.Split(image, "/")[1]
-	}
-
-	if originalImage != image {
-		logrus.Debugf("stripped %s to %s for parsing", originalImage, image)
-	}
-
-	return image
-}

cli/internal/common.go

@@ -1,11 +1,7 @@
 package internal
 
 import (
-	"os"
+	"github.com/urfave/cli/v2"
-
-	logrusStack "github.com/Gurpartap/logrus-stack"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
 // Secrets stores the variable from SecretsFlag
@@ -13,7 +9,9 @@ var Secrets bool
 
 // SecretsFlag turns on/off automatically generating secrets
 var SecretsFlag = &cli.BoolFlag{
-	Name:        "secrets, S",
+	Name:        "secrets",
+	Aliases:     []string{"S"},
+	Value:       false,
 	Usage:       "Automatically generate secrets",
 	Destination: &Secrets,
 }
@@ -23,19 +21,22 @@ var Pass bool
 
 // PassFlag turns on/off storing generated secrets in pass
 var PassFlag = &cli.BoolFlag{
-	Name:        "pass, p",
+	Name:        "pass",
+	Aliases:     []string{"P"},
+	Value:       false,
 	Usage:       "Store the generated secrets in a local pass store",
 	Destination: &Pass,
 }
 
-// PassRemove stores the variable for PassRemoveFlag
+// Context is temp
-var PassRemove bool
+var Context string
 
-// PassRemoveFlag turns on/off removing generated secrets from pass
+// ContextFlag is temp
-var PassRemoveFlag = &cli.BoolFlag{
+var ContextFlag = &cli.StringFlag{
-	Name:        "pass, p",
+	Name:        "context",
-	Usage:       "Remove generated secrets from a local pass store",
+	Value:       "",
-	Destination: &PassRemove,
+	Aliases:     []string{"c"},
+	Destination: &Context,
 }
 
 // Force force functionality without asking.
@@ -43,8 +44,9 @@ var Force bool
 
 // ForceFlag turns on/off force functionality.
 var ForceFlag = &cli.BoolFlag{
-	Name:        "force, f",
+	Name:        "force",
-	Usage:       "Perform action without further prompt. Use with care!",
+	Value:       false,
+	Aliases:     []string{"f"},
 	Destination: &Force,
 }
 
@@ -53,7 +55,9 @@ var Chaos bool
 
 // ChaosFlag turns on/off chaos functionality.
 var ChaosFlag = &cli.BoolFlag{
-	Name:        "chaos, C",
+	Name:        "chaos",
+	Value:       false,
+	Aliases:     []string{"ch"},
 	Usage:       "Deploy uncommitted recipes changes. Use with care!",
 	Destination: &Chaos,
 }
@@ -63,15 +67,18 @@ var DNSProvider string
 
 // DNSProviderFlag selects a DNS provider.
 var DNSProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
+	Name:        "provider",
 	Value:       "",
+	Aliases:     []string{"p"},
 	Usage:       "DNS provider",
 	Destination: &DNSProvider,
 }
 
 var NoInput bool
 var NoInputFlag = &cli.BoolFlag{
-	Name:        "no-input, n",
+	Name:        "no-input",
+	Value:       false,
+	Aliases:     []string{"n"},
 	Usage:       "Toggle non-interactive mode",
 	Destination: &NoInput,
 }
@@ -79,8 +86,9 @@ var NoInputFlag = &cli.BoolFlag{
 var DNSType string
 
 var DNSTypeFlag = &cli.StringFlag{
-	Name:        "record-type, rt",
+	Name:        "type",
 	Value:       "",
+	Aliases:     []string{"t"},
 	Usage:       "Domain name record type (e.g. A)",
 	Destination: &DNSType,
 }
@@ -88,8 +96,9 @@ var DNSTypeFlag = &cli.StringFlag{
 var DNSName string
 
 var DNSNameFlag = &cli.StringFlag{
-	Name:        "record-name, rn",
+	Name:        "name",
 	Value:       "",
+	Aliases:     []string{"n"},
 	Usage:       "Domain name record name (e.g. mysubdomain)",
 	Destination: &DNSName,
 }
@@ -97,25 +106,29 @@ var DNSNameFlag = &cli.StringFlag{
 var DNSValue string
 
 var DNSValueFlag = &cli.StringFlag{
-	Name:        "record-value, rv",
+	Name:        "value",
 	Value:       "",
+	Aliases:     []string{"v"},
 	Usage:       "Domain name record value (e.g. 192.168.1.1)",
 	Destination: &DNSValue,
 }
 
-var DNSTTL string
+var DNSTTL int
-var DNSTTLFlag = &cli.StringFlag{
+
-	Name:        "record-ttl, rl",
+var DNSTTLFlag = &cli.IntFlag{
-	Value:       "600s",
+	Name:        "ttl",
-	Usage:       "Domain name TTL value (seconds)",
+	Value:       86400,
+	Aliases:     []string{"T"},
+	Usage:       "Domain name TTL value)",
 	Destination: &DNSTTL,
 }
 
 var DNSPriority int
 
 var DNSPriorityFlag = &cli.IntFlag{
-	Name:        "record-priority, rp",
+	Name:        "priority",
 	Value:       10,
+	Aliases:     []string{"P"},
 	Usage:       "Domain name priority value",
 	Destination: &DNSPriority,
 }
@@ -123,7 +136,8 @@ var DNSPriorityFlag = &cli.IntFlag{
 var ServerProvider string
 
 var ServerProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
+	Name:        "provider",
+	Aliases:     []string{"p"},
 	Usage:       "3rd party server provider",
 	Destination: &ServerProvider,
 }
@@ -131,8 +145,9 @@ var ServerProviderFlag = &cli.StringFlag{
 var CapsulInstanceURL string
 
 var CapsulInstanceURLFlag = &cli.StringFlag{
-	Name:        "capsul-url, cu",
+	Name:        "capsul-url",
 	Value:       "yolo.servers.coop",
+	Aliases:     []string{"cu"},
 	Usage:       "capsul instance URL",
 	Destination: &CapsulInstanceURL,
 }
@@ -140,8 +155,9 @@ var CapsulInstanceURLFlag = &cli.StringFlag{
 var CapsulName string
 
 var CapsulNameFlag = &cli.StringFlag{
-	Name:        "capsul-name, cn",
+	Name:        "capsul-name",
 	Value:       "",
+	Aliases:     []string{"cn"},
 	Usage:       "capsul name",
 	Destination: &CapsulName,
 }
@@ -149,8 +165,9 @@ var CapsulNameFlag = &cli.StringFlag{
 var CapsulType string
 
 var CapsulTypeFlag = &cli.StringFlag{
-	Name:        "capsul-type, ct",
+	Name:        "capsul-type",
 	Value:       "f1-xs",
+	Aliases:     []string{"ct"},
 	Usage:       "capsul type",
 	Destination: &CapsulType,
 }
@@ -158,33 +175,38 @@ var CapsulTypeFlag = &cli.StringFlag{
 var CapsulImage string
 
 var CapsulImageFlag = &cli.StringFlag{
-	Name:        "capsul-image, ci",
+	Name:        "capsul-image",
 	Value:       "debian10",
+	Aliases:     []string{"ci"},
 	Usage:       "capsul image",
 	Destination: &CapsulImage,
 }
 
 var CapsulSSHKeys cli.StringSlice
+
 var CapsulSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "capsul-ssh-keys, cs",
+	Name:        "capsul-ssh-keys",
+	Aliases:     []string{"cs"},
 	Usage:       "capsul SSH key",
-	Value: &CapsulSSHKeys,
+	Destination: &CapsulSSHKeys,
 }
 
 var CapsulAPIToken string
 
 var CapsulAPITokenFlag = &cli.StringFlag{
-	Name:        "capsul-token, ca",
+	Name:        "capsul-token",
+	Aliases:     []string{"ca"},
 	Usage:       "capsul API token",
-	EnvVar:      "CAPSUL_TOKEN",
+	EnvVars:     []string{"CAPSUL_TOKEN"},
 	Destination: &CapsulAPIToken,
 }
 
 var HetznerCloudName string
 
 var HetznerCloudNameFlag = &cli.StringFlag{
-	Name:        "hetzner-name, hn",
+	Name:        "hetzner-name",
 	Value:       "",
+	Aliases:     []string{"hn"},
 	Usage:       "hetzner cloud name",
 	Destination: &HetznerCloudName,
 }
@@ -192,7 +214,8 @@ var HetznerCloudNameFlag = &cli.StringFlag{
 var HetznerCloudType string
 
 var HetznerCloudTypeFlag = &cli.StringFlag{
-	Name:        "hetzner-type, ht",
+	Name:        "hetzner-type",
+	Aliases:     []string{"ht"},
 	Usage:       "hetzner cloud type",
 	Destination: &HetznerCloudType,
 	Value:       "cx11",
@@ -201,7 +224,8 @@ var HetznerCloudTypeFlag = &cli.StringFlag{
 var HetznerCloudImage string
 
 var HetznerCloudImageFlag = &cli.StringFlag{
-	Name:        "hetzner-image, hi",
+	Name:        "hetzner-image",
+	Aliases:     []string{"hi"},
 	Usage:       "hetzner cloud image",
 	Value:       "debian-10",
 	Destination: &HetznerCloudImage,
@@ -210,15 +234,17 @@ var HetznerCloudImageFlag = &cli.StringFlag{
 var HetznerCloudSSHKeys cli.StringSlice
 
 var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "hetzner-ssh-keys, hs",
+	Name:        "hetzner-ssh-keys",
+	Aliases:     []string{"hs"},
 	Usage:       "hetzner cloud SSH keys (e.g. me@foo.com)",
-	Value: &HetznerCloudSSHKeys,
+	Destination: &HetznerCloudSSHKeys,
 }
 
 var HetznerCloudLocation string
 
 var HetznerCloudLocationFlag = &cli.StringFlag{
-	Name:        "hetzner-location, hl",
+	Name:        "hetzner-location",
+	Aliases:     []string{"hl"},
 	Usage:       "hetzner cloud server location",
 	Value:       "hel1",
 	Destination: &HetznerCloudLocation,
@@ -227,9 +253,10 @@ var HetznerCloudLocationFlag = &cli.StringFlag{
 var HetznerCloudAPIToken string
 
 var HetznerCloudAPITokenFlag = &cli.StringFlag{
-	Name:        "hetzner-token, ha",
+	Name:        "hetzner-token",
+	Aliases:     []string{"ha"},
 	Usage:       "hetzner cloud API token",
-	EnvVar:      "HCLOUD_TOKEN",
+	EnvVars:     []string{"HCLOUD_TOKEN"},
 	Destination: &HetznerCloudAPIToken,
 }
 
@@ -238,121 +265,13 @@ var Debug bool
 
 // DebugFlag turns on/off verbose logging down to the DEBUG level.
 var DebugFlag = &cli.BoolFlag{
-	Name:        "debug, d",
+	Name:        "debug",
+	Aliases:     []string{"d"},
+	Value:       false,
 	Destination: &Debug,
 	Usage:       "Show DEBUG messages",
 }
 
-// RC signifies the latest release candidate
-var RC bool
-
-// RCFlag chooses the latest release candidate for install
-var RCFlag = &cli.BoolFlag{
-	Name:        "rc, r",
-	Destination: &RC,
-	Usage:       "Insatll the latest release candidate",
-}
-
-var Major bool
-var MajorFlag = &cli.BoolFlag{
-	Name:        "major, x",
-	Usage:       "Increase the major part of the version",
-	Destination: &Major,
-}
-
-var Minor bool
-var MinorFlag = &cli.BoolFlag{
-	Name:        "minor, y",
-	Usage:       "Increase the minor part of the version",
-	Destination: &Minor,
-}
-
-var Patch bool
-var PatchFlag = &cli.BoolFlag{
-	Name:        "patch, z",
-	Usage:       "Increase the patch part of the version",
-	Destination: &Patch,
-}
-
-var Dry bool
-var DryFlag = &cli.BoolFlag{
-	Name:        "dry-run, r",
-	Usage:       "Only reports changes that would be made",
-	Destination: &Dry,
-}
-
-var Publish bool
-var PublishFlag = &cli.BoolFlag{
-	Name:        "publish, p",
-	Usage:       "Publish changes to git.coopcloud.tech",
-	Destination: &Publish,
-}
-
-var Domain string
-var DomainFlag = &cli.StringFlag{
-	Name:        "domain, D",
-	Value:       "",
-	Usage:       "Choose a domain name",
-	Destination: &Domain,
-}
-
-var NewAppServer string
-var NewAppServerFlag = &cli.StringFlag{
-	Name:        "server, s",
-	Value:       "",
-	Usage:       "Show apps of a specific server",
-	Destination: &NewAppServer,
-}
-
-var NoDomainChecks bool
-var NoDomainChecksFlag = &cli.BoolFlag{
-	Name:        "no-domain-checks, D",
-	Usage:       "Disable app domain sanity checks",
-	Destination: &NoDomainChecks,
-}
-
-var StdErrOnly bool
-var StdErrOnlyFlag = &cli.BoolFlag{
-	Name:        "stderr, s",
-	Usage:       "Only tail stderr",
-	Destination: &StdErrOnly,
-}
-
-var DontWaitConverge bool
-var DontWaitConvergeFlag = &cli.BoolFlag{
-	Name:        "no-converge-checks, c",
-	Usage:       "Don't wait for converge logic checks",
-	Destination: &DontWaitConverge,
-}
-
-var Watch bool
-var WatchFlag = &cli.BoolFlag{
-	Name:        "watch, w",
-	Usage:       "Watch status by polling repeatedly",
-	Destination: &Watch,
-}
-
-var OnlyErrors bool
-var OnlyErrorFlag = &cli.BoolFlag{
-	Name:        "errors, e",
-	Usage:       "Only show errors",
-	Destination: &OnlyErrors,
-}
-
-var SkipUpdates bool
-var SkipUpdatesFlag = &cli.BoolFlag{
-	Name:        "skip-updates, s",
-	Usage:       "Skip updating recipe repositories",
-	Destination: &SkipUpdates,
-}
-
-var AllTags bool
-var AllTagsFlag = &cli.BoolFlag{
-	Name:        "all-tags, a",
-	Usage:       "List all tags, not just upgrades",
-	Destination: &AllTags,
-}
-
 // SSHFailMsg is a hopefully helpful SSH failure message
 var SSHFailMsg = `
 Woops, Abra is unable to connect to connect to %s.
@@ -375,10 +294,6 @@ If your SSH private key loaded? You can check by running the following command:
 
     ssh-add -L
 
-If, you can add it with:
-
-    ssh-add ~/.ssh/<private-key-part>
-
 If you are using a non-default public/private key, you can configure this in
 your ~/.ssh/config file which Abra will read in order to figure out connection
 details:
@@ -401,33 +316,3 @@ Host foo.coopcloud.tech
 Good luck!
 
 `
-
-var ServerAddFailMsg = `
-Failed to add server %s.
-
-This could be caused by two things.
-
-Abra isn't picking up your SSH configuration or you need to specify it on the
-command-line (e.g you use a non-standard port or username to connect). Run
-"server add" with "-d/--debug" to learn more about what Abra is doing under the
-hood.
-
-Docker is not installed on your server. You can pass "-p/--provision" to
-install Docker and initialise Docker Swarm mode. See help output for "server
-add"
-
-See "abra server add -h" for more.
-
-`
-
-// SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
-func SubCommandBefore(c *cli.Context) error {
-	if Debug {
-		logrus.SetLevel(logrus.DebugLevel)
-		logrus.SetFormatter(&logrus.TextFormatter{})
-		logrus.SetOutput(os.Stderr)
-		logrus.AddHook(logrusStack.StandardHook())
-	}
-
-	return nil
-}

cli/internal/copy.go

@@ -0,0 +1,74 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+func ConfigureAndCp(c *cli.Context, app config.App, srcPath string, dstPath string, service string, isToContainer bool) error {
+	appFiles, err := config.LoadAppFiles("")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	appEnv, err := config.GetApp(appFiles, app.Name)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
+	containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if len(containers) != 1 {
+		logrus.Fatalf("expected 1 container but got %v", len(containers))
+	}
+	container := containers[0]
+
+	logrus.Debugf("retrieved '%s' as target container on '%s'", formatter.ShortenID(container.ID), app.Server)
+
+	if isToContainer {
+		if _, err := os.Stat(srcPath); err != nil {
+			logrus.Fatalf("'%s' does not exist?", srcPath)
+		}
+
+		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		content, err := archive.TarWithOptions(srcPath, toTarOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+		if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		defer content.Close()
+		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+	return nil
+}

cli/internal/deploy.go

@@ -1,103 +1,80 @@
 package internal
 
 import (
-	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
 	"strings"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/dns"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // DeployAction is the main command-line action for this package
 func DeployAction(c *cli.Context) error {
 	app := ValidateApp(c)
-
+	stackName := app.StackName()
-	if !Chaos {
-		if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	r, err := recipe.Get(app.Recipe)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := lint.LintForErrors(r); err != nil {
-		logrus.Fatal(err)
-	}
 
 	cl, err := client.New(app.Server)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("checking whether %s is already deployed", app.StackName())
+	logrus.Debugf("checking whether %s is already deployed", stackName)
 
-	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+	isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
 	if isDeployed {
 		if Force || Chaos {
-			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
+			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", stackName)
 		} else {
-			logrus.Fatalf("%s is already deployed", app.Name)
+			logrus.Fatalf("%s is already deployed", stackName)
 		}
 	}
 
 	version := deployedVersion
-	if version == "unknown" && !Chaos {
+	if version == "" && !Chaos {
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		if len(versions) > 0 {
 			version = versions[len(versions)-1]
 			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+			if err := recipe.EnsureVersion(app.Type, version); err != nil {
 				logrus.Fatal(err)
 			}
 		} else {
-			head, err := git.GetRecipeHead(app.Recipe)
+			version = "latest commit"
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			version = formatter.SmallSHA(head.String())
 			logrus.Warn("no versions detected, using latest commit")
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Type); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 	}
 
-	if version == "unknown" && !Chaos {
+	if version == "" && !Chaos {
 		logrus.Debugf("choosing %s as version to deploy", version)
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+		if err := recipe.EnsureVersion(app.Type, version); err != nil {
 			logrus.Fatal(err)
 		}
 	}
 
-	if version != "unknown" && !Chaos {
+	if version != "" && !Chaos {
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+		if err := recipe.EnsureVersion(app.Type, version); err != nil {
 			logrus.Fatal(err)
 		}
 	}
@@ -105,13 +82,13 @@ func DeployAction(c *cli.Context) error {
 	if Chaos {
 		logrus.Warnf("chaos mode engaged")
 		var err error
-		version, err = recipe.ChaosVersion(app.Recipe)
+		version, err = recipe.ChaosVersion(app.Type)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 	}
 
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
 	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 	if err != nil {
 		logrus.Fatal(err)
@@ -120,13 +97,13 @@ func DeployAction(c *cli.Context) error {
 		app.Env[k] = v
 	}
 
-	composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+	composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 	deployOpts := stack.Deploy{
 		Composefiles: composeFiles,
-		Namespace:    app.StackName(),
+		Namespace:    stackName,
 		Prune:        false,
 		ResolveImage: stack.ResolveImageAlways,
 	}
@@ -139,16 +116,17 @@ func DeployAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	if !NoDomainChecks {
 	domainName := app.Env["DOMAIN"]
+	ipv4, err := dns.EnsureIPv4(domainName)
+	if err != nil || ipv4 == "" {
+		logrus.Fatalf("could not find an IP address assigned to %s?", domainName)
+	}
+
 	if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
 		logrus.Fatal(err)
 	}
-	} else {
-		logrus.Warn("skipping domain checks as requested")
-	}
 
-	if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, DontWaitConverge); err != nil {
+	if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
 		logrus.Fatal(err)
 	}
 
@@ -157,8 +135,8 @@ func DeployAction(c *cli.Context) error {
 
 // DeployOverview shows a deployment overview
 func DeployOverview(app config.App, version, message string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "version"}
+	tableCol := []string{"server", "compose", "domain", "stack", "version"}
-	table := formatter.CreateTable(tableCol)
+	table := abraFormatter.CreateTable(tableCol)
 
 	deployConfig := "compose.yml"
 	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
@@ -170,7 +148,7 @@ func DeployOverview(app config.App, version, message string) error {
 		server = "local"
 	}
 
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, version})
+	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), version})
 	table.Render()
 
 	if NoInput {
@@ -194,9 +172,9 @@ func DeployOverview(app config.App, version, message string) error {
 }
 
 // NewVersionOverview shows an upgrade or downgrade overview
-func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
+func NewVersionOverview(app config.App, currentVersion, newVersion string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
+	tableCol := []string{"server", "compose", "domain", "stack", "current version", "to be deployed"}
-	table := formatter.CreateTable(tableCol)
+	table := abraFormatter.CreateTable(tableCol)
 
 	deployConfig := "compose.yml"
 	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
@@ -208,24 +186,9 @@ func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes
 		server = "local"
 	}
 
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
+	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), currentVersion, newVersion})
 	table.Render()
 
-	if releaseNotes == "" {
-		var err error
-		releaseNotes, err = GetReleaseNotes(app.Recipe, newVersion)
-		if err != nil {
-			return err
-		}
-	}
-
-	if releaseNotes != "" && newVersion != "" {
-		fmt.Println()
-		fmt.Println(fmt.Sprintf("%s release notes:\n\n%s", newVersion, releaseNotes))
-	} else {
-		logrus.Warnf("no release notes available for %s", newVersion)
-	}
-
 	if NoInput {
 		return nil
 	}
@@ -245,22 +208,3 @@ func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes
 
 	return nil
 }
-
-// GetReleaseNotes prints release notes for a recipe version
-func GetReleaseNotes(recipeName, version string) (string, error) {
-	if version == "" {
-		return "", nil
-	}
-
-	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
-
-	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
-		releaseNotes, err := ioutil.ReadFile(fpath)
-		if err != nil {
-			return "", err
-		}
-		return string(releaseNotes), nil
-	}
-
-	return "", nil
-}

cli/internal/errors.go

@@ -4,7 +4,7 @@ import (
 	"os"
 
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // ShowSubcommandHelpAndError exits the program on error, logs the error to the

cli/internal/errors_test.go

@@ -0,0 +1,38 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Testing functions that call os.Exit
+// https://stackoverflow.com/questions/26225513/how-to-test-os-exit-scenarios-in-go
+// https://talks.golang.org/2014/testing.slide#23
+
+var testapp = &cli.App{
+	Name:  "abra",
+	Usage: `The Co-op Cloud command-line utility belt 🎩🐇`,
+}
+
+// not testing output as that changes. just if it exits with code 1
+// does not work because of some weird errors on cli's part. Its a hard lib to test effectively.
+// func TestShowSubcommandHelpAndError(t *testing.T) {
+// 	if os.Getenv("HelpAndError") == "1" {
+// 		ShowSubcommandHelpAndError(cli.NewContext(testapp, nil, nil), errors.New("Test error"))
+// 		return
+// 	}
+// 	cmd := exec.Command(os.Args[0], "-test.run=TestShowSubcommandHelpAndError")
+// 	cmd.Env = append(os.Environ(), "HelpAndError=1")
+// 	var out bytes.Buffer
+// 	cmd.Stderr = &out
+// 	err := cmd.Run()
+// 	println(out.String())
+// 	if !strings.Contains(out.String(), "Test error") {
+
+// 		t.Fatalf("expected command to show the error causing the exit, did not get correct stdout output")
+// 	}
+
+// 	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
+// 		return
+// 	}
+// 	t.Fatalf("process ran with err %v, want exit status 1", err)
+// }

cli/internal/list.go

@@ -1,10 +0,0 @@
-package internal
-
-// ReverseStringList reverses a list of a strings. Roll on Go generics.
-func ReverseStringList(strings []string) []string {
-	for i, j := 0, len(strings)-1; i < j; i, j = i+1, j-1 {
-		strings[i], strings[j] = strings[j], strings[i]
-	}
-
-	return strings
-}

cli/internal/logs.go

@@ -0,0 +1,68 @@
+package internal
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Stderr bool
+var StderrFlag = &cli.BoolFlag{
+	Name:        "stderr",
+	Aliases:     []string{"e"},
+	Value:       false,
+	Destination: &Stderr,
+}
+
+var Stdout bool
+var StdoutFlag = &cli.BoolFlag{
+	Name:        "stdout",
+	Aliases:     []string{"o"},
+	Value:       false,
+	Destination: &Stdout,
+}
+
+var Healthcheck bool
+var HealthcheckFlag = &cli.BoolFlag{
+	Name:        "healthcheck",
+	Aliases:     []string{"c"},
+	Value:       false,
+	Destination: &Healthcheck,
+}
+
+// StackLogs lists logs for all stack services
+func StackLogs(c *cli.Context, stackName string, logOpts types.ContainerLogsOptions, client *dockerClient.Client) {
+	filters := filters.NewArgs()
+	filters.Add("name", stackName)
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(c.Context, serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			logs, err := client.ServiceLogs(c.Context, s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			// defer after err check as any err returns a nil io.ReadCloser
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+	wg.Wait()
+	os.Exit(0)
+}

cli/internal/new.go

@@ -4,28 +4,51 @@ import (
 	"fmt"
 	"path"
 
-	"coopcloud.tech/abra/pkg/app"
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/secret"
 	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// AppSecrets represents all app secrest
 type AppSecrets map[string]string
 
+var Domain string
+var DomainFlag = &cli.StringFlag{
+	Name:        "domain",
+	Aliases:     []string{"d"},
+	Value:       "",
+	Usage:       "Choose a domain name",
+	Destination: &Domain,
+}
+
+var NewAppServer string
+var NewAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &NewAppServer,
+}
+
+var NewAppName string
+var NewAppNameFlag = &cli.StringFlag{
+	Name:        "app-name",
+	Aliases:     []string{"a"},
+	Value:       "",
+	Usage:       "Choose an app name",
+	Destination: &NewAppName,
+}
+
 // RecipeName is used for configuring recipe name programmatically
 var RecipeName string
 
 // createSecrets creates all secrets for a new app.
 func createSecrets(sanitisedAppName string) (AppSecrets, error) {
-	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", Domain))
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", sanitisedAppName))
 	appEnv, err := config.ReadEnv(appEnvPath)
 	if err != nil {
 		return nil, err
@@ -40,7 +63,7 @@ func createSecrets(sanitisedAppName string) (AppSecrets, error) {
 	if Pass {
 		for secretName := range secrets {
 			secretValue := secrets[secretName]
-			if err := secret.PassInsertSecret(secretValue, secretName, Domain, NewAppServer); err != nil {
+			if err := secret.PassInsertSecret(secretValue, secretName, sanitisedAppName, NewAppServer); err != nil {
 				return nil, err
 			}
 		}
@@ -67,31 +90,6 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 	return nil
 }
 
-// promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(appName string) error {
-	app, err := app.Get(appName)
-	if err != nil {
-		return err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(app.Env)
-	if len(secretEnvVars) == 0 {
-		logrus.Debugf("%s has no secrets to generate, skipping...", app.Recipe)
-		return nil
-	}
-
-	if !Secrets && !NoInput {
-		prompt := &survey.Confirm{
-			Message: "Generate app secrets?",
-		}
-		if err := survey.AskOne(prompt, &Secrets); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // ensureServerFlag checks if the server flag was used. if not, asks the user for it.
 func ensureServerFlag() error {
 	servers, err := config.GetServers()
@@ -116,11 +114,30 @@ func ensureServerFlag() error {
 	return nil
 }
 
+// ensureServerFlag checks if the AppName flag was used. if not, asks the user for it.
+func ensureAppNameFlag() error {
+	if NewAppName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify app name:",
+			Default: config.SanitiseAppName(Domain),
+		}
+		if err := survey.AskOne(prompt, &NewAppName); err != nil {
+			return err
+		}
+	}
+
+	if NewAppName == "" {
+		return fmt.Errorf("no app name provided")
+	}
+
+	return nil
+}
+
 // NewAction is the new app creation logic
 func NewAction(c *cli.Context) error {
-	recipe := ValidateRecipeWithPrompt(c, false)
+	recipe := ValidateRecipeWithPrompt(c)
 
-	if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+	if err := config.EnsureAbraDirExists(); err != nil {
 		logrus.Fatal(err)
 	}
 
@@ -132,45 +149,48 @@ func NewAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	if err := promptForSecrets(Domain); err != nil {
+	if err := ensureAppNameFlag(); err != nil {
 		logrus.Fatal(err)
 	}
 
-	sanitisedAppName := config.SanitiseAppName(Domain)
+	sanitisedAppName := config.SanitiseAppName(NewAppName)
-	logrus.Debugf("%s sanitised as %s for new app", Domain, sanitisedAppName)
+	if len(sanitisedAppName) > 45 {
+		logrus.Fatalf("'%s' cannot be longer than 45 characters", sanitisedAppName)
+	}
+	logrus.Debugf("'%s' sanitised as '%s' for new app", NewAppName, sanitisedAppName)
 
-	if err := config.TemplateAppEnvSample(recipe.Name, Domain, NewAppServer, Domain); err != nil {
+	if err := config.TemplateAppEnvSample(recipe.Name, NewAppName, NewAppServer, Domain, recipe.Name); err != nil {
 		logrus.Fatal(err)
 	}
 
-	var secrets AppSecrets
-	var secretTable *tablewriter.Table
 	if Secrets {
 		if err := ssh.EnsureHostKey(NewAppServer); err != nil {
 			logrus.Fatal(err)
 		}
 
-		var err error
+		secrets, err := createSecrets(sanitisedAppName)
-		secrets, err = createSecrets(sanitisedAppName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		secretCols := []string{"Name", "Value"}
-		secretTable = formatter.CreateTable(secretCols)
+		secretTable := abraFormatter.CreateTable(secretCols)
 		for secret := range secrets {
 			secretTable.Append([]string{secret, secrets[secret]})
 		}
 
+		if len(secrets) > 0 {
+			defer secretTable.Render()
+		}
 	}
 
 	if NewAppServer == "default" {
 		NewAppServer = "local"
 	}
 
-	tableCol := []string{"server", "recipe", "domain"}
+	tableCol := []string{"Name", "Domain", "Type", "Server"}
-	table := formatter.CreateTable(tableCol)
+	table := abraFormatter.CreateTable(tableCol)
-	table.Append([]string{NewAppServer, recipe.Name, Domain})
+	table.Append([]string{sanitisedAppName, Domain, recipe.Name, NewAppServer})
 
 	fmt.Println("")
 	fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
@@ -178,19 +198,11 @@ func NewAction(c *cli.Context) error {
 	table.Render()
 	fmt.Println("")
 	fmt.Println("You can configure this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app config %s", Domain))
+	fmt.Println(fmt.Sprintf("\n    abra app config %s", sanitisedAppName))
 	fmt.Println("")
 	fmt.Println("You can deploy this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", Domain))
+	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", sanitisedAppName))
 	fmt.Println("")
 
-	if len(secrets) > 0 {
-		fmt.Println("Here are your generated secrets:")
-		fmt.Println("")
-		secretTable.Render()
-		fmt.Println("")
-		logrus.Warn("generated secrets are not shown again, please take note of them *now*")
-	}
-
 	return nil
 }

cli/internal/recipe.go

@@ -2,55 +2,104 @@ package internal
 
 import (
 	"fmt"
+	"strings"
 
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
 )
 
+var Major bool
+var MajorFlag = &cli.BoolFlag{
+	Name:        "major",
+	Usage:       "Increase the major part of the version",
+	Value:       false,
+	Aliases:     []string{"ma", "x"},
+	Destination: &Major,
+}
+
+var Minor bool
+var MinorFlag = &cli.BoolFlag{
+	Name:        "minor",
+	Usage:       "Increase the minor part of the version",
+	Value:       false,
+	Aliases:     []string{"mi", "y"},
+	Destination: &Minor,
+}
+
+var Patch bool
+var PatchFlag = &cli.BoolFlag{
+	Name:        "patch",
+	Usage:       "Increase the patch part of the version",
+	Value:       false,
+	Aliases:     []string{"p", "z"},
+	Destination: &Patch,
+}
+
+var Dry bool
+var DryFlag = &cli.BoolFlag{
+	Name:        "dry-run",
+	Usage:       "No changes are made, only reports changes that would be made",
+	Value:       false,
+	Aliases:     []string{"d"},
+	Destination: &Dry,
+}
+
+var Push bool
+var PushFlag = &cli.BoolFlag{
+	Name:        "push",
+	Usage:       "Git push changes",
+	Value:       false,
+	Aliases:     []string{"P"},
+	Destination: &Push,
+}
+
+var CommitMessage string
+var CommitMessageFlag = &cli.StringFlag{
+	Name:        "commit-message",
+	Usage:       "Commit message (implies --commit)",
+	Aliases:     []string{"cm"},
+	Destination: &CommitMessage,
+}
+
+var Commit bool
+var CommitFlag = &cli.BoolFlag{
+	Name:        "commit",
+	Usage:       "Commits compose.**yml file changes to recipe repository",
+	Value:       false,
+	Aliases:     []string{"c"},
+	Destination: &Commit,
+}
+
+var TagMessage string
+var TagMessageFlag = &cli.StringFlag{
+	Name:        "tag-comment",
+	Usage:       "Description for release tag",
+	Aliases:     []string{"t", "tm"},
+	Destination: &TagMessage,
+}
+
 // PromptBumpType prompts for version bump type
-func PromptBumpType(tagString, latestRelease string) error {
+func PromptBumpType(tagString string) error {
 	if (!Major && !Minor && !Patch) && tagString == "" {
 		fmt.Printf(`
-You need to make a decision about what kind of an update this new recipe
+semver cheat sheet (more via semver.org):
-version is. If someone else performs this upgrade, do they have to do some
+  major: new features/bug fixes, backwards incompatible
-migration work or take care of some breaking changes? This can be signaled in
+  minor: new features/bug fixes, backwards compatible
-the version you specify on the recipe deploy label and is called a semantic
+  patch: bug fixes, backwards compatible
-version.
-
-The latest published version is %s.
-
-Here is a semver cheat sheet (more on https://semver.org):
-
-    major: new features/bug fixes, backwards incompatible (e.g 1.0.0 -> 2.0.0).
-           the upgrade won't work without some preparation work and others need
-           to take care when performing it. "it could go wrong".
-
-    minor: new features/bug fixes, backwards compatible (e.g. 0.1.0 -> 0.2.0).
-           the upgrade should Just Work and there are no breaking changes in
-           the app and the recipe config. "it should go fine".
-
-    patch: bug fixes, backwards compatible (e.g. 0.0.1 -> 0.0.2). this upgrade
-           should also Just Work and is mostly to do with minor bug fixes
-           and/or security patches. "nothing to worry about".
-
-`, latestRelease)
 
+`)
 		var chosenBumpType string
 		prompt := &survey.Select{
 			Message: fmt.Sprintf("select recipe version increment type"),
 			Options: []string{"major", "minor", "patch"},
 		}
-
 		if err := survey.AskOne(prompt, &chosenBumpType); err != nil {
 			return err
 		}
-
 		SetBumpType(chosenBumpType)
 	}
-
 	return nil
 }
 
@@ -84,27 +133,20 @@ func SetBumpType(bumpType string) {
 	}
 }
 
-// GetMainAppImage retrieves the main 'app' image name
+// GetMainApp retrieves the main 'app' image name
-func GetMainAppImage(recipe recipe.Recipe) (string, error) {
+func GetMainApp(recipe recipe.Recipe) string {
-	var path string
+	var app string
 
 	for _, service := range recipe.Config.Services {
-		if service.Name == "app" {
+		name := service.Name
-			img, err := reference.ParseNormalizedNamed(service.Image)
+		if name == "app" {
-			if err != nil {
+			app = strings.Split(service.Image, ":")[0]
-				return "", err
-			}
-
-			path = reference.Path(img)
-			path = formatter.StripTagMeta(path)
-
-			return path, nil
 		}
 	}
 
-	if path == "" {
+	if app == "" {
-		return path, fmt.Errorf("%s has no main 'app' service?", recipe.Name)
+		logrus.Fatalf("%s has no main 'app' service?", recipe.Name)
 	}
 
-	return path, nil
+	return app
 }

cli/internal/record.go

@@ -0,0 +1,106 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/urfave/cli/v2"
+)
+
+// EnsureDNSProvider ensures a DNS provider is chosen.
+func EnsureDNSProvider() error {
+	if DNSProvider == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select DNS provider",
+			Options: []string{"gandi"},
+		}
+
+		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
+			return err
+		}
+	}
+
+	if DNSProvider == "" {
+		return fmt.Errorf("missing DNS provider?")
+	}
+
+	return nil
+}
+
+// EnsureDNSTypeFlag ensures a DNS type flag is present.
+func EnsureDNSTypeFlag(c *cli.Context) error {
+	if DNSType == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record type",
+			Default: "A",
+		}
+		if err := survey.AskOne(prompt, &DNSType); err != nil {
+			return err
+		}
+	}
+
+	if DNSType == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
+	}
+
+	return nil
+}
+
+// EnsureDNSNameFlag ensures a DNS name flag is present.
+func EnsureDNSNameFlag(c *cli.Context) error {
+	if DNSName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record name",
+			Default: "mysubdomain",
+		}
+		if err := survey.AskOne(prompt, &DNSName); err != nil {
+			return err
+		}
+	}
+
+	if DNSName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
+	}
+
+	return nil
+}
+
+// EnsureDNSValueFlag ensures a DNS value flag is present.
+func EnsureDNSValueFlag(c *cli.Context) error {
+	if DNSValue == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record value",
+			Default: "192.168.1.2",
+		}
+		if err := survey.AskOne(prompt, &DNSValue); err != nil {
+			return err
+		}
+	}
+
+	if DNSName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
+	}
+
+	return nil
+}
+
+// EnsureZoneArgument ensures a zone argument is present.
+func EnsureZoneArgument(c *cli.Context) (string, error) {
+	var zone string
+	if c.Args().First() == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify a domain name zone",
+			Default: "example.com",
+		}
+		if err := survey.AskOne(prompt, &zone); err != nil {
+			return zone, err
+		}
+	}
+
+	if zone == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
+	}
+
+	return zone, nil
+}

cli/internal/server.go

@@ -0,0 +1,208 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/urfave/cli/v2"
+)
+
+// EnsureServerProvider ensures a 3rd party server provider is chosen.
+func EnsureServerProvider() error {
+	if ServerProvider == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select server provider",
+			Options: []string{"capsul", "hetzner-cloud"},
+		}
+
+		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
+			return err
+		}
+	}
+
+	if ServerProvider == "" {
+		return fmt.Errorf("missing server provider?")
+	}
+
+	return nil
+}
+
+// EnsureNewCapsulVPSFlags ensure all flags are present.
+func EnsureNewCapsulVPSFlags(c *cli.Context) error {
+	if CapsulName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul name",
+		}
+		if err := survey.AskOne(prompt, &CapsulName); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul instance URL",
+			Default: CapsulInstanceURL,
+		}
+		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul type",
+			Default: CapsulType,
+		}
+		if err := survey.AskOne(prompt, &CapsulType); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul image",
+			Default: CapsulImage,
+		}
+		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
+			return err
+		}
+	}
+
+	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
+		var sshKeys string
+		prompt := &survey.Input{
+			Message: "specify capsul SSH keys (e.g. me@foo.com)",
+			Default: "",
+		}
+		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
+			return err
+		}
+		CapsulSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
+	}
+
+	if CapsulAPIToken == "" && !NoInput {
+		token, ok := os.LookupEnv("CAPSUL_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify capsul API token",
+			}
+			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
+				return err
+			}
+		} else {
+			CapsulAPIToken = token
+		}
+	}
+
+	if CapsulName == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
+	}
+	if CapsulInstanceURL == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
+	}
+	if CapsulType == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
+	}
+	if CapsulImage == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
+	}
+	if len(CapsulSSHKeys.Value()) == 0 {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
+	}
+	if CapsulAPIToken == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
+	}
+
+	return nil
+}
+
+// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
+func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
+	if HetznerCloudName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS name",
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS type",
+			Default: HetznerCloudType,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS image",
+			Default: HetznerCloudImage,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
+			return err
+		}
+	}
+
+	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
+		var sshKeys string
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
+			Default: "",
+		}
+		if err := survey.AskOne(prompt, &sshKeys); err != nil {
+			return err
+		}
+		HetznerCloudSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS location",
+			Default: HetznerCloudLocation,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
+			return err
+		}
+	}
+
+	if HetznerCloudAPIToken == "" && !NoInput {
+		token, ok := os.LookupEnv("HCLOUD_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify hetzner cloud API token",
+			}
+			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
+				return err
+			}
+		} else {
+			HetznerCloudAPIToken = token
+		}
+	}
+
+	if HetznerCloudName == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
+	}
+	if HetznerCloudType == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
+	}
+	if HetznerCloudImage == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
+	}
+	if len(HetznerCloudSSHKeys.Value()) == 0 {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud ssh keys?"))
+	}
+	if HetznerCloudLocation == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
+	}
+	if HetznerCloudAPIToken == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
+	}
+
+	return nil
+}

cli/internal/validate.go

@@ -2,86 +2,57 @@ package internal
 
 import (
 	"errors"
-	"fmt"
-	"os"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // AppName is used for configuring app name programmatically
 var AppName string
 
 // ValidateRecipe ensures the recipe arg is valid.
-func ValidateRecipe(c *cli.Context, ensureLatest bool) recipe.Recipe {
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
 
 	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
+		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName)
+	recipe, err := recipe.Get(recipeName)
 	if err != nil {
 		if c.Command.Name == "generate" {
-			if strings.Contains(err.Error(), "missing a compose") {
-				logrus.Fatal(err)
-			}
 			logrus.Warn(err)
 		} else {
 			logrus.Fatal(err)
 		}
 	}
 
-	if ensureLatest {
+	logrus.Debugf("validated '%s' as recipe argument", recipeName)
-		if err := recipe.EnsureLatest(recipeName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
 
-	logrus.Debugf("validated %s as recipe argument", recipeName)
+	return recipe
-
-	return chosenRecipe
 }
 
 // ValidateRecipeWithPrompt ensures a recipe argument is present before
 // validating, asking for input if required.
-func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
+func ValidateRecipeWithPrompt(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
 
 	if recipeName == "" && !NoInput {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
 		var recipes []string
-
-		catl, err := recipe.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		knownRecipes := make(map[string]bool)
 		for name := range catl {
-			knownRecipes[name] = true
+			recipes = append(recipes, name)
 		}
-
-		localRecipes, err := recipe.GetRecipesLocal()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, recipeLocal := range localRecipes {
-			if _, ok := knownRecipes[recipeLocal]; !ok {
-				knownRecipes[recipeLocal] = true
-			}
-		}
-
-		for recipeName := range knownRecipes {
-			recipes = append(recipes, recipeName)
-		}
-
 		prompt := &survey.Select{
 			Message: "Select recipe",
 			Options: recipes,
@@ -97,23 +68,17 @@ func ValidateRecipeWithPrompt(c *cli.Context, ensureLatest bool) recipe.Recipe {
 	}
 
 	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
+		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName)
+	recipe, err := recipe.Get(recipeName)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	if ensureLatest {
+	logrus.Debugf("validated '%s' as recipe argument", recipeName)
-		if err := recipe.EnsureLatest(recipeName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
 
-	logrus.Debugf("validated %s as recipe argument", recipeName)
+	return recipe
-
-	return chosenRecipe
 }
 
 // ValidateApp ensures the app name arg is valid.
@@ -134,7 +99,7 @@ func ValidateApp(c *cli.Context) config.App {
 		logrus.Fatal(err)
 	}
 
-	if err := recipe.EnsureExists(app.Recipe); err != nil {
+	if err := recipe.EnsureExists(app.Type); err != nil {
 		logrus.Fatal(err)
 	}
 
@@ -142,13 +107,42 @@ func ValidateApp(c *cli.Context) config.App {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("validated %s as app argument", appName)
+	logrus.Debugf("validated '%s' as app argument", appName)
+
+	return app
+}
+
+// ValidateAppByName ensures the app is valid and takes an app name as an argument, not context.
+func ValidateAppByName(c *cli.Context, appName string) config.App {
+	if AppName != "" {
+		appName = AppName
+		logrus.Debugf("programmatically setting app name to %s", appName)
+	}
+
+	if appName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+	}
+
+	app, err := app.Get(appName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := recipe.EnsureExists(app.Type); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ssh.EnsureHostKey(app.Server); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated '%s' as app argument", appName)
 
 	return app
 }
 
 // ValidateDomain ensures the domain name arg is valid.
-func ValidateDomain(c *cli.Context) string {
+func ValidateDomain(c *cli.Context) (string, error) {
 	domainName := c.Args().First()
 
 	if domainName == "" && !NoInput {
@@ -157,7 +151,7 @@ func ValidateDomain(c *cli.Context) string {
 			Default: "example.com",
 		}
 		if err := survey.AskOne(prompt, &domainName); err != nil {
-			logrus.Fatal(err)
+			return domainName, err
 		}
 	}
 
@@ -165,16 +159,16 @@ func ValidateDomain(c *cli.Context) string {
 		ShowSubcommandHelpAndError(c, errors.New("no domain provided"))
 	}
 
-	logrus.Debugf("validated %s as domain argument", domainName)
+	logrus.Debugf("validated '%s' as domain argument", domainName)
 
-	return domainName
+	return domainName, nil
 }
 
 // ValidateSubCmdFlags ensures flag order conforms to correct order
 func ValidateSubCmdFlags(c *cli.Context) bool {
-	for argIdx, arg := range c.Args() {
+	for argIdx, arg := range c.Args().Slice() {
 		if !strings.HasPrefix(arg, "--") {
-			for _, flag := range c.Args()[argIdx:] {
+			for _, flag := range c.Args().Slice()[argIdx:] {
 				if strings.HasPrefix(flag, "--") {
 					return false
 				}
@@ -185,12 +179,12 @@ func ValidateSubCmdFlags(c *cli.Context) bool {
 }
 
 // ValidateServer ensures the server name arg is valid.
-func ValidateServer(c *cli.Context) string {
+func ValidateServer(c *cli.Context) (string, error) {
 	serverName := c.Args().First()
 
 	serverNames, err := config.ReadServerNames()
 	if err != nil {
-		logrus.Fatal(err)
+		return serverName, err
 	}
 
 	if serverName == "" && !NoInput {
@@ -199,320 +193,15 @@ func ValidateServer(c *cli.Context) string {
 			Options: serverNames,
 		}
 		if err := survey.AskOne(prompt, &serverName); err != nil {
-			logrus.Fatal(err)
+			return serverName, err
 		}
 	}
 
-	matched := false
-	for _, name := range serverNames {
-		if name == serverName {
-			matched = true
-		}
-	}
-
-	if !matched {
-		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
-	}
-
 	if serverName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
 	}
 
-	logrus.Debugf("validated %s as server argument", serverName)
+	logrus.Debugf("validated '%s' as server argument", serverName)
 
-	return serverName
+	return serverName, nil
-}
-
-// EnsureDNSProvider ensures a DNS provider is chosen.
-func EnsureDNSProvider() error {
-	if DNSProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select DNS provider",
-			Options: []string{"gandi"},
-		}
-
-		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
-			return err
-		}
-	}
-
-	if DNSProvider == "" {
-		return fmt.Errorf("missing DNS provider?")
-	}
-
-	return nil
-}
-
-// EnsureDNSTypeFlag ensures a DNS type flag is present.
-func EnsureDNSTypeFlag(c *cli.Context) error {
-	if DNSType == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record type",
-			Default: "A",
-		}
-		if err := survey.AskOne(prompt, &DNSType); err != nil {
-			return err
-		}
-	}
-
-	if DNSType == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSNameFlag ensures a DNS name flag is present.
-func EnsureDNSNameFlag(c *cli.Context) error {
-	if DNSName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record name",
-			Default: "mysubdomain",
-		}
-		if err := survey.AskOne(prompt, &DNSName); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSValueFlag ensures a DNS value flag is present.
-func EnsureDNSValueFlag(c *cli.Context) error {
-	if DNSValue == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record value",
-			Default: "192.168.1.2",
-		}
-		if err := survey.AskOne(prompt, &DNSValue); err != nil {
-			return err
-		}
-	}
-
-	if DNSValue == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
-	}
-
-	return nil
-}
-
-// EnsureZoneArgument ensures a zone argument is present.
-func EnsureZoneArgument(c *cli.Context) (string, error) {
-	zone := c.Args().First()
-
-	if zone == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name zone",
-			Default: "example.com",
-		}
-
-		if err := survey.AskOne(prompt, &zone); err != nil {
-			return zone, err
-		}
-	}
-
-	if zone == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
-	}
-
-	return zone, nil
-}
-
-// EnsureServerProvider ensures a 3rd party server provider is chosen.
-func EnsureServerProvider() error {
-	if ServerProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select server provider",
-			Options: []string{"capsul", "hetzner-cloud"},
-		}
-
-		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
-			return err
-		}
-	}
-
-	if ServerProvider == "" {
-		return fmt.Errorf("missing server provider?")
-	}
-
-	return nil
-}
-
-// EnsureNewCapsulVPSFlags ensure all flags are present.
-func EnsureNewCapsulVPSFlags(c *cli.Context) error {
-	if CapsulName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul name",
-		}
-		if err := survey.AskOne(prompt, &CapsulName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul instance URL",
-			Default: CapsulInstanceURL,
-		}
-		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul type",
-			Default: CapsulType,
-		}
-		if err := survey.AskOne(prompt, &CapsulType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul image",
-			Default: CapsulImage,
-		}
-		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
-			return err
-		}
-	}
-
-	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify capsul SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
-			return err
-		}
-		CapsulSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if CapsulAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("CAPSUL_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify capsul API token",
-			}
-			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
-				return err
-			}
-		} else {
-			CapsulAPIToken = token
-		}
-	}
-
-	if CapsulName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
-	}
-	if CapsulInstanceURL == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
-	}
-	if CapsulType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
-	}
-	if CapsulImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
-	}
-	if len(CapsulSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
-	}
-	if CapsulAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
-	}
-
-	return nil
-}
-
-// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
-func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
-	if HetznerCloudName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS type",
-			Default: HetznerCloudType,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS image",
-			Default: HetznerCloudImage,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
-			return err
-		}
-	}
-
-	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &sshKeys); err != nil {
-			return err
-		}
-		HetznerCloudSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS location",
-			Default: HetznerCloudLocation,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
-			return err
-		}
-	}
-
-	if HetznerCloudAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			HetznerCloudAPIToken = token
-		}
-	}
-
-	if HetznerCloudName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
-	}
-	if HetznerCloudType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
-	}
-	if HetznerCloudImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
-	}
-	if HetznerCloudLocation == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
-	}
-	if HetznerCloudAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
-	}
-
-	return nil
 }

cli/recipe/lint.go

@@ -2,78 +2,112 @@ package recipe
 
 import (
 	"fmt"
+	"os"
+	"strconv"
 
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/tagcmp"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeLintCommand = cli.Command{
+var recipeLintCommand = &cli.Command{
 	Name:      "lint",
 	Usage:     "Lint a recipe",
 	Aliases:   []string{"l"},
 	ArgsUsage: "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OnlyErrorFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, true)
+		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+		expectedVersion := false
+		if recipe.Config.Version == "3.8" {
+			expectedVersion = true
+		}
+
+		envSampleProvided := false
+		envSample := fmt.Sprintf("%s/%s/.env.sample", config.APPS_DIR, recipe.Name)
+		if _, err := os.Stat(envSample); !os.IsNotExist(err) {
+			envSampleProvided = true
+		} else if err != nil {
 			logrus.Fatal(err)
 		}
 
-		tableCol := []string{"ref", "rule", "satisfied", "severity", "resolve"}
+		serviceNamedApp := false
-		table := formatter.CreateTable(tableCol)
+		traefikEnabled := false
+		healthChecksForAllServices := true
+		allImagesTagged := true
+		noUnstableTags := true
+		semverLikeTags := true
+		for _, service := range recipe.Config.Services {
+			if service.Name == "app" {
+				serviceNamedApp = true
+			}
 
-		hasError := false
+			for label := range service.Deploy.Labels {
-		bar := formatter.CreateProgressbar(-1, "running recipe lint rules...")
+				if label == "traefik.enable" {
-		for level := range lint.LintRules {
+					if service.Deploy.Labels[label] == "true" {
-			for _, rule := range lint.LintRules[level] {
+						traefikEnabled = true
-				ok, err := rule.Function(recipe)
+					}
+				}
+			}
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
 			if err != nil {
-					logrus.Warn(err)
+				logrus.Fatal(err)
+			}
+			if reference.IsNameOnly(img) {
+				allImagesTagged = false
 			}
 
-				if !ok && rule.Level == "error" {
+			var tag string
-					hasError = true
+			switch img.(type) {
+			case reference.NamedTagged:
+				tag = img.(reference.NamedTagged).Tag()
+			case reference.Named:
+				noUnstableTags = false
 			}
 
-				var result string
+			if tag == "latest" {
-				if ok {
+				noUnstableTags = false
-					result = "yes"
-				} else {
-					result = "NO"
 			}
 
-				if internal.OnlyErrors {
+			if !tagcmp.IsParsable(tag) {
-					if !ok && rule.Level == "error" {
+				semverLikeTags = false
-						table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
-						bar.Add(1)
-					}
-				} else {
-					table.Append([]string{rule.Ref, rule.Description, result, rule.Level, rule.HowToResolve})
-					bar.Add(1)
 			}
+
+			if service.HealthCheck == nil {
+				healthChecksForAllServices = false
 			}
 		}
 
-		if table.NumLines() > 0 {
+		tableCol := []string{"rule", "satisfied"}
-			fmt.Println()
+		table := formatter.CreateTable(tableCol)
+		table.Append([]string{"compose files have the expected version", strconv.FormatBool(expectedVersion)})
+		table.Append([]string{"environment configuration is provided", strconv.FormatBool(envSampleProvided)})
+		table.Append([]string{"recipe contains a service named 'app'", strconv.FormatBool(serviceNamedApp)})
+		table.Append([]string{"traefik routing enabled on at least one service", strconv.FormatBool(traefikEnabled)})
+		table.Append([]string{"all services have a healthcheck enabled", strconv.FormatBool(healthChecksForAllServices)})
+		table.Append([]string{"all images are using a tag", strconv.FormatBool(allImagesTagged)})
+		table.Append([]string{"no usage of unstable 'latest' tags", strconv.FormatBool(noUnstableTags)})
+		table.Append([]string{"all tags are using a semver-like format", strconv.FormatBool(semverLikeTags)})
 		table.Render()
-		}
-
-		if hasError {
-			logrus.Warn("watch out, some critical errors are present in your recipe config")
-		}
 
 		return nil
 	},
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
 }

cli/recipe/list.go

@@ -3,74 +3,36 @@ package recipe
 import (
 	"fmt"
 	"sort"
-	"strconv"
-	"strings"
 
-	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var pattern string
+var recipeListCommand = &cli.Command{
-var patternFlag = &cli.StringFlag{
-	Name:        "pattern, p",
-	Value:       "",
-	Usage:       "Simple string to filter recipes",
-	Destination: &pattern,
-}
-
-var recipeListCommand = cli.Command{
 	Name:    "list",
 	Usage:   "List available recipes",
 	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		patternFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err.Error())
 		}
 
 		recipes := catl.Flatten()
-		sort.Sort(recipe.ByRecipeName(recipes))
+		sort.Sort(catalogue.ByRecipeName(recipes))
 
-		tableCol := []string{"name", "category", "status", "healthcheck", "backups", "email", "tests", "SSO"}
+		tableCol := []string{"name", "category", "status"}
 		table := formatter.CreateTable(tableCol)
 
-		len := 0
 		for _, recipe := range recipes {
-			tableRow := []string{
+			status := fmt.Sprintf("%v", recipe.Features.Status)
-				recipe.Name,
+			tableRow := []string{recipe.Name, recipe.Category, status}
-				recipe.Category,
-				strconv.Itoa(recipe.Features.Status),
-				recipe.Features.Healthcheck,
-				recipe.Features.Backups,
-				recipe.Features.Email,
-				recipe.Features.Tests,
-				recipe.Features.SSO,
-			}
-
-			if pattern != "" {
-				if strings.Contains(recipe.Name, pattern) {
 			table.Append(tableRow)
-					len++
-				}
-			} else {
-				table.Append(tableRow)
-				len++
-			}
 		}
 
-		table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
-
-		if table.NumLines() > 0 {
 		table.Render()
-		}
 
 		return nil
 	},

cli/recipe/new.go

@@ -1,10 +1,8 @@
 package recipe
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"text/template"
@@ -13,32 +11,13 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/git"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// recipeMetadata is the recipe metadata for the README.md
+var recipeNewCommand = &cli.Command{
-type recipeMetadata struct {
-	Name        string
-	Description string
-	Category    string
-	Status      string
-	Image       string
-	Healthcheck string
-	Backups     string
-	Email       string
-	Tests       string
-	SSO         string
-}
-
-var recipeNewCommand = cli.Command{
 	Name:      "new",
-	Aliases: []string{"n"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:    internal.SubCommandBefore,
 	Usage:     "Create a new recipe",
+	Aliases:   []string{"n"},
 	ArgsUsage: "<recipe>",
 	Description: `
 This command creates a new recipe.
@@ -50,17 +29,19 @@ Abra uses our built-in example repository which is available here:
 Files within the example repository make use of the Golang templating system
 which Abra uses to inject values into the generated recipe folder (e.g. name of
 recipe and domain in the sample environment config).
+
+The new example repository is cloned to ~/.abra/apps/<recipe>.
 `,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
 
 		if recipeName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
+			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
 		}
 
-		directory := path.Join(config.RECIPES_DIR, recipeName)
+		directory := path.Join(config.APPS_DIR, recipeName)
 		if _, err := os.Stat(directory); !os.IsNotExist(err) {
-			logrus.Fatalf("%s recipe directory already exists?", directory)
+			logrus.Fatalf("'%s' recipe directory already exists?", directory)
 		}
 
 		url := fmt.Sprintf("%s/example.git", config.REPOS_BASE_URL)
@@ -68,73 +49,49 @@ recipe and domain in the sample environment config).
 			logrus.Fatal(err)
 		}
 
-		gitRepo := path.Join(config.RECIPES_DIR, recipeName, ".git")
+		gitRepo := path.Join(config.APPS_DIR, recipeName, ".git")
 		if err := os.RemoveAll(gitRepo); err != nil {
 			logrus.Fatal(err)
 		}
-		logrus.Debugf("removed example git repo in %s", gitRepo)
+		logrus.Debugf("removed git repo in '%s'", gitRepo)
-
-		meta := newRecipeMeta(recipeName)
 
 		toParse := []string{
-			path.Join(config.RECIPES_DIR, recipeName, "README.md"),
+			path.Join(config.APPS_DIR, recipeName, "README.md"),
-			path.Join(config.RECIPES_DIR, recipeName, ".env.sample"),
+			path.Join(config.APPS_DIR, recipeName, ".env.sample"),
+			path.Join(config.APPS_DIR, recipeName, ".drone.yml"),
 		}
 		for _, path := range toParse {
+			file, err := os.OpenFile(path, os.O_RDWR, 0755)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
 			tpl, err := template.ParseFiles(path)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-			var templated bytes.Buffer
+			// TODO: ask for description and probably other things so that the
-			if err := tpl.Execute(&templated, meta); err != nil {
+			// template repository is more "ready" to go than the current best-guess
+			// mode of templating
+			if err := tpl.Execute(file, struct {
+				Name        string
+				Description string
+			}{recipeName, "TODO"}); err != nil {
 				logrus.Fatal(err)
 			}
-
-			if err := ioutil.WriteFile(path, templated.Bytes(), 0644); err != nil {
-				logrus.Fatal(err)
 		}
 
-		}
+		newGitRepo := path.Join(config.APPS_DIR, recipeName)
-
-		newGitRepo := path.Join(config.RECIPES_DIR, recipeName)
 		if err := git.Init(newGitRepo, true); err != nil {
 			logrus.Fatal(err)
 		}
 
-		fmt.Print(fmt.Sprintf(`
+		logrus.Infof(
-Your new %s recipe has been created in %s.
+			"new recipe '%s' created in %s, happy hacking!\n",
-
+			recipeName, path.Join(config.APPS_DIR, recipeName),
-In order to share your recipe, you can upload it the git repository to:
+		)
-
-    https://git.coopcloud.tech/coop-cloud/%s
-
-If you're not sure how to do that, come chat with us:
-
-    https://docs.coopcloud.tech/contact
-
-See "abra recipe -h" for additional recipe maintainer commands.
-
-Happy Hacking!
-
-`, recipeName, path.Join(config.RECIPES_DIR, recipeName), recipeName))
 
 		return nil
 	},
 }
-
-// newRecipeMeta creates a new recipeMetadata instance with defaults
-func newRecipeMeta(recipeName string) recipeMetadata {
-	return recipeMetadata{
-		Name:        recipeName,
-		Description: "> One line description of the recipe",
-		Category:    "Apps",
-		Status:      "0",
-		Image:       fmt.Sprintf("[`%s`](https://hub.docker.com/r/%s), 4, upstream", recipeName, recipeName),
-		Healthcheck: "No",
-		Backups:     "No",
-		Email:       "No",
-		Tests:       "No",
-		SSO:         "No",
-	}
-}

cli/recipe/recipe.go

@@ -1,26 +1,21 @@
 package recipe
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // RecipeCommand defines all recipe related sub-commands.
-var RecipeCommand = cli.Command{
+var RecipeCommand = &cli.Command{
 	Name:      "recipe",
-	Aliases:   []string{"r"},
+	Usage:     "Manage recipes (for maintainers)",
-	Usage:     "Manage recipes",
 	ArgsUsage: "<recipe>",
+	Aliases:   []string{"r"},
 	Description: `
-A recipe is a blueprint for an app. It is a bunch of config files which
+A recipe is a blueprint for an app. It is a bunch of configuration files which
 describe how to deploy and maintain an app. Recipes are maintained by the Co-op
 Cloud community and you can use Abra to read them and create apps for you.
-
-Anyone who uses a recipe can become a maintainer. Maintainers typically make
-sure the recipe is in good working order and the config upgraded in a timely
-manner. Abra supports convenient automation for recipe maintainenace, see the
-"abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands.
 `,
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		recipeListCommand,
 		recipeVersionCommand,
 		recipeReleaseCommand,

cli/recipe/release.go

@@ -6,120 +6,290 @@ import (
 	"strconv"
 	"strings"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeReleaseCommand = cli.Command{
+var recipeReleaseCommand = &cli.Command{
 	Name:      "release",
-	Aliases:   []string{"rl"},
 	Usage:     "Release a new recipe version",
+	Aliases:   []string{"rl"},
 	ArgsUsage: "<recipe> [<version>]",
 	Description: `
-This command is used to specify a new version of a recipe. These versions are
+This command is used to specify a new tag for a recipe. These tags are used to
-then published on the Co-op Cloud recipe catalogue. These versions take the
+identify different versions of the recipe and are published on the Co-op Cloud
-following form:
+recipe catalogue.
+
+These tags take the following form:
 
     a.b.c+x.y.z
 
-Where the "a.b.c" part is a semantic version determined by the maintainer. And
+Where the "a.b.c" part is maintained as a semantic version of the recipe by the
-the "x.y.z" part is the image tag of the recipe "app" service (the main
+recipe maintainer. And the "x.y.z" part is the image tag of the recipe "app"
-container which contains the software to be used).
+service (the main container which contains the software to be used).
 
 We maintain a semantic versioning scheme ("a.b.c") alongside the libre app
-versioning scheme ("x.y.z") in order to maximise the chances that the nature of
+versioning scheme in order to maximise the chances that the nature of recipe
-recipe updates are properly communicated. I.e. developers of an app might
+updates are properly communicated.
-publish a minor version but that might lead to changes in the recipe which are
+
-major and therefore require intervention while doing the upgrade work.
+Abra does its best to read the "a.b.c" version scheme and communicate what
+action needs to be taken when performing different operations such as an update
+or a rollback of an app.
+
+You may invoke this command in "wizard" mode and be prompted for input:
+
+    abra recipe release gitea
 
-Publish your new release to git.coopcloud.tech with "-p/--publish". This
-requires that you have permission to git push to these repositories and have
-your SSH keys configured on your account.
 `,
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.DryFlag,
 		internal.MajorFlag,
 		internal.MinorFlag,
 		internal.PatchFlag,
-		internal.PublishFlag,
+		internal.PushFlag,
+		internal.CommitFlag,
+		internal.CommitMessageFlag,
+		internal.TagMessageFlag,
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, false)
+		recipe := internal.ValidateRecipeWithPrompt(c)
+		directory := path.Join(config.APPS_DIR, recipe.Name)
+		tagString := c.Args().Get(1)
+		mainApp := internal.GetMainApp(recipe)
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		mainAppVersion := imagesTmp[mainApp]
 
-		mainApp, err := internal.GetMainAppImage(recipe)
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
-		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		mainAppVersion := imagesTmp[mainApp]
 		if mainAppVersion == "" {
-			logrus.Fatalf("main app service version for %s is empty?", recipe.Name)
+			logrus.Fatalf("main 'app' service version for %s is empty?", recipe.Name)
 		}
 
-		tagString := c.Args().Get(1)
 		if tagString != "" {
 			if _, err := tagcmp.Parse(tagString); err != nil {
-				logrus.Fatalf("cannot parse %s, invalid tag specified?", tagString)
+				logrus.Fatal("invalid tag specified")
 			}
 		}
 
+		if (!internal.Major && !internal.Minor && !internal.Patch) && tagString != "" {
+			logrus.Fatal("please specify <version> or bump type (--major/--minor/--patch)")
+		}
+
 		if (internal.Major || internal.Minor || internal.Patch) && tagString != "" {
 			logrus.Fatal("cannot specify tag and bump type at the same time")
 		}
 
-		if tagString != "" {
+		// bumpType is used to decide what part of the tag should be incremented
-			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+		if bumpType != 0 {
+			// a bitwise check if the number is a power of 2
+			if (bumpType & (bumpType - 1)) != 0 {
+				logrus.Fatal("you can only use one of: --major, --minor, --patch.")
+			}
+		}
+
+		if err := internal.PromptBumpType(tagString); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if internal.TagMessage == "" {
+			prompt := &survey.Input{
+				Message: "tag message",
+				Default: fmt.Sprintf("chore: publish new %s version", internal.GetBumpType()),
+			}
+			if err := survey.AskOne(prompt, &internal.TagMessage); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		tags, err := recipe.Tags()
+		var createTagOptions git.CreateTagOptions
+		createTagOptions.Message = internal.TagMessage
+
+		if !internal.Commit {
+			prompt := &survey.Confirm{
+				Message: "git commit changes also?",
+			}
+			if err := survey.AskOne(prompt, &internal.Commit); err != nil {
+				return err
+			}
+		}
+
+		if !internal.Push {
+			prompt := &survey.Confirm{
+				Message: "git push changes also?",
+			}
+			if err := survey.AskOne(prompt, &internal.Push); err != nil {
+				return err
+			}
+		}
+
+		if internal.Commit || internal.CommitMessage != "" {
+			commitRepo, err := git.PlainOpen(directory)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			commitWorktree, err := commitRepo.Worktree()
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-		if tagString == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+			if internal.CommitMessage == "" {
-			var err error
+				prompt := &survey.Input{
-			tagString, err = getLabelVersion(recipe, false)
+					Message: "commit message",
-			if err != nil {
+					Default: fmt.Sprintf("chore: publish new %s version", internal.GetBumpType()),
+				}
+				if err := survey.AskOne(prompt, &internal.CommitMessage); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-		if len(tags) > 0 {
+			err = commitWorktree.AddGlob("compose.**yml")
-			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
+			if err != nil {
-			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
 				logrus.Fatal(err)
 			}
+			logrus.Debug("staged compose.**yml for commit")
+
+			if !internal.Dry {
+				_, err = commitWorktree.Commit(internal.CommitMessage, &git.CommitOptions{})
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info("changes commited")
 			} else {
-			logrus.Warnf("no tag specified and no previous tag available for %s, assuming this is the initial release", recipe.Name)
+				logrus.Info("dry run only: NOT committing changes")
-
-			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
-				if cleanUpErr := cleanUpTag(tagString, recipe.Name); err != nil {
-					logrus.Fatal(cleanUpErr)
 			}
+		}
+
+		repo, err := git.PlainOpen(directory)
+		if err != nil {
 			logrus.Fatal(err)
 		}
+		head, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if tagString != "" {
+			tag, err := tagcmp.Parse(tagString)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if tag.MissingMinor {
+				tag.Minor = "0"
+				tag.MissingMinor = false
+			}
+			if tag.MissingPatch {
+				tag.Patch = "0"
+				tag.MissingPatch = false
+			}
+			tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
+			if internal.Dry {
+				hash := abraFormatter.SmallSHA(head.Hash().String())
+				logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", tagString, hash))
+				return nil
+			}
+
+			repo.CreateTag(tagString, head.Hash(), &createTagOptions)
+			hash := abraFormatter.SmallSHA(head.Hash().String())
+			logrus.Info(fmt.Sprintf("created tag %s at %s", tagString, hash))
+			if internal.Push && !internal.Dry {
+				if err := repo.Push(&git.PushOptions{}); err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("pushed tag %s to remote", tagString))
+			} else {
+				logrus.Info("dry run only: NOT pushing changes")
+			}
+
+			return nil
+		}
+
+		// get the latest tag with its hash, name etc
+		var lastGitTag tagcmp.Tag
+		iter, err := repo.Tags()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if err := iter.ForEach(func(ref *plumbing.Reference) error {
+			obj, err := repo.TagObject(ref.Hash())
+			if err != nil {
+				return err
+			}
+			tagcmpTag, err := tagcmp.Parse(obj.Name)
+			if err != nil {
+				return err
+			}
+			if (lastGitTag == tagcmp.Tag{}) {
+				lastGitTag = tagcmpTag
+			} else if tagcmpTag.IsGreaterThan(lastGitTag) {
+				lastGitTag = tagcmpTag
+			}
+			return nil
+		}); err != nil {
+			logrus.Fatal(err)
+		}
+
+		newTag := lastGitTag
+		var newtagString string
+		if bumpType > 0 {
+			if internal.Patch {
+				now, err := strconv.Atoi(newTag.Patch)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = strconv.Itoa(now + 1)
+			} else if internal.Minor {
+				now, err := strconv.Atoi(newTag.Minor)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = "0"
+				newTag.Minor = strconv.Itoa(now + 1)
+			} else if internal.Major {
+				now, err := strconv.Atoi(newTag.Major)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = "0"
+				newTag.Minor = "0"
+				newTag.Major = strconv.Itoa(now + 1)
+			}
+		}
+
+		newTag.Metadata = mainAppVersion
+		newtagString = newTag.String()
+		if internal.Dry {
+			hash := abraFormatter.SmallSHA(head.Hash().String())
+			logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", newtagString, hash))
+			return nil
+		}
+
+		repo.CreateTag(newtagString, head.Hash(), &createTagOptions)
+		hash := abraFormatter.SmallSHA(head.Hash().String())
+		logrus.Info(fmt.Sprintf("created tag %s at %s", newtagString, hash))
+		if internal.Push && !internal.Dry {
+			if err := repo.Push(&git.PushOptions{}); err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info(fmt.Sprintf("pushed tag %s to remote", newtagString))
+		} else {
+			logrus.Info("gry run only: NOT pushing changes")
 		}
 
 		return nil
@@ -130,7 +300,6 @@ your SSH keys configured on your account.
 func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
 	var services = make(map[string]string)
 
-	missingTag := false
 	for _, service := range recipe.Config.Services {
 		if service.Image == "" {
 			continue
@@ -142,74 +311,24 @@ func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
 		}
 
 		path := reference.Path(img)
-
+		if strings.Contains(path, "library") {
-		path = formatter.StripTagMeta(path)
+			path = strings.Split(path, "/")[1]
+		}
 
 		var tag string
 		switch img.(type) {
 		case reference.NamedTagged:
 			tag = img.(reference.NamedTagged).Tag()
 		case reference.Named:
-			if service.Name == "app" {
+			logrus.Fatalf("%s service is missing image tag?", path)
-				missingTag = true
-			}
-			continue
 		}
 
 		services[path] = tag
 	}
 
-	if missingTag {
-		return services, fmt.Errorf("app service is missing image tag?")
-	}
-
 	return services, nil
 }
 
-// createReleaseFromTag creates a new release based on a supplied recipe version string
-func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string) error {
-	var err error
-
-	directory := path.Join(config.RECIPES_DIR, recipe.Name)
-	repo, err := git.PlainOpen(directory)
-	if err != nil {
-		return err
-	}
-
-	tag, err := tagcmp.Parse(tagString)
-	if err != nil {
-		return err
-	}
-
-	if tag.MissingMinor {
-		tag.Minor = "0"
-		tag.MissingMinor = false
-	}
-
-	if tag.MissingPatch {
-		tag.Patch = "0"
-		tag.MissingPatch = false
-	}
-
-	if tagString == "" {
-		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
-	}
-
-	if err := commitRelease(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := tagRelease(tagString, repo); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := pushRelease(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
-	return nil
-}
-
 // btoi converts a boolean value into an integer
 func btoi(b bool) int {
 	if b {
@@ -218,237 +337,3 @@ func btoi(b bool) int {
 
 	return 0
 }
-
-// getTagCreateOptions constructs git tag create options
-func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
-	msg := fmt.Sprintf("chore: publish %s release", tag)
-	return git.CreateTagOptions{Message: msg}, nil
-}
-
-func commitRelease(recipe recipe.Recipe, tag string) error {
-	if internal.Dry {
-		logrus.Debugf("dry run: no changes committed")
-		return nil
-	}
-
-	isClean, err := gitPkg.IsClean(recipe.Dir())
-	if err != nil {
-		return err
-	}
-
-	if isClean {
-		if !internal.Dry {
-			return fmt.Errorf("no changes discovered in %s, nothing to publish?", recipe.Dir())
-		}
-	}
-
-	msg := fmt.Sprintf("chore: publish %s release", tag)
-	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func tagRelease(tagString string, repo *git.Repository) error {
-	if internal.Dry {
-		logrus.Debugf("dry run: no git tag created (%s)", tagString)
-		return nil
-	}
-
-	head, err := repo.Head()
-	if err != nil {
-		return err
-	}
-
-	createTagOptions, err := getTagCreateOptions(tagString)
-	if err != nil {
-		return err
-	}
-
-	_, err = repo.CreateTag(tagString, head.Hash(), &createTagOptions)
-	if err != nil {
-		return err
-	}
-
-	hash := formatter.SmallSHA(head.Hash().String())
-	logrus.Debugf(fmt.Sprintf("created tag %s at %s", tagString, hash))
-
-	return nil
-}
-
-func pushRelease(recipe recipe.Recipe, tagString string) error {
-	if internal.Dry {
-		logrus.Info("dry run: no changes published")
-		return nil
-	}
-
-	if !internal.Publish && !internal.NoInput {
-		prompt := &survey.Confirm{
-			Message: "publish new release?",
-		}
-
-		if err := survey.AskOne(prompt, &internal.Publish); err != nil {
-			return err
-		}
-	}
-
-	if internal.Publish {
-		if err := recipe.Push(internal.Dry); err != nil {
-			return err
-		}
-		url := fmt.Sprintf("%s/%s/src/tag/%s", config.REPOS_BASE_URL, recipe.Name, tagString)
-		logrus.Infof("new release published: %s", url)
-	} else {
-		logrus.Info("no -p/--publish passed, not publishing")
-	}
-
-	return nil
-}
-
-func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recipe.Recipe, tags []string) error {
-	directory := path.Join(config.RECIPES_DIR, recipe.Name)
-	repo, err := git.PlainOpen(directory)
-	if err != nil {
-		return err
-	}
-
-	bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
-	if bumpType != 0 {
-		if (bumpType & (bumpType - 1)) != 0 {
-			return fmt.Errorf("you can only use one of: --major, --minor, --patch")
-		}
-	}
-
-	var lastGitTag tagcmp.Tag
-	for _, tag := range tags {
-		parsed, err := tagcmp.Parse(tag)
-		if err != nil {
-			return err
-		}
-
-		if (lastGitTag == tagcmp.Tag{}) {
-			lastGitTag = parsed
-		} else if parsed.IsGreaterThan(lastGitTag) {
-			lastGitTag = parsed
-		}
-	}
-
-	newTag := lastGitTag
-	if internal.Patch {
-		now, err := strconv.Atoi(newTag.Patch)
-		if err != nil {
-			return err
-		}
-
-		newTag.Patch = strconv.Itoa(now + 1)
-	} else if internal.Minor {
-		now, err := strconv.Atoi(newTag.Minor)
-		if err != nil {
-			return err
-		}
-
-		newTag.Patch = "0"
-		newTag.Minor = strconv.Itoa(now + 1)
-	} else if internal.Major {
-		now, err := strconv.Atoi(newTag.Major)
-		if err != nil {
-			return err
-		}
-
-		newTag.Patch = "0"
-		newTag.Minor = "0"
-		newTag.Major = strconv.Itoa(now + 1)
-	}
-
-	if tagString == "" {
-		if err := internal.PromptBumpType(tagString, lastGitTag.String()); err != nil {
-			return err
-		}
-	}
-
-	if internal.Major || internal.Minor || internal.Patch {
-		newTag.Metadata = mainAppVersion
-		tagString = newTag.String()
-	}
-
-	if lastGitTag.String() == tagString {
-		logrus.Fatalf("latest git tag (%s) and synced lable (%s) are the same?", lastGitTag, tagString)
-	}
-
-	if !internal.NoInput {
-		prompt := &survey.Confirm{
-			Message: fmt.Sprintf("current: %s, new: %s, correct?", lastGitTag, tagString),
-		}
-
-		var ok bool
-		if err := survey.AskOne(prompt, &ok); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !ok {
-			logrus.Fatal("exiting as requested")
-		}
-	}
-
-	if err := commitRelease(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := tagRelease(tagString, repo); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := pushRelease(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
-	return nil
-}
-
-// cleanUpTag removes a freshly created tag
-func cleanUpTag(tag, recipeName string) error {
-	directory := path.Join(config.RECIPES_DIR, recipeName)
-	repo, err := git.PlainOpen(directory)
-	if err != nil {
-		return err
-	}
-
-	if err := repo.DeleteTag(tag); err != nil {
-		if !strings.Contains(err.Error(), "not found") {
-			return err
-		}
-	}
-
-	logrus.Debugf("removed freshly created tag %s", tag)
-
-	return nil
-}
-
-func getLabelVersion(recipe recipe.Recipe, prompt bool) (string, error) {
-	initTag, err := recipePkg.GetVersionLabelLocal(recipe)
-	if err != nil {
-		return "", err
-	}
-
-	if initTag == "" {
-		logrus.Fatalf("unable to read version for %s from synced label. Did you try running \"abra recipe sync %s\" already?", recipe.Name, recipe.Name)
-	}
-
-	logrus.Warnf("discovered %s as currently synced recipe label", initTag)
-
-	if prompt && !internal.NoInput {
-		var response bool
-		prompt := &survey.Confirm{Message: fmt.Sprintf("use %s as the new version?", initTag)}
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return "", err
-		}
-
-		if !response {
-			return "", fmt.Errorf("please fix your synced label for %s and re-run this command", recipe.Name)
-		}
-	}
-
-	return initTag, nil
-}

cli/recipe/sync.go

@@ -6,53 +6,51 @@ import (
 	"strconv"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeSyncCommand = cli.Command{
+var recipeSyncCommand = &cli.Command{
 	Name:      "sync",
+	Usage:     "Ensure recipe version labels are up-to-date",
 	Aliases:   []string{"s"},
-	Usage:     "Sync recipe version label",
 	ArgsUsage: "<recipe> [<version>]",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.DryFlag,
 		internal.MajorFlag,
 		internal.MinorFlag,
 		internal.PatchFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command will generate labels for the main recipe service (i.e. by
-convention, the service named 'app') which corresponds to the following format:
+convention, the service named "app") which corresponds to the following format:
 
     coop-cloud.${STACK_NAME}.version=<version>
 
-Where <version> can be specifed on the command-line or Abra can attempt to
+The <version> is determined by the recipe maintainer and is specified on the
-auto-generate it for you. The <recipe> configuration will be updated on the
+command-line. The <recipe> configuration will be updated on the local file
-local file system.
+system.
+
+You may invoke this command in "wizard" mode and be prompted for input:
+
+    abra recipe sync
+
 `,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, false)
+		recipe := internal.ValidateRecipeWithPrompt(c)
 
-		mainApp, err := internal.GetMainAppImage(recipe)
+		mainApp := internal.GetMainApp(recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
 
 		imagesTmp, err := getImageVersions(recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
 		mainAppVersion := imagesTmp[mainApp]
 
 		tags, err := recipe.Tags()
@@ -62,75 +60,49 @@ local file system.
 
 		nextTag := c.Args().Get(1)
 		if len(tags) == 0 && nextTag == "" {
-			logrus.Warnf("no git tags found for %s", recipe.Name)
+			logrus.Warnf("no tags found for %s", recipe.Name)
-			fmt.Println(fmt.Sprintf(`
-The following options are two types of initial semantic version that you can
-pick for %s that will be published in the recipe catalogue. This follows the
-semver convention (more on https://semver.org), here is a short cheatsheet
-
-    0.1.0: development release, still hacking. when you make a major upgrade
-           you increment the "y" part (i.e. 0.1.0 -> 0.2.0) and only move to
-           using the "x" part when things are stable.
-
-    1.0.0: public release, assumed to be working. you already have a stable
-           and reliable deployment of this app and feel relatively confident
-           about it.
-
-If you want people to be able alpha test your current config for %s but don't
-think it is quite reliable, go with 0.1.0 and people will know that things are
-likely to change.
-
-`, recipe.Name, recipe.Name))
 			var chosenVersion string
 			edPrompt := &survey.Select{
 				Message: "which version do you want to begin with?",
 				Options: []string{"0.1.0", "1.0.0"},
 			}
-
 			if err := survey.AskOne(edPrompt, &chosenVersion); err != nil {
 				logrus.Fatal(err)
 			}
-
 			nextTag = fmt.Sprintf("%s+%s", chosenVersion, mainAppVersion)
 		}
 
 		if nextTag == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
-			latestRelease := tags[len(tags)-1]
+			if err := internal.PromptBumpType(""); err != nil {
-			if err := internal.PromptBumpType("", latestRelease); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
 		if nextTag == "" {
-			recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+			recipeDir := path.Join(config.APPS_DIR, recipe.Name)
 			repo, err := git.PlainOpen(recipeDir)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-
 			var lastGitTag tagcmp.Tag
 			iter, err := repo.Tags()
 			if err != nil {
 				logrus.Fatal(err)
 			}
-
 			if err := iter.ForEach(func(ref *plumbing.Reference) error {
 				obj, err := repo.TagObject(ref.Hash())
 				if err != nil {
 					return err
 				}
-
 				tagcmpTag, err := tagcmp.Parse(obj.Name)
 				if err != nil {
 					return err
 				}
-
 				if (lastGitTag == tagcmp.Tag{}) {
 					lastGitTag = tagcmpTag
 				} else if tagcmpTag.IsGreaterThan(lastGitTag) {
 					lastGitTag = tagcmpTag
 				}
-
 				return nil
 			}); err != nil {
 				logrus.Fatal(err)
@@ -141,7 +113,7 @@ likely to change.
 			if bumpType != 0 {
 				// a bitwise check if the number is a power of 2
 				if (bumpType & (bumpType - 1)) != 0 {
-					logrus.Fatal("you can only use one version flag: --major, --minor or --patch")
+					logrus.Fatal("you can only use one of: --major, --minor, --patch.")
 				}
 			}
 
@@ -152,14 +124,12 @@ likely to change.
 					if err != nil {
 						logrus.Fatal(err)
 					}
-
 					newTag.Patch = strconv.Itoa(now + 1)
 				} else if internal.Minor {
 					now, err := strconv.Atoi(newTag.Minor)
 					if err != nil {
 						logrus.Fatal(err)
 					}
-
 					newTag.Patch = "0"
 					newTag.Minor = strconv.Itoa(now + 1)
 				} else if internal.Major {
@@ -167,7 +137,6 @@ likely to change.
 					if err != nil {
 						logrus.Fatal(err)
 					}
-
 					newTag.Patch = "0"
 					newTag.Minor = "0"
 					newTag.Major = strconv.Itoa(now + 1)
@@ -184,16 +153,44 @@ likely to change.
 		}
 
 		mainService := "app"
+		var services []string
+		hasAppService := false
+		for _, service := range recipe.Config.Services {
+			services = append(services, service.Name)
+			if service.Name == "app" {
+				hasAppService = true
+				logrus.Debugf("detected app service in %s", recipe.Name)
+			}
+		}
+
+		if !hasAppService {
+			logrus.Fatalf("%s has no main 'app' service?", recipe.Name)
+		}
+
+		logrus.Debugf("selecting %s as the service to sync version label", mainService)
+
 		label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", nextTag)
 		if !internal.Dry {
-			if err := recipe.UpdateLabel("compose.y*ml", mainService, label); err != nil {
+			if err := recipe.UpdateLabel(mainService, label); err != nil {
 				logrus.Fatal(err)
 			}
+			logrus.Infof("synced label '%s' to service '%s'", label, mainService)
 		} else {
-			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
+			logrus.Infof("dry run only: NOT syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
 }

cli/recipe/upgrade.go

@@ -9,16 +9,14 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 type imgPin struct {
@@ -26,10 +24,10 @@ type imgPin struct {
 	version tagcmp.Tag
 }
 
-var recipeUpgradeCommand = cli.Command{
+var recipeUpgradeCommand = &cli.Command{
 	Name:    "upgrade",
-	Aliases: []string{"u"},
 	Usage:   "Upgrade recipe image tags",
+	Aliases: []string{"u"},
 	Description: `
 This command reads and attempts to parse all image tags within the given
 <recipe> configuration and prompt with more recent tags to upgrade to. It will
@@ -39,27 +37,19 @@ Some image tags cannot be parsed because they do not follow some sort of
 semver-like convention. In this case, all possible tags will be listed and it
 is up to the end-user to decide.
 
-The command is interactive and will show a select input which allows you to
-make a seclection. Use the "?" key to see more help on navigating this
-interface.
-
 You may invoke this command in "wizard" mode and be prompted for input:
 
     abra recipe upgrade
+
 `,
-	BashComplete: autocomplete.RecipeNameComplete,
 	ArgsUsage: "<recipe>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.PatchFlag,
 		internal.MinorFlag,
 		internal.MajorFlag,
-		internal.AllTagsFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, true)
+		recipe := internal.ValidateRecipeWithPrompt(c)
 
 		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
 		if bumpType != 0 {
@@ -71,7 +61,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 
 		// check for versions file and load pinned versions
 		versionsPresent := false
-		recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+		recipeDir := path.Join(config.ABRA_DIR, "apps", recipe.Name)
 		versionsPath := path.Join(recipeDir, "versions")
 		var servicePins = make(map[string]imgPin)
 		if _, err := os.Stat(versionsPath); err == nil {
@@ -107,41 +97,43 @@ You may invoke this command in "wizard" mode and be prompted for input:
 		}
 
 		for _, service := range recipe.Config.Services {
+			catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
 			img, err := reference.ParseNormalizedNamed(service.Image)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-			regVersions, err := client.GetRegistryTags(img)
+			image := reference.Path(img)
+			regVersions, err := client.GetRegistryTags(image)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+			logrus.Debugf("retrieved '%s' from remote registry for '%s'", regVersions, image)
 
-			image := reference.Path(img)
+			if strings.Contains(image, "library") {
-			logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
-			image = formatter.StripTagMeta(image)
+				// postgres:<tag>, i.e. images which do not have a username in the
-
+				// first position of the string
-			switch img.(type) {
+				image = strings.Split(image, "/")[1]
-			case reference.NamedTagged:
-				if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
-					logrus.Debugf("%s not considered semver-like", img.(reference.NamedTagged).Tag())
 			}
-			default:
+			semverLikeTag := true
-				logrus.Warnf("unable to read tag for image %s, is it missing? skipping upgrade for %s", image, service.Name)
+			if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
-				continue
+				logrus.Debugf("'%s' not considered semver-like", img.(reference.NamedTagged).Tag())
+				semverLikeTag = false
 			}
 
 			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
-			if err != nil {
+			if err != nil && semverLikeTag {
-				logrus.Warnf("unable to parse %s, error was: %s, skipping upgrade for %s", image, err.Error(), service.Name)
+				logrus.Fatal(err)
-				continue
 			}
-
+			logrus.Debugf("parsed '%s' for '%s'", tag, service.Name)
-			logrus.Debugf("parsed %s for %s", tag, service.Name)
-
 			var compatible []tagcmp.Tag
 			for _, regVersion := range regVersions {
-				other, err := tagcmp.Parse(regVersion)
+				other, err := tagcmp.Parse(regVersion.Name)
 				if err != nil {
 					continue // skip tags that cannot be parsed
 				}
@@ -151,21 +143,16 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 
-			logrus.Debugf("detected potential upgradable tags %s for %s", compatible, service.Name)
+			logrus.Debugf("detected potential upgradable tags '%s' for '%s'", compatible, service.Name)
 
 			sort.Sort(tagcmp.ByTagDesc(compatible))
 
-			if len(compatible) == 0 && !internal.AllTags {
+			if len(compatible) == 0 && semverLikeTag {
-				logrus.Info(fmt.Sprintf("no new versions available for %s, assuming %s is the latest (use -a/--all-tags to see all anyway)", image, tag))
+				logrus.Info(fmt.Sprintf("no new versions available for '%s', '%s' is the latest", image, tag))
 				continue // skip on to the next tag and don't update any compose files
 			}
 
-			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name)
+			var compatibleStrings []string
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			compatibleStrings := []string{"skip"}
 			for _, compat := range compatible {
 				skip := false
 				for _, catlVersion := range catlVersions {
@@ -178,7 +165,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 
-			logrus.Debugf("detected compatible upgradable tags %s for %s", compatibleStrings, service.Name)
+			logrus.Debugf("detected compatible upgradable tags '%s' for '%s'", compatibleStrings, service.Name)
 
 			var upgradeTag string
 			_, ok := servicePins[service.Name]
@@ -195,13 +182,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
 						}
 					}
 					if contains {
-						logrus.Infof("upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
+						logrus.Infof("Upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
 					} else {
 						logrus.Infof("service %s, image %s pinned to %s, no compatible upgrade found", service.Name, servicePins[service.Name].image, pinnedTagString)
 						continue
 					}
 				} else {
-					logrus.Fatalf("service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
+					logrus.Fatalf("Service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
 					continue
 				}
 			} else {
@@ -218,27 +205,23 @@ You may invoke this command in "wizard" mode and be prompted for input:
 						}
 					}
 					if upgradeTag == "" {
-						logrus.Warnf("not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants", tag.String(), compatible[0].String(), image)
+						logrus.Warnf("not upgrading from '%s' to '%s' for '%s', because the upgrade type is more serious than what user wants.", tag.String(), compatible[0].String(), image)
 						continue
 					}
 				} else {
-					msg := fmt.Sprintf("upgrade to which tag? (service: %s, image: %s, tag: %s)", service.Name, image, tag)
+					msg := fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
-					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) || internal.AllTags {
+					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
 						tag := img.(reference.NamedTagged).Tag()
-						if !internal.AllTags {
+						logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of '%s', listing all tags", tag))
-							logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of %s, listing all tags", tag))
-						}
 						msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
-						compatibleStrings = []string{"skip"}
+						compatibleStrings = []string{}
 						for _, regVersion := range regVersions {
-							compatibleStrings = append(compatibleStrings, regVersion)
+							compatibleStrings = append(compatibleStrings, regVersion.Name)
 						}
 					}
 
 					prompt := &survey.Select{
 						Message: msg,
-						Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
-						VimMode: true,
 						Options: compatibleStrings,
 					}
 					if err := survey.AskOne(prompt, &upgradeTag); err != nil {
@@ -246,17 +229,10 @@ You may invoke this command in "wizard" mode and be prompted for input:
 					}
 				}
 			}
-			if upgradeTag != "skip" {
+			if err := recipe.UpdateTag(image, upgradeTag); err != nil {
-				ok, err := recipe.UpdateTag(image, upgradeTag)
-				if err != nil {
 				logrus.Fatal(err)
 			}
-				if ok {
+			logrus.Infof("tag upgraded from '%s' to '%s' for '%s'", tag.String(), upgradeTag, image)
-					logrus.Infof("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)
-				}
-			} else {
-				logrus.Warnf("not upgrading %s, skipping as requested", image)
-			}
 		}
 
 		return nil

cli/recipe/version.go

@@ -1,42 +1,36 @@
 package recipe
 
 import (
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/formatter"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeVersionCommand = cli.Command{
+var recipeVersionCommand = &cli.Command{
 	Name:      "versions",
-	Aliases:   []string{"v"},
 	Usage:     "List recipe versions",
+	Aliases:   []string{"v"},
 	ArgsUsage: "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, false)
+		recipe := internal.ValidateRecipe(c)
 
-		catalogue, err := recipePkg.ReadRecipeCatalogue()
+		catalogue, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		recipeMeta, ok := catalogue[recipe.Name]
 		if !ok {
-			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
+			logrus.Fatalf("'%s' recipe doesn't exist?", recipe.Name)
 		}
 
 		tableCol := []string{"Version", "Service", "Image", "Tag", "Digest"}
 		table := formatter.CreateTable(tableCol)
 
-		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
+		for _, serviceVersion := range recipeMeta.Versions {
-			for tag, meta := range recipeMeta.Versions[i] {
+			for tag, meta := range serviceVersion {
 				for service, serviceMeta := range meta {
 					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag, serviceMeta.Digest})
 				}
@@ -44,12 +38,7 @@ var recipeVersionCommand = cli.Command{
 		}
 
 		table.SetAutoMergeCells(true)
-
-		if table.NumLines() > 0 {
 		table.Render()
-		} else {
-			logrus.Fatalf("%s has no published versions?", recipe.Name)
-		}
 
 		return nil
 	},

cli/record/list.go

@@ -1,29 +1,26 @@
 package record
 
 import (
-	"context"
 	"fmt"
 	"strconv"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/libdns/gandi"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // RecordListCommand lists domains.
-var RecordListCommand = cli.Command{
+var RecordListCommand = &cli.Command{
 	Name:      "list",
 	Usage:     "List domain name records",
 	Aliases:   []string{"ls"},
 	ArgsUsage: "<zone>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
 		internal.DNSProviderFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command lists all domain name records managed by a 3rd party provider for
 a specific zone.
@@ -49,16 +46,16 @@ are listed. This zone must already be created on your provider account.
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
 		}
 
-		records, err := provider.GetRecords(context.Background(), zone)
+		records, err := provider.GetRecords(c.Context, zone)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 
 		for _, record := range records {
 			value := record.Value

cli/record/new.go

@@ -1,29 +1,26 @@
 package record
 
 import (
-	"context"
 	"fmt"
 	"strconv"
+	"time"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/dns"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/libdns/gandi"
 	"github.com/libdns/libdns"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // RecordNewCommand creates a new domain name record.
-var RecordNewCommand = cli.Command{
+var RecordNewCommand = &cli.Command{
 	Name:      "new",
 	Usage:     "Create a new domain record",
 	Aliases:   []string{"n"},
 	ArgsUsage: "<zone>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.DNSProviderFlag,
 		internal.DNSTypeFlag,
 		internal.DNSNameFlag,
@@ -31,7 +28,6 @@ var RecordNewCommand = cli.Command{
 		internal.DNSTTLFlag,
 		internal.DNSPriorityFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command creates a new domain name record for a specific zone.
 
@@ -42,7 +38,7 @@ Example:
 
     abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
 
-You may also invoke this command in "wizard" mode and be prompted for input:
+You may also invoke this command in "wizard" mode and be prompted for input
 
     abra record new
 `,
@@ -64,7 +60,7 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
 		}
 
 		if err := internal.EnsureDNSTypeFlag(c); err != nil {
@@ -79,23 +75,18 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 			logrus.Fatal(err)
 		}
 
-		ttl, err := dns.GetTTL(internal.DNSTTL)
-		if err != nil {
-			return err
-		}
-
 		record := libdns.Record{
 			Type:  internal.DNSType,
 			Name:  internal.DNSName,
 			Value: internal.DNSValue,
-			TTL:   ttl,
+			TTL:   time.Duration(internal.DNSTTL),
 		}
 
 		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
 			record.Priority = internal.DNSPriority
 		}
 
-		records, err := provider.GetRecords(context.Background(), zone)
+		records, err := provider.GetRecords(c.Context, zone)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -104,18 +95,15 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 			if existingRecord.Type == record.Type &&
 				existingRecord.Name == record.Name &&
 				existingRecord.Value == record.Value {
-				logrus.Fatalf("%s record for %s already exists?", record.Type, zone)
+				logrus.Fatal("provider library reports that this record already exists?")
 			}
 		}
 
 		createdRecords, err := provider.SetRecords(
-			context.Background(),
+			c.Context,
 			zone,
 			[]libdns.Record{record},
 		)
-		if err != nil {
-			logrus.Fatal(err)
-		}
 
 		if len(createdRecords) == 0 {
 			logrus.Fatal("provider library reports that no record was created?")
@@ -124,7 +112,7 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 		createdRecord := createdRecords[0]
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 
 		value := createdRecord.Value
 		if len(createdRecord.Value) > 30 {

cli/record/record.go

@@ -1,13 +1,13 @@
 package record
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // RecordCommand supports managing DNS entries.
-var RecordCommand = cli.Command{
+var RecordCommand = &cli.Command{
 	Name:      "record",
-	Usage:     "Manage domain name records",
+	Usage:     "Manage domain name records via 3rd party providers",
 	Aliases:   []string{"rc"},
 	ArgsUsage: "<record>",
 	Description: `
@@ -28,8 +28,9 @@ library documentation for more. It supports many existing providers and allows
 to implement new provider support easily.
 
     https://pkg.go.dev/github.com/libdns/libdns
+
 `,
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		RecordListCommand,
 		RecordNewCommand,
 		RecordRemoveCommand,

cli/record/remove.go

@@ -1,34 +1,30 @@
 package record
 
 import (
-	"context"
 	"fmt"
 	"strconv"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/libdns/gandi"
 	"github.com/libdns/libdns"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // RecordRemoveCommand lists domains.
-var RecordRemoveCommand = cli.Command{
+var RecordRemoveCommand = &cli.Command{
 	Name:      "remove",
 	Usage:     "Remove a domain name record",
 	Aliases:   []string{"rm"},
 	ArgsUsage: "<zone>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.DNSProviderFlag,
 		internal.DNSTypeFlag,
 		internal.DNSNameFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command removes a domain name record for a specific zone.
 
@@ -41,7 +37,7 @@ Example:
 
     abra record remove foo.com -p gandi -t A -n myapp
 
-You may also invoke this command in "wizard" mode and be prompted for input:
+You may also invoke this command in "wizard" mode and be prompted for input
 
     abra record rm
 `,
@@ -63,7 +59,7 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
 		}
 
 		if err := internal.EnsureDNSTypeFlag(c); err != nil {
@@ -74,7 +70,7 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 			logrus.Fatal(err)
 		}
 
-		records, err := provider.GetRecords(context.Background(), zone)
+		records, err := provider.GetRecords(c.Context, zone)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -92,7 +88,7 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 		}
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 
 		value := toDelete.Value
 		if len(toDelete.Value) > 30 {
@@ -109,7 +105,6 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 
 		table.Render()
 
-		if !internal.NoInput {
 		response := false
 		prompt := &survey.Confirm{
 			Message: "continue with record deletion?",
@@ -122,9 +117,8 @@ You may also invoke this command in "wizard" mode and be prompted for input:
 		if !response {
 			logrus.Fatal("exiting as requested")
 		}
-		}
 
-		_, err = provider.DeleteRecords(context.Background(), zone, []libdns.Record{toDelete})
+		_, err = provider.DeleteRecords(c.Context, zone, []libdns.Record{toDelete})
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/server/add.go

@@ -1,15 +1,16 @@
 package server
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"os/user"
+	"path"
 	"path/filepath"
 	"strings"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -22,7 +23,7 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var (
@@ -41,26 +42,32 @@ such purposes. Docker stable is now installed by default by this script. The
 source for this script can be seen here:
 
     https://github.com/docker/docker-install
+
 `
 )
 
 var local bool
 var localFlag = &cli.BoolFlag{
-	Name:        "local, l",
+	Name:        "local",
+	Aliases:     []string{"l"},
+	Value:       false,
 	Usage:       "Use local server",
 	Destination: &local,
 }
 
 var provision bool
 var provisionFlag = &cli.BoolFlag{
-	Name:        "provision, p",
+	Name:        "provision",
+	Aliases:     []string{"p"},
+	Value:       false,
 	Usage:       "Provision server so it can deploy apps",
 	Destination: &provision,
 }
 
 var sshAuth string
 var sshAuthFlag = &cli.StringFlag{
-	Name:        "ssh-auth, s",
+	Name:        "ssh-auth",
+	Aliases:     []string{"sh"},
 	Value:       "identity-file",
 	Usage:       "Select SSH authentication method (identity-file, password)",
 	Destination: &sshAuth,
@@ -68,11 +75,22 @@ var sshAuthFlag = &cli.StringFlag{
 
 var askSudoPass bool
 var askSudoPassFlag = &cli.BoolFlag{
-	Name:        "ask-sudo-pass, a",
+	Name:        "ask-sudo-pass",
+	Aliases:     []string{"as"},
+	Value:       false,
 	Usage:       "Ask for sudo password",
 	Destination: &askSudoPass,
 }
 
+var traefik bool
+var traefikFlag = &cli.BoolFlag{
+	Name:        "traefik",
+	Aliases:     []string{"t"},
+	Value:       false,
+	Usage:       "Deploy traefik",
+	Destination: &traefik,
+}
+
 func cleanUp(domainName string) {
 	logrus.Warnf("cleaning up context for %s", domainName)
 	if err := client.DeleteContext(domainName); err != nil {
@@ -80,7 +98,7 @@ func cleanUp(domainName string) {
 	}
 
 	logrus.Warnf("cleaning up server directory for %s", domainName)
-	if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, domainName)); err != nil {
+	if err := os.RemoveAll(filepath.Join(config.ABRA_SERVER_FOLDER, domainName)); err != nil {
 		logrus.Fatal(err)
 	}
 }
@@ -99,17 +117,7 @@ func installDockerLocal(c *cli.Context) error {
 		logrus.Fatal("exiting as requested")
 	}
 
-	for _, exe := range []string{"wget", "bash"} {
+	cmd := exec.Command("bash", "-c", "curl -s https://get.docker.com | bash")
-		exists, err := ensureLocalExecutable(exe)
-		if err != nil {
-			return err
-		}
-		if !exists {
-			return fmt.Errorf("%s missing, please install it", exe)
-		}
-	}
-
-	cmd := exec.Command("bash", "-c", "wget -O- https://get.docker.com | bash")
 	if err := internal.RunCmd(cmd); err != nil {
 		return err
 	}
@@ -128,17 +136,15 @@ func newLocalServer(c *cli.Context, domainName string) error {
 	}
 
 	if provision {
-		exists, err := ensureLocalExecutable("docker")
+		out, err := exec.Command("which", "docker").Output()
 		if err != nil {
 			return err
 		}
-
+		if string(out) == "" {
-		if !exists {
 			if err := installDockerLocal(c); err != nil {
 				return err
 			}
 		}
-
 		if err := initSwarmLocal(c, cl, domainName); err != nil {
 			if !strings.Contains(err.Error(), "proxy already exists") {
 				logrus.Fatal(err)
@@ -146,6 +152,12 @@ func newLocalServer(c *cli.Context, domainName string) error {
 		}
 	}
 
+	if traefik {
+		if err := deployTraefik(c, cl, domainName); err != nil {
+			return err
+		}
+	}
+
 	logrus.Info("local server has been added")
 
 	return nil
@@ -183,133 +195,65 @@ func newClient(c *cli.Context, domainName string) (*dockerClient.Client, error)
 }
 
 func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, domainName string) error {
-	exists, err := ensureRemoteExecutable("docker", sshCl)
+	result, err := sshCl.Exec("which docker")
-	if err != nil {
+	if err != nil && string(result) != "" {
 		return err
 	}
 
-	if !exists {
+	if string(result) == "" {
 		fmt.Println(fmt.Sprintf(dockerInstallMsg, domainName))
 
 		response := false
 		prompt := &survey.Confirm{
 			Message: fmt.Sprintf("attempt install docker on %s?", domainName),
 		}
-
 		if err := survey.AskOne(prompt, &response); err != nil {
 			return err
 		}
-
 		if !response {
 			logrus.Fatal("exiting as requested")
 		}
 
-		exes := []string{"wget", "bash"}
+		cmd := "curl -s https://get.docker.com | bash"
-		if askSudoPass {
-			exes = append(exes, "ssh-askpass")
-		}
-
-		for _, exe := range exes {
-			exists, err := ensureRemoteExecutable(exe, sshCl)
-			if err != nil {
-				return err
-			}
-			if !exists {
-				return fmt.Errorf("%s missing on remote, please install it", exe)
-			}
-		}
 
 		var sudoPass string
 		if askSudoPass {
-			cmd := "wget -O- https://get.docker.com | bash"
-
 			prompt := &survey.Password{
 				Message: "sudo password?",
 			}
-
 			if err := survey.AskOne(prompt, &sudoPass); err != nil {
 				return err
 			}
-
+			logrus.Debugf("running '%s' on %s now with sudo password", cmd, domainName)
-			logrus.Debugf("running %s on %s now with sudo password", cmd, domainName)
-
-			if sudoPass == "" {
-				return fmt.Errorf("missing sudo password but requested --ask-sudo-pass?")
-			}
-
-			logrus.Warn("installing docker, this could take some time...")
-
 			if err := ssh.RunSudoCmd(cmd, sudoPass, sshCl); err != nil {
-				fmt.Print(fmt.Sprintf(`
-Abra was unable to bootstrap Docker, see below for logs:
-
-
-%s
-
-If nothing works, you try running the Docker install script manually on your server:
-
-    wget -O- https://get.docker.com | bash
-
-`, string(err.Error())))
-				logrus.Fatal("Process exited with status 1")
-			}
-
-			logrus.Infof("docker is installed on %s", domainName)
-
-			remoteUser := sshCl.SSHClient.Conn.User()
-			logrus.Infof("adding %s to docker group", remoteUser)
-			permsCmd := fmt.Sprintf("sudo usermod -aG docker %s", remoteUser)
-			if err := ssh.RunSudoCmd(permsCmd, sudoPass, sshCl); err != nil {
 				return err
 			}
 		} else {
-			cmd := "wget -O- https://get.docker.com | bash"
+			logrus.Debugf("running '%s' on %s now without sudo password", cmd, domainName)
-
+			if err := ssh.Exec(cmd, sshCl); err != nil {
-			logrus.Debugf("running %s on %s now without sudo password", cmd, domainName)
+				return err
-
-			logrus.Warn("installing docker, this could take some time...")
-
-			if out, err := sshCl.Exec(cmd); err != nil {
-				fmt.Print(fmt.Sprintf(`
-Abra was unable to bootstrap Docker, see below for logs:
-
-
-%s
-
-This could be due to a number of things but one of the most common is that your
-server user account does not have sudo access, and if it does, you need to pass
-"--ask-sudo-pass" in order to supply Abra with your password.
-
-If nothing works, you try running the Docker install script manually on your server:
-
-    wget -O- https://get.docker.com | bash
-
-`, string(out)))
-				logrus.Fatal(err)
-			}
-
-			logrus.Infof("docker is installed on %s", domainName)
 			}
 		}
+	}
+
+	logrus.Infof("docker is already installed on %s", domainName)
 
 	return nil
 }
 
 func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string) error {
 	initReq := swarm.InitRequest{ListenAddr: "0.0.0.0:2377"}
-	if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
+	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
-		if strings.Contains(err.Error(), "is already part of a swarm") ||
+		if !strings.Contains(err.Error(), "is already part of a swarm") {
-			strings.Contains(err.Error(), "must specify a listening address") {
-			logrus.Infof("swarm mode already initialised on %s", domainName)
-		} else {
 			return err
 		}
+		logrus.Info("swarm mode already initialised on local server")
 	} else {
 		logrus.Infof("initialised swarm mode on local server")
 	}
 
 	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
-	if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
+	if _, err := cl.NetworkCreate(c.Context, "proxy", netOpts); err != nil {
 		if !strings.Contains(err.Error(), "proxy already exists") {
 			return err
 		}
@@ -331,19 +275,18 @@ func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error
 		ListenAddr:    "0.0.0.0:2377",
 		AdvertiseAddr: ipv4,
 	}
-	if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
+	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
-		if strings.Contains(err.Error(), "is already part of a swarm") ||
+		if !strings.Contains(err.Error(), "is already part of a swarm") ||
-			strings.Contains(err.Error(), "must specify a listening address") {
+			!strings.Contains(err.Error(), "must specify a listening address") {
-			logrus.Infof("swarm mode already initialised on %s", domainName)
-		} else {
 			return err
 		}
+		logrus.Infof("swarm mode already initialised on %s", domainName)
 	} else {
 		logrus.Infof("initialised swarm mode on %s", domainName)
 	}
 
 	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
-	if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
+	if _, err := cl.NetworkCreate(c.Context, "proxy", netOpts); err != nil {
 		if !strings.Contains(err.Error(), "proxy already exists") {
 			return err
 		}
@@ -365,33 +308,78 @@ func createServerDir(domainName string) error {
 	return nil
 }
 
-var serverAddCommand = cli.Command{
+func deployTraefik(c *cli.Context, cl *dockerClient.Client, domainName string) error {
+	internal.NoInput = true
+
+	internal.RecipeName = "traefik"
+	internal.NewAppServer = domainName
+	internal.Domain = fmt.Sprintf("%s.%s", "traefik", domainName)
+	internal.NewAppName = fmt.Sprintf("%s_%s", "traefik", config.SanitiseAppName(domainName))
+
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", internal.Domain, fmt.Sprintf("%s.env", internal.NewAppName))
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
+		fmt.Println(fmt.Sprintf(`
+	You specified "--traefik/-t" and that means that Abra will now try to
+	automatically create a new Traefik app on %s.
+	`, internal.NewAppServer))
+
+		tableCol := []string{"recipe", "domain", "server", "name"}
+		table := abraFormatter.CreateTable(tableCol)
+		table.Append([]string{internal.RecipeName, internal.Domain, internal.NewAppServer, internal.NewAppName})
+
+		if err := internal.NewAction(c); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		logrus.Infof("%s already exists, not creating again", appEnvPath)
+	}
+
+	internal.AppName = internal.NewAppName
+	if err := internal.DeployAction(c); err != nil {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}
+
+var serverAddCommand = &cli.Command{
 	Name:  "add",
-	Aliases: []string{"a"},
 	Usage: "Add a server to your configuration",
 	Description: `
 This command adds a new server to your configuration so that it can be managed
-by Abra. This command can also provision your server ("--provision/-p") with a
+by Abra. This can be useful when you already have a server provisioned and want
-Docker installation so that it is capable of hosting Co-op Cloud apps.
+to start running Abra commands against it. 
 
-Abra will default to expecting that you have a running ssh-agent and are using
+This command can also provision your server ("--provision/-p") so that it is
-SSH keys to connect to your new server. Abra will also read your SSH config
+capable of hosting Co-op Cloud apps. Abra will default to expecting that you
-(matching "Host" as <domain>). SSH connection details precedence follows as
+have a running ssh-agent and are using SSH keys to connect to your new server.
-such: command-line > SSH config > guessed defaults.
+Abra will also read your SSH config (matching "Host" as <domain>). SSH
+connection details precedence follows as such: command-line > SSH config >
+guessed defaults.
 
 If you have no SSH key configured for this host and are instead using password
 authentication, you may pass "--ssh-auth password" to have Abra ask you for the
 password. "--ask-sudo-pass" may be passed if you run your provisioning commands
 via sudo privilege escalation.
 
-The <domain> argument must be a publicy accessible domain name which points to
+If "--local" is passed, then Abra assumes that the current local server is
-your server. You should working SSH access to this server already, Abra will
+intended as the target server. This is useful when you want to have your entire
-assume port 22 and will use your current system username to make an initial
+Co-op Cloud config located on the server itself, and not on your local
-connection. You can use the <user> and <port> arguments to adjust this.
+developer machine.
 
 Example:
 
-    abra server add varia.zone glodemodem 12345 -p
+    abra server add --local
+
+Otherwise, you may specify a remote server. The <domain> argument must be a
+publicy accessible domain name which points to your server. You should have SSH
+access to this server, Abra will assume port 22 and will use your current
+system username to make an initial connection. You can use the <user> and
+<port> arguments to adjust this.
+
+Example:
+
+    abra server add --provision --traefik varia.zone glodemodem 12345
 
 Abra will construct the following SSH connection and Docker context:
 
@@ -399,34 +387,34 @@ Abra will construct the following SSH connection and Docker context:
 
 All communication between Abra and the server will use this SSH connection.
 
-If "--local" is passed, then Abra assumes that the current local server is
+In this example, Abra will run the following operations:
-intended as the target server. This is useful when you want to have your entire
+
-Co-op Cloud config located on the server itself, and not on your local
+    1. Install Docker
-developer machine.
+		2. Initialise Swarm mode
+		3. Deploy Traefik (core web proxy)
+
+You may omit flags to avoid performing this provisioning logic.
 `,
+	Aliases: []string{"a"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		localFlag,
 		provisionFlag,
 		sshAuthFlag,
 		askSudoPassFlag,
+		traefikFlag,
 	},
-	Before:    internal.SubCommandBefore,
 	ArgsUsage: "<domain> [<user>] [<port>]",
 	Action: func(c *cli.Context) error {
-		if len(c.Args()) > 0 && local || !internal.ValidateSubCmdFlags(c) {
+		if c.Args().Len() > 0 && local || !internal.ValidateSubCmdFlags(c) {
-			err := errors.New("cannot use <domain> and --local together")
+			err := errors.New("cannot use '<domain>' and '--local' together")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
 		if sshAuth != "password" && sshAuth != "identity-file" {
-			err := errors.New("--ssh-auth only accepts identity-file or password")
+			err := errors.New("--ssh-auth only accepts 'identity-file' or 'password'")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
-		domainName := internal.ValidateDomain(c)
-
 		if local {
 			if err := newLocalServer(c, "default"); err != nil {
 				logrus.Fatal(err)
@@ -434,6 +422,11 @@ developer machine.
 			return nil
 		}
 
+		domainName, err := internal.ValidateDomain(c)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		username := c.Args().Get(1)
 		if username == "" {
 			systemUser, err := user.Current()
@@ -458,17 +451,14 @@ developer machine.
 
 		cl, err := newClient(c, domainName)
 		if err != nil {
-			cleanUp(domainName)
+			logrus.Fatal(err)
-			logrus.Debugf("failed to construct client for %s, saw %s", domainName, err.Error())
-			logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
 		}
 
 		if provision {
 			logrus.Debugf("attempting to construct SSH client for %s", domainName)
 			sshCl, err := ssh.New(domainName, sshAuth, username, port)
 			if err != nil {
-				cleanUp(domainName)
+				logrus.Fatal(err)
-				logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
 			}
 			defer sshCl.Close()
 			logrus.Debugf("successfully created SSH client for %s", domainName)
@@ -481,31 +471,17 @@ developer machine.
 			}
 		}
 
-		if _, err := cl.Info(context.Background()); err != nil {
+		if _, err := cl.Info(c.Context); err != nil {
 			cleanUp(domainName)
-			logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
+			logrus.Fatalf("couldn't make a remote docker connection to %s? use --provision/-p to attempt to install", domainName)
+		}
+
+		if traefik {
+			if err := deployTraefik(c, cl, domainName); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 		return nil
 	},
 }
-
-// ensureLocalExecutable ensures that an executable is present on the local machine
-func ensureLocalExecutable(exe string) (bool, error) {
-	out, err := exec.Command("which", exe).Output()
-	if err != nil {
-		return false, err
-	}
-
-	return string(out) != "", nil
-}
-
-// ensureRemoteExecutable ensures that an executable is present on a remote machine
-func ensureRemoteExecutable(exe string, sshCl *ssh.Client) (bool, error) {
-	out, err := sshCl.Exec(fmt.Sprintf("which %s", exe))
-	if err != nil && string(out) != "" {
-		return false, err
-	}
-
-	return string(out) != "", nil
-}

cli/server/list.go

@@ -3,23 +3,20 @@ package server
 import (
 	"strings"
 
-	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/context"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/docker/cli/cli/connhelper/ssh"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var serverListCommand = cli.Command{
+var serverListCommand = &cli.Command{
 	Name:      "list",
 	Aliases:   []string{"ls"},
 	Usage:     "List managed servers",
-	Flags: []cli.Flag{
+	ArgsUsage: " ",
-		internal.DebugFlag,
+	HideHelp:  true,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		dockerContextStore := context.NewDefaultDockerContextStore()
 		contexts, err := dockerContextStore.Store.List()

cli/server/logs.go

@@ -0,0 +1,101 @@
+package server
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Taillen string
+var TaillenFlag = &cli.StringFlag{
+	Name:        "tail",
+	Aliases:     []string{"t"},
+	Value:       "5",
+	Destination: &Taillen,
+	Usage:       "change how many lines are shown",
+}
+
+var serverLogsCommand = &cli.Command{
+	Name:      "logs",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<server>",
+	Usage:     "show logs from all apps from server",
+	Flags: []cli.Flag{
+		TaillenFlag,
+		internal.StderrFlag,
+		internal.StdoutFlag,
+		internal.HealthcheckFlag,
+	},
+	Action: func(c *cli.Context) error {
+		serverName, err := internal.ValidateServer(c)
+		serviceName := ""
+		if !internal.Stderr && !internal.Stdout && !internal.Healthcheck {
+			internal.Stderr = true
+			internal.Stdout = true
+			internal.Healthcheck = true
+		}
+
+		logrus.Debugf("flags parsed. --stderr: %t, --stdout: %t, --healthcheck: %t", internal.Stderr, internal.Stdout, internal.Healthcheck)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		appMap, err := config.LoadAppFiles(serverName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logOpts := types.ContainerLogsOptions{
+			Details:    false,
+			Follow:     false,
+			ShowStderr: internal.Stderr,
+			ShowStdout: internal.Stdout,
+			Tail:       Taillen,
+			Timestamps: true,
+		}
+
+		var appFiles []config.App
+		for appname, _ := range appMap {
+			app := internal.ValidateAppByName(c, appname)
+			appFiles = append(appFiles, app)
+		}
+		for _, app := range appFiles {
+			fmt.Println(app)
+			logrus.Debugf("checking logs for: %s", app.Name)
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("tailing logs for all services")
+			filters := filters.NewArgs()
+			filters.Add("name", service)
+			serviceOpts := types.ServiceListOptions{Filters: filters}
+			services, err := cl.ServiceList(c.Context, serviceOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info(app.StackName())
+			for {
+				_, err = io.Copy(os.Stdout, logs)
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			logs.Close()
+		}
+		return nil
+	},
+}

cli/server/new.go

@@ -1,17 +1,16 @@
 package server
 
 import (
-	"context"
 	"fmt"
 	"strings"
 
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/libcapsul"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 func newHetznerCloudVPS(c *cli.Context) error {
@@ -28,7 +27,7 @@ func newHetznerCloudVPS(c *cli.Context) error {
 			continue
 		}
 
-		sshKey, _, err := client.SSHKey.GetByName(context.Background(), sshKey)
+		sshKey, _, err := client.SSHKey.GetByName(c.Context, sshKey)
 		if err != nil {
 			return err
 		}
@@ -44,18 +43,13 @@ func newHetznerCloudVPS(c *cli.Context) error {
 		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
 	}
 
-	sshKeyIDs := strings.Join(sshKeysRaw, "\n")
-	if sshKeyIDs == "" {
-		sshKeyIDs = "N/A (password auth)"
-	}
-
 	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
 	table := formatter.CreateTable(tableColumns)
 	table.Append([]string{
 		internal.HetznerCloudName,
 		internal.HetznerCloudType,
 		internal.HetznerCloudImage,
-		sshKeyIDs,
+		strings.Join(sshKeysRaw, "\n"),
 		internal.HetznerCloudLocation,
 	})
 	table.Render()
@@ -73,7 +67,7 @@ func newHetznerCloudVPS(c *cli.Context) error {
 		logrus.Fatal("exiting as requested")
 	}
 
-	res, _, err := client.Server.Create(context.Background(), serverOpts)
+	res, _, err := client.Server.Create(c.Context, serverOpts)
 	if err != nil {
 		return err
 	}
@@ -102,18 +96,9 @@ Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
 not list this server)! You will need to assign a domain name record ("abra
 record new") and add the server to your Abra configuration ("abra server add")
 to have a working server that you can deploy Co-op Cloud apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A %s
-    *  1800 IN A %s
-
 	`,
 		internal.HetznerCloudName, ip, rootPassword,
-		ip, ip, ip,
+		ip,
 	))
 
 	return nil
@@ -184,21 +169,12 @@ Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
 not list this server)! You will need to assign a domain name record ("abra
 record new") and add the server to your Abra configuration ("abra server add")
 to have a working server that you can deploy Co-op Cloud apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A <your-capsul-ip>
-    *  1800 IN A <your-capsul-ip>
-
 	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
 
 	return nil
 }
 
-var serverNewCommand = cli.Command{
+var serverNewCommand = &cli.Command{
 	Name:    "new",
 	Aliases: []string{"n"},
 	Usage:   "Create a new server using a 3rd party provider",
@@ -220,9 +196,8 @@ API tokens are read from the environment if specified, e.g.
 
 Where "$provider_TOKEN" is the expected env var format.
 `,
+	ArgsUsage: "<provider>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ServerProviderFlag,
 
 		// Capsul
@@ -240,7 +215,6 @@ Where "$provider_TOKEN" is the expected env var format.
 		internal.HetznerCloudLocationFlag,
 		internal.HetznerCloudAPITokenFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		if err := internal.EnsureServerProvider(); err != nil {
 			logrus.Fatal(err)

cli/server/remove.go

@@ -1,24 +1,25 @@
 package server
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var rmServer bool
 var rmServerFlag = &cli.BoolFlag{
-	Name:        "server, s",
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       false,
 	Usage:       "remove the actual server also",
 	Destination: &rmServer,
 }
@@ -49,7 +50,7 @@ func rmHetznerCloudVPS(c *cli.Context) error {
 
 	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
 
-	server, _, err := client.Server.Get(context.Background(), internal.HetznerCloudName)
+	server, _, err := client.Server.Get(c.Context, internal.HetznerCloudName)
 	if err != nil {
 		return err
 	}
@@ -88,7 +89,7 @@ destroyed.
 		logrus.Fatal("exiting as requested")
 	}
 
-	_, err = client.Server.Delete(context.Background(), server)
+	_, err = client.Server.Delete(c.Context, server)
 	if err != nil {
 		return err
 	}
@@ -98,10 +99,10 @@ destroyed.
 	return nil
 }
 
-var serverRemoveCommand = cli.Command{
+var serverRemoveCommand = &cli.Command{
 	Name:      "remove",
 	Aliases:   []string{"rm"},
-	ArgsUsage: "[<server>]",
+	ArgsUsage: "<server>",
 	Usage:     "Remove a managed server",
 	Description: `
 This command removes a server from Abra management.
@@ -115,44 +116,17 @@ underlying client connection context. This server will then be lost in time,
 like tears in rain.
 `,
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		rmServerFlag,
-		internal.ServerProviderFlag,
 
 		// Hetzner
 		internal.HetznerCloudNameFlag,
 		internal.HetznerCloudAPITokenFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		serverName := internal.ValidateServer(c)
+		serverName, err := internal.ValidateServer(c)
-
+		if err != nil {
-		warnMsg := `Did not pass -s/--server for actual server deletion, prompting!
-
-Abra doesn't currently know if it helped you create this server with one of the
-3rd party integrations (e.g. Capsul). You have a choice here to actually,
-really and finally destroy this server using those integrations. If you want to
-do this, choose Yes.
-
-If you just want to remove the server config files & context, choose No.
-`
-
-		if !rmServer {
-			logrus.Warn(fmt.Sprintf(warnMsg))
-
-			response := false
-			prompt := &survey.Confirm{
-				Message: "delete actual live server?",
-			}
-			if err := survey.AskOne(prompt, &response); err != nil {
 			logrus.Fatal(err)
 		}
-			if response {
-				logrus.Info("setting -s/--server and attempting to remove actual server")
-				rmServer = true
-			}
-		}
 
 		if rmServer {
 			if err := internal.EnsureServerProvider(); err != nil {
@@ -167,17 +141,18 @@ If you just want to remove the server config files & context, choose No.
 					logrus.Fatal(err)
 				}
 			}
+
 		}
 
 		if err := client.DeleteContext(serverName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, serverName)); err != nil {
+		if err := os.RemoveAll(filepath.Join(config.ABRA_SERVER_FOLDER, serverName)); err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Infof("server at %s has been lost in time, like tears in rain", serverName)
+		logrus.Infof("server at '%s' has been lost in time, like tears in rain", serverName)
 
 		return nil
 	},

cli/server/server.go

@@ -1,14 +1,14 @@
 package server
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // ServerCommand defines the `abra server` command and its subcommands
-var ServerCommand = cli.Command{
+var ServerCommand = &cli.Command{
 	Name:    "server",
 	Aliases: []string{"s"},
-	Usage:   "Manage servers",
+	Usage:   "Manage servers via 3rd party providers",
 	Description: `
 These commands support creating, managing and removing servers using 3rd party
 integrations. 
@@ -18,10 +18,11 @@ already have a server, you can add it to your configuration using "abra server
 add". Abra can provision servers so that they are ready to deploy Co-op Cloud
 apps, see available flags on "server add" for more.
 `,
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		serverNewCommand,
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,
+		serverLogsCommand,
 	},
 }

cli/upgrade.go

@@ -0,0 +1,35 @@
+package cli
+
+import (
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var RC bool
+var RCFlag = &cli.BoolFlag{
+	Name:        "rc",
+	Value:       false,
+	Destination: &RC,
+	Usage:       "Insatll the latest Release Candidate",
+}
+
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = &cli.Command{
+	Name:  "upgrade",
+	Usage: "Upgrade abra",
+	Flags: []cli.Flag{RCFlag},
+	Action: func(c *cli.Context) error {
+		cmd := exec.Command("bash", "-c", "curl -s https://install.abra.coopcloud.tech | bash")
+		if RC {
+			cmd = exec.Command("bash", "-c", "curl -s https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer | bash -s -- --rc")
+		}
+		logrus.Debugf("attempting to run '%s'", cmd)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+		return nil
+	},
+}

cmd/abra/main.go

@@ -5,13 +5,14 @@ import (
 	"coopcloud.tech/abra/cli"
 )
 
-// Version is the current version of Abra
+// Version is the current version of abra.
 var Version string
 
-// Commit is the current git commit of Abra
+// Commit is the current commit of abra.
 var Commit string
 
 func main() {
+	// If not set in the ld-flags
 	if Version == "" {
 		Version = "dev"
 	}

go.mod

@@ -5,31 +5,29 @@ go 1.16
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
 	github.com/AlecAivazis/survey/v2 v2.3.2
-	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
+	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.13+incompatible
+	github.com/docker/cli v20.10.11+incompatible
-	github.com/docker/distribution v2.8.1+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.13+incompatible
+	github.com/docker/docker v20.10.11+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/hetznercloud/hcloud-go v1.33.1
-	github.com/moby/sys/signal v0.7.0
+	github.com/moby/sys/signal v0.6.0
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.8.6
+	github.com/schollz/progressbar/v3 v3.8.3
 	github.com/schultz-is/passgen v1.0.1
 	github.com/sirupsen/logrus v1.8.1
-	gotest.tools/v3 v3.1.0
+	github.com/urfave/cli/v2 v2.3.0
+	gotest.tools/v3 v3.0.3
 )
 
 require (
 	coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
-	github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 // indirect
+	github.com/Microsoft/hcsshim v0.8.21 // indirect
-	github.com/buger/goterm v1.0.4
+	github.com/containerd/containerd v1.5.5 // indirect
-	github.com/containerd/containerd v1.5.9 // indirect
-	github.com/containers/image v3.0.2+incompatible
-	github.com/containers/storage v1.38.2 // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
@@ -41,13 +39,10 @@ require (
 	github.com/libdns/gandi v1.0.2
 	github.com/libdns/libdns v0.2.1
 	github.com/moby/sys/mount v0.2.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
-	github.com/spf13/cobra v1.3.0 // indirect
 	github.com/theupdateframework/notary v0.7.0 // indirect
-	github.com/urfave/cli v1.22.5
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	github.com/xanzy/ssh-agent v0.3.1 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
+	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
-	golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
-	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
 )

pkg/app/app.go

@@ -23,7 +23,7 @@ func Get(appName string) (config.App, error) {
 		return config.App{}, err
 	}
 
-	logrus.Debugf("retrieved %s for %s", app, appName)
+	logrus.Debugf("retrieved '%s' for '%s'", app, appName)
 
 	return app, nil
 }
@@ -57,9 +57,9 @@ func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App)
 	deployed := len(services) > 0
 
 	if deployed {
-		logrus.Debugf("detected %s as deployed versions of %s", appSpec, app.Name)
+		logrus.Debugf("detected '%s' as deployed versions of '%s'", appSpec, app.Name)
 	} else {
-		logrus.Debugf("detected %s as not deployed", app.Name)
+		logrus.Debugf("detected '%s' as not deployed", app.Name)
 	}
 
 	return appSpec, len(services) > 0, nil
@@ -71,15 +71,15 @@ func ParseVersionLabel(label string) (string, string) {
 	idx := strings.LastIndex(label, "-")
 	version := label[:idx]
 	digest := label[idx+1:]
-	logrus.Debugf("parsed %s as version from %s", version, label)
+	logrus.Debugf("parsed '%s' as version from '%s'", version, label)
-	logrus.Debugf("parsed %s as digest from %s", digest, label)
+	logrus.Debugf("parsed '%s' as digest from '%s'", digest, label)
 	return version, digest
 }
 
-// ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label.
+// ParseVersionName parses a $STACK_NAME_$SERVICE_NAME service label.
 func ParseServiceName(label string) string {
 	idx := strings.LastIndex(label, "_")
 	serviceName := label[idx+1:]
-	logrus.Debugf("parsed %s as service name from %s", serviceName, label)
+	logrus.Debugf("parsed '%s' as service name from '%s'", serviceName, label)
 	return serviceName
 }

pkg/autocomplete/autocomplete.go

@@ -1,63 +0,0 @@
-package autocomplete
-
-import (
-	"fmt"
-
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// AppNameComplete copletes app names
-func AppNameComplete(c *cli.Context) {
-	appNames, err := config.GetAppNames()
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	if c.NArg() > 0 {
-		return
-	}
-
-	for _, a := range appNames {
-		fmt.Println(a)
-	}
-}
-
-// RecipeNameComplete completes recipe names
-func RecipeNameComplete(c *cli.Context) {
-	catl, err := recipe.ReadRecipeCatalogue()
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	if c.NArg() > 0 {
-		return
-	}
-
-	for name := range catl {
-		fmt.Println(name)
-	}
-}
-
-// SubcommandComplete completes subcommands.
-func SubcommandComplete(c *cli.Context) {
-	if c.NArg() > 0 {
-		return
-	}
-
-	subcmds := []string{
-		"app",
-		"autocomplete",
-		"catalogue",
-		"recipe",
-		"record",
-		"server",
-		"upgrade",
-	}
-
-	for _, cmd := range subcmds {
-		fmt.Println(cmd)
-	}
-}

pkg/catalogue/catalogue.go

@@ -0,0 +1,628 @@
+// Package catalogue provides ways of interacting with recipe catalogues which
+// are JSON data structures which contain meta information about recipes (e.g.
+// what versions of the Nextcloud recipe are available?).
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/web"
+	"github.com/docker/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// RecipeCatalogueURL is the only current recipe catalogue available.
+const RecipeCatalogueURL = "https://apps.coopcloud.tech"
+
+// ReposMetadataURL is the recipe repository metadata
+const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
+
+// image represents a recipe container image.
+type image struct {
+	Image  string `json:"image"`
+	Rating string `json:"rating"`
+	Source string `json:"source"`
+	URL    string `json:"url"`
+}
+
+// features represent what top-level features a recipe supports (e.g. does this
+// recipe support backups?).
+type features struct {
+	Backups     string `json:"backups"`
+	Email       string `json:"email"`
+	Healthcheck string `json:"healthcheck"`
+	Image       image  `json:"image"`
+	Status      int    `json:"status"`
+	Tests       string `json:"tests"`
+	SSO         string `json:"sso"`
+}
+
+// tag represents a git tag.
+type tag = string
+
+// service represents a service within a recipe.
+type service = string
+
+// ServiceMeta represents meta info associated with a service.
+type ServiceMeta struct {
+	Digest string `json:"digest"`
+	Image  string `json:"image"`
+	Tag    string `json:"tag"`
+}
+
+// RecipeVersions are the versions associated with a recipe.
+type RecipeVersions []map[tag]map[service]ServiceMeta
+
+// RecipeMeta represents metadata for a recipe in the abra catalogue.
+type RecipeMeta struct {
+	Category      string         `json:"category"`
+	DefaultBranch string         `json:"default_branch"`
+	Description   string         `json:"description"`
+	Features      features       `json:"features"`
+	Icon          string         `json:"icon"`
+	Name          string         `json:"name"`
+	Repository    string         `json:"repository"`
+	Versions      RecipeVersions `json:"versions"`
+	Website       string         `json:"website"`
+}
+
+// LatestVersion returns the latest version of a recipe.
+func (r RecipeMeta) LatestVersion() string {
+	var version string
+
+	// apps.json versions are sorted so the last key is latest
+	latest := r.Versions[len(r.Versions)-1]
+
+	for tag := range latest {
+		version = tag
+	}
+
+	logrus.Debugf("choosing '%s' as latest version of '%s'", version, r.Name)
+
+	return version
+}
+
+// Name represents a recipe name.
+type Name = string
+
+// RecipeCatalogue represents the entire recipe catalogue.
+type RecipeCatalogue map[Name]RecipeMeta
+
+// Flatten converts AppCatalogue to slice
+func (r RecipeCatalogue) Flatten() []RecipeMeta {
+	recipes := make([]RecipeMeta, 0, len(r))
+
+	for name := range r {
+		recipes = append(recipes, r[name])
+	}
+
+	return recipes
+}
+
+// ByRecipeName sorts recipes by name.
+type ByRecipeName []RecipeMeta
+
+func (r ByRecipeName) Len() int      { return len(r) }
+func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+func (r ByRecipeName) Less(i, j int) bool {
+	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
+}
+
+// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
+// is up to date.
+func recipeCatalogueFSIsLatest() (bool, error) {
+	httpClient := web.NewHTTPRetryClient()
+	res, err := httpClient.Head(RecipeCatalogueURL)
+	if err != nil {
+		return false, err
+	}
+
+	lastModified := res.Header["Last-Modified"][0]
+	parsed, err := time.Parse(time.RFC1123, lastModified)
+	if err != nil {
+		return false, err
+	}
+
+	info, err := os.Stat(config.APPS_JSON)
+	if err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no recipe catalogue found in file system cache")
+			return false, nil
+		}
+		return false, err
+	}
+
+	localModifiedTime := info.ModTime().Unix()
+	remoteModifiedTime := parsed.Unix()
+
+	if localModifiedTime < remoteModifiedTime {
+		logrus.Debug("file system cached recipe catalogue is out-of-date")
+		return false, nil
+	}
+
+	logrus.Debug("file system cached recipe catalogue is up-to-date")
+
+	return true, nil
+}
+
+// ReadRecipeCatalogue reads the recipe catalogue.
+func ReadRecipeCatalogue() (RecipeCatalogue, error) {
+	recipes := make(RecipeCatalogue)
+
+	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
+	if err != nil {
+		return nil, err
+	}
+
+	if !recipeFSIsLatest {
+		logrus.Debugf("reading recipe catalogue from web to get latest")
+		if err := readRecipeCatalogueWeb(&recipes); err != nil {
+			return nil, err
+		}
+		return recipes, nil
+	}
+
+	logrus.Debugf("reading recipe catalogue from file system cache to get latest")
+	if err := readRecipeCatalogueFS(&recipes); err != nil {
+		return nil, err
+	}
+
+	return recipes, nil
+}
+
+// readRecipeCatalogueFS reads the catalogue from the file system.
+func readRecipeCatalogueFS(target interface{}) error {
+	recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
+	if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
+		return err
+	}
+
+	logrus.Debugf("read recipe catalogue from file system cache in '%s'", config.APPS_JSON)
+
+	return nil
+}
+
+// readRecipeCatalogueWeb reads the catalogue from the web.
+func readRecipeCatalogueWeb(target interface{}) error {
+	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
+		return err
+	}
+
+	recipesJSON, err := json.MarshalIndent(target, "", "    ")
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
+		return err
+	}
+
+	logrus.Debugf("read recipe catalogue from web at '%s'", RecipeCatalogueURL)
+
+	return nil
+}
+
+// VersionsOfService lists the version of a service.
+func VersionsOfService(recipe, serviceName string) ([]string, error) {
+	catalogue, err := ReadRecipeCatalogue()
+	if err != nil {
+		return nil, err
+	}
+
+	rec, ok := catalogue[recipe]
+	if !ok {
+		return nil, fmt.Errorf("recipe '%s' does not exist?", recipe)
+	}
+
+	versions := []string{}
+	alreadySeen := make(map[string]bool)
+	for _, serviceVersion := range rec.Versions {
+		for tag := range serviceVersion {
+			if _, ok := alreadySeen[tag]; !ok {
+				alreadySeen[tag] = true
+				versions = append(versions, tag)
+			}
+		}
+	}
+
+	logrus.Debugf("detected versions '%s' for '%s'", strings.Join(versions, ", "), recipe)
+
+	return versions, nil
+}
+
+// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
+func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
+	catl, err := ReadRecipeCatalogue()
+	if err != nil {
+		return RecipeMeta{}, err
+	}
+
+	recipeMeta, ok := catl[recipeName]
+	if !ok {
+		err := fmt.Errorf("recipe '%s' does not exist?", recipeName)
+		return RecipeMeta{}, err
+	}
+
+	if err := recipe.EnsureExists(recipeName); err != nil {
+		return RecipeMeta{}, err
+	}
+
+	logrus.Debugf("recipe metadata retrieved for '%s'", recipeName)
+
+	return recipeMeta, nil
+}
+
+// RepoMeta is a single recipe repo metadata.
+type RepoMeta struct {
+	ID                        int `json:"id"`
+	Owner                     Owner
+	Name                      string      `json:"name"`
+	FullName                  string      `json:"full_name"`
+	Description               string      `json:"description"`
+	Empty                     bool        `json:"empty"`
+	Private                   bool        `json:"private"`
+	Fork                      bool        `json:"fork"`
+	Template                  bool        `json:"template"`
+	Parent                    interface{} `json:"parent"`
+	Mirror                    bool        `json:"mirror"`
+	Size                      int         `json:"size"`
+	HTMLURL                   string      `json:"html_url"`
+	SSHURL                    string      `json:"ssh_url"`
+	CloneURL                  string      `json:"clone_url"`
+	OriginalURL               string      `json:"original_url"`
+	Website                   string      `json:"website"`
+	StarsCount                int         `json:"stars_count"`
+	ForksCount                int         `json:"forks_count"`
+	WatchersCount             int         `json:"watchers_count"`
+	OpenIssuesCount           int         `json:"open_issues_count"`
+	OpenPRCount               int         `json:"open_pr_counter"`
+	ReleaseCounter            int         `json:"release_counter"`
+	DefaultBranch             string      `json:"default_branch"`
+	Archived                  bool        `json:"archived"`
+	CreatedAt                 string      `json:"created_at"`
+	UpdatedAt                 string      `json:"updated_at"`
+	Permissions               Permissions
+	HasIssues                 bool `json:"has_issues"`
+	InternalTracker           InternalTracker
+	HasWiki                   bool   `json:"has_wiki"`
+	HasPullRequests           bool   `json:"has_pull_requests"`
+	HasProjects               bool   `json:"has_projects"`
+	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
+	AllowMergeCommits         bool   `json:"allow_merge_commits"`
+	AllowRebase               bool   `json:"allow_rebase"`
+	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
+	AllowSquashMerge          bool   `json:"allow_squash_merge"`
+	AvatarURL                 string `json:"avatar_url"`
+	Internal                  bool   `json:"internal"`
+	MirrorInterval            string `json:"mirror_interval"`
+}
+
+// Owner is the repo organisation owner metadata.
+type Owner struct {
+	ID         int    `json:"id"`
+	Login      string `json:"login"`
+	FullName   string `json:"full_name"`
+	Email      string `json:"email"`
+	AvatarURL  string `json:"avatar_url"`
+	Language   string `json:"language"`
+	IsAdmin    bool   `json:"is_admin"`
+	LastLogin  string `json:"last_login"`
+	Created    string `json:"created"`
+	Restricted bool   `json:"restricted"`
+	Username   string `json:"username"`
+}
+
+// Permissions is perms metadata for a repo.
+type Permissions struct {
+	Admin bool `json:"admin"`
+	Push  bool `json:"push"`
+	Pull  bool `json:"pull"`
+}
+
+// InternalTracker is issue tracker metadata for a repo.
+type InternalTracker struct {
+	EnableTimeTracker                bool `json:"enable_time_tracker"`
+	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
+	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
+}
+
+// RepoCatalogue represents all the recipe repo metadata.
+type RepoCatalogue map[string]RepoMeta
+
+// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
+func ReadReposMetadata() (RepoCatalogue, error) {
+	reposMeta := make(RepoCatalogue)
+
+	pageIdx := 1
+	for {
+		var reposList []RepoMeta
+
+		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
+
+		logrus.Debugf("fetching repo metadata from '%s'", pagedURL)
+
+		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
+			return reposMeta, err
+		}
+
+		if len(reposList) == 0 {
+			break
+		}
+
+		for idx, repo := range reposList {
+			reposMeta[repo.Name] = reposList[idx]
+		}
+
+		pageIdx++
+	}
+
+	return reposMeta, nil
+}
+
+func GetStringInBetween(str, start, end string) (result string, err error) {
+	// GetStringInBetween returns empty string if no start or end string found
+	s := strings.Index(str, start)
+	if s == -1 {
+		return "", fmt.Errorf("marker string '%s' not found", start)
+	}
+	s += len(start)
+	e := strings.Index(str[s:], end)
+	if e == -1 {
+		return "", fmt.Errorf("end marker '%s' not found", end)
+	}
+	return str[s : s+e], nil
+}
+
+func GetImageMetadata(imageRowString string) (image, error) {
+	img := image{}
+
+	imgFields := strings.Split(imageRowString, ",")
+
+	for i, elem := range imgFields {
+		imgFields[i] = strings.TrimSpace(elem)
+	}
+
+	if len(imgFields) < 3 {
+		logrus.Warnf("image string has incorrect format: %s", imageRowString)
+		return img, nil
+	}
+
+	img.Rating = imgFields[1]
+	img.Source = imgFields[2]
+
+	imgString := imgFields[0]
+
+	imageName, err := GetStringInBetween(imgString, "[", "]")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	img.Image = strings.ReplaceAll(imageName, "`", "")
+
+	imageURL, err := GetStringInBetween(imgString, "(", ")")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	img.URL = imageURL
+
+	return img, nil
+}
+
+func GetRecipeFeaturesAndCategory(recipeName string) (features, string, error) {
+	feat := features{}
+
+	var category string
+
+	readmePath := path.Join(config.ABRA_DIR, "apps", recipeName, "README.md")
+
+	logrus.Debugf("attempting to open '%s'", readmePath)
+
+	readmeFS, err := ioutil.ReadFile(readmePath)
+	if err != nil {
+		return feat, category, err
+	}
+
+	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
+		string(readmeFS),
+		"<!-- metadata -->", "<!-- endmetadata -->",
+	)
+	if err != nil {
+		return feat, category, err
+	}
+
+	readmeLines := strings.Split( // Array item from lines
+		strings.ReplaceAll( // Remove \t tabs
+			readmeMetadata, "\t", "",
+		),
+		"\n")
+
+	for _, val := range readmeLines {
+		if strings.Contains(val, "**Category**") {
+			category = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Category**:"),
+			)
+		}
+		if strings.Contains(val, "**Backups**") {
+			feat.Backups = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Backups**:"),
+			)
+		}
+		if strings.Contains(val, "**Email**") {
+			feat.Email = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Email**:"),
+			)
+		}
+		if strings.Contains(val, "**SSO**") {
+			feat.SSO = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **SSO**:"),
+			)
+		}
+		if strings.Contains(val, "**Healthcheck**") {
+			feat.Healthcheck = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Healthcheck**:"),
+			)
+		}
+		if strings.Contains(val, "**Tests**") {
+			feat.Tests = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Tests**:"),
+			)
+		}
+		if strings.Contains(val, "**Image**") {
+			imageMetadata, err := GetImageMetadata(strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Image**:"),
+			))
+			if err != nil {
+				continue
+			}
+			feat.Image = imageMetadata
+		}
+	}
+
+	return feat, category, nil
+}
+
+// GetRecipeVersions retrieves all recipe versions.
+func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
+	versions := RecipeVersions{}
+
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return versions, err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	gitTags, err := repo.Tags()
+	if err != nil {
+		return versions, err
+	}
+
+	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
+		tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
+
+		logrus.Debugf("processing '%s' for '%s'", tag, recipeName)
+
+		checkOutOpts := &git.CheckoutOptions{
+			Create: false,
+			Keep:   true,
+			Branch: plumbing.ReferenceName(ref.Name()),
+		}
+		if err := worktree.Checkout(checkOutOpts); err != nil {
+			logrus.Debugf("failed to check out '%s' in '%s'", tag, recipeDir)
+			return err
+		}
+
+		logrus.Debugf("successfully checked out '%s' in '%s'", ref.Name(), recipeDir)
+
+		recipe, err := recipe.Get(recipeName)
+		if err != nil {
+			return err
+		}
+
+		versionMeta := make(map[string]ServiceMeta)
+		for _, service := range recipe.Config.Services {
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return err
+			}
+
+			path := reference.Path(img)
+			if strings.Contains(path, "library") {
+				path = strings.Split(path, "/")[1]
+			}
+
+			var tag string
+			switch img.(type) {
+			case reference.NamedTagged:
+				tag = img.(reference.NamedTagged).Tag()
+			case reference.Named:
+				logrus.Warnf("%s service is missing image tag?", path)
+				continue
+			}
+
+			logrus.Debugf("looking up image: '%s' from '%s'", img, path)
+
+			digest, err := client.GetTagDigest(img)
+			if err != nil {
+				logrus.Warn(err)
+				continue
+			}
+
+			versionMeta[service.Name] = ServiceMeta{
+				Digest: digest,
+				Image:  path,
+				Tag:    img.(reference.NamedTagged).Tag(),
+			}
+
+			logrus.Debugf("collecting digest: '%s', image: '%s', tag: '%s'", digest, path, tag)
+		}
+
+		versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
+
+		return nil
+	}); err != nil {
+		return versions, err
+	}
+
+	branch := "master"
+	if _, err := repo.Branch("master"); err != nil {
+		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", recipeDir)
+			logrus.Fatal(err)
+		}
+		branch = "main"
+	}
+
+	refName := fmt.Sprintf("refs/heads/%s", branch)
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Keep:   true,
+		Branch: plumbing.ReferenceName(refName),
+	}
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", branch, recipeDir)
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("switched back to '%s' in '%s'", branch, recipeDir)
+	logrus.Debugf("collected '%s' for '%s'", versions, recipeName)
+
+	return versions, nil
+}
+
+// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
+func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
+	var versions []string
+
+	if recipeMeta, exists := catl[recipeName]; exists {
+		for _, versionMeta := range recipeMeta.Versions {
+			for tag := range versionMeta {
+				versions = append(versions, tag)
+			}
+		}
+	}
+
+	return versions, nil
+}

pkg/client/client.go

@@ -27,11 +27,7 @@ func New(contextName string) (*client.Client, error) {
 			return nil, err
 		}
 
-		helper, err := commandconnPkg.NewConnectionHelper(ctxEndpoint)
+		helper := commandconnPkg.NewConnectionHelper(ctxEndpoint)
-		if err != nil {
-			return nil, err
-		}
-
 		httpClient := &http.Client{
 			// No tls, no proxy
 			Transport: &http.Transport{
@@ -59,7 +55,7 @@ func New(contextName string) (*client.Client, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("created client for %s", contextName)
+	logrus.Debugf("created client for '%s'", contextName)
 
 	return cl, nil
 }

pkg/client/context.go

@@ -26,7 +26,7 @@ func CreateContext(contextName string, user string, port string) error {
 	if err := createContext(contextName, host); err != nil {
 		return err
 	}
-	logrus.Debugf("created the %s context", contextName)
+	logrus.Debugf("created the '%s' context", contextName)
 	return nil
 }
 
@@ -72,6 +72,8 @@ func DeleteContext(name string) error {
 		return err
 	}
 
+	// remove any context that might be loaded
+	// TODO: Check if the context we are removing is the active one rather than doing it all the time
 	cfg := dConfig.LoadDefaultConfigFile(nil)
 	cfg.CurrentContext = ""
 	if err := cfg.Save(); err != nil {

pkg/client/registry.go

@@ -1,57 +1,171 @@
 package client
 
 import (
-	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"strings"
 
-	"github.com/containers/image/docker"
+	"coopcloud.tech/abra/pkg/web"
-	"github.com/containers/image/types"
 	"github.com/docker/distribution/reference"
-	"github.com/docker/docker/client"
+	"github.com/hashicorp/go-retryablehttp"
-	"github.com/sirupsen/logrus"
 )
 
-// GetRegistryTags retrieves all tags of an image from a container registry.
+type RawTag struct {
-func GetRegistryTags(img reference.Named) ([]string, error) {
+	Layer string
-	var tags []string
+	Name  string
+}
 
-	ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
+type RawTags []RawTag
-	if err != nil {
-		return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
-	}
 
-	ctx := context.Background()
+var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
-	tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
+
-	if err != nil {
+func GetRegistryTags(image string) (RawTags, error) {
+	var tags RawTags
+
+	tagsUrl := fmt.Sprintf(registryURL, image)
+	if err := web.ReadJSON(tagsUrl, &tags); err != nil {
 		return tags, err
 	}
 
 	return tags, nil
 }
 
-// GetTagDigest retrieves an image digest from a container registry.
+// getRegv2Token retrieves a registry v2 authentication token.
-func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
+func getRegv2Token(image reference.Named) (string, error) {
-	target := fmt.Sprintf("//%s", reference.Path(image))
+	img := reference.Path(image)
-
+	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
-	ref, err := docker.ParseReference(target)
+	req, err := retryablehttp.NewRequest("GET", authTokenURL, nil)
 	if err != nil {
-		return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
+		return "", err
 	}
 
-	ctx := context.Background()
+	client := web.NewHTTPRetryClient()
-	img, err := ref.NewImage(ctx, nil)
+	res, err := client.Do(req)
 	if err != nil {
-		logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
+		return "", err
-		return "", fmt.Errorf("unable to read digest for %s", image)
 	}
-	defer img.Close()
+	defer res.Body.Close()
 
-	digest := img.ConfigInfo().Digest.String()
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", nil
+	}
+
+	tokenRes := struct {
+		Token  string
+		Expiry string
+		Issued string
+	}{}
+
+	if err := json.Unmarshal(body, &tokenRes); err != nil {
+		return "", err
+	}
+
+	return tokenRes.Token, nil
+}
+
+// GetTagDigest retrieves an image digest from a v2 registry
+func GetTagDigest(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	tag := image.(reference.NamedTagged).Tag()
+	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
+
+	req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := getRegv2Token(image)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header = http.Header{
+		"Accept": []string{
+			"application/vnd.docker.distribution.manifest.v2+json",
+			"application/vnd.docker.distribution.manifest.list.v2+json",
+		},
+		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
+	}
+
+	client := web.NewHTTPRetryClient()
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+
+	registryResT1 := struct {
+		SchemaVersion int
+		MediaType     string
+		Manifests     []struct {
+			MediaType string
+			Size      int
+			Digest    string
+			Platform  struct {
+				Architecture string
+				Os           string
+			}
+		}
+	}{}
+
+	registryResT2 := struct {
+		SchemaVersion int
+		MediaType     string
+		Config        struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+		Layers []struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+	}{}
+
+	if err := json.Unmarshal(body, &registryResT1); err != nil {
+		return "", err
+	}
+
+	var digest string
+	for _, manifest := range registryResT1.Manifests {
+		if string(manifest.Platform.Architecture) == "amd64" {
+			digest = strings.Split(manifest.Digest, ":")[1][:7]
+		}
+	}
 
 	if digest == "" {
-		return digest, fmt.Errorf("unable to read digest for %s", image)
+		if err := json.Unmarshal(body, &registryResT2); err != nil {
+			return "", err
+		}
+		digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
 	}
 
-	return strings.Split(digest, ":")[1][:7], nil
+	if digest == "" {
+		return "", fmt.Errorf("Unable to retrieve amd64 digest for '%s'", image)
+	}
+
+	return digest, nil
 }

pkg/client/volumes.go

@@ -5,9 +5,11 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
 )
 
 func GetVolumes(ctx context.Context, server string, appName string) ([]*types.Volume, error) {
+
 	cl, err := New(server)
 	if err != nil {
 		return nil, err
@@ -19,7 +21,7 @@ func GetVolumes(ctx context.Context, server string, appName string) ([]*types.Vo
 	volumeListOKBody, err := cl.VolumeList(ctx, fs)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
-		return volumeList, err
+		logrus.Fatal(err)
 	}
 
 	return volumeList, nil
@@ -27,11 +29,9 @@ func GetVolumes(ctx context.Context, server string, appName string) ([]*types.Vo
 
 func GetVolumeNames(volumes []*types.Volume) []string {
 	var volumeNames []string
-
 	for _, vol := range volumes {
 		volumeNames = append(volumeNames, vol.Name)
 	}
-
 	return volumeNames
 }
 
@@ -40,13 +40,12 @@ func RemoveVolumes(ctx context.Context, server string, volumeNames []string, for
 	if err != nil {
 		return err
 	}
-
 	for _, volName := range volumeNames {
 		err := cl.VolumeRemove(ctx, volName, force)
 		if err != nil {
 			return err
 		}
 	}
-
 	return nil
+
 }

pkg/compose/compose.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	composetypes "github.com/docker/cli/cli/compose/types"
@@ -17,26 +16,26 @@ import (
 )
 
 // UpdateTag updates an image tag in-place on file system local compose files.
-func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
+func UpdateTag(pattern, image, tag, recipeName string) error {
 	composeFiles, err := filepath.Glob(pattern)
 	if err != nil {
-		return false, err
+		return err
 	}
 
-	logrus.Debugf("considering %s config(s) for tag update", strings.Join(composeFiles, ", "))
+	logrus.Debugf("considering '%s' config(s) for tag update", strings.Join(composeFiles, ", "))
 
 	for _, composeFile := range composeFiles {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
-		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
 		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
-			return false, err
+			return err
 		}
 
 		compose, err := loader.LoadComposefile(opts, sampleEnv)
 		if err != nil {
-			return false, err
+			return err
 		}
 
 		for _, service := range compose.Services {
@@ -46,42 +45,40 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 
 			img, _ := reference.ParseNormalizedNamed(service.Image)
 			if err != nil {
-				return false, err
+				return err
 			}
 
-			var composeTag string
+			composeImage := reference.Path(img)
-			switch img.(type) {
+			if strings.Contains(composeImage, "library") {
-			case reference.NamedTagged:
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
-				composeTag = img.(reference.NamedTagged).Tag()
+				// postgres:<tag>, i.e. images which do not have a username in the
-			default:
+				// first position of the string
-				logrus.Debugf("unable to parse %s, skipping", img)
+				composeImage = strings.Split(composeImage, "/")[1]
-				continue
 			}
+			composeTag := img.(reference.NamedTagged).Tag()
 
-			composeImage := formatter.StripTagMeta(reference.Path(img))
+			logrus.Debugf("parsed '%s' from '%s'", composeTag, service.Image)
-
-			logrus.Debugf("parsed %s from %s", composeTag, service.Image)
 
 			if image == composeImage {
 				bytes, err := ioutil.ReadFile(composeFile)
 				if err != nil {
-					return false, err
+					return err
 				}
 
 				old := fmt.Sprintf("%s:%s", composeImage, composeTag)
 				new := fmt.Sprintf("%s:%s", composeImage, tag)
 				replacedBytes := strings.Replace(string(bytes), old, new, -1)
 
-				logrus.Debugf("updating %s to %s in %s", old, new, compose.Filename)
+				logrus.Debugf("updating '%s' to '%s' in '%s'", old, new, compose.Filename)
 
-				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0764); err != nil {
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
-					return false, err
+					return err
 				}
 			}
 		}
 	}
 
-	return false, nil
+	return nil
 }
 
 // UpdateLabel updates a label in-place on file system local compose files.
@@ -91,12 +88,12 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		return err
 	}
 
-	logrus.Debugf("considering %s config(s) for label update", strings.Join(composeFiles, ", "))
+	logrus.Debugf("considering '%s' config(s) for label update", strings.Join(composeFiles, ", "))
 
 	for _, composeFile := range composeFiles {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
-		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
 		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
 			return err
@@ -133,25 +130,19 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 				old := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", value)
 				replacedBytes := strings.Replace(string(bytes), old, label, -1)
 
-				if old == label {
-					logrus.Warnf("%s is already set, nothing to do?", label)
-					return nil
-				}
-
 				logrus.Debugf("updating %s to %s in %s", old, label, compose.Filename)
 
-				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0764); err != nil {
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
 					return err
 				}
-
-				logrus.Infof("synced label %s to service %s", label, serviceName)
 			}
 		}
 
 		if !discovered {
-			logrus.Warn("no existing label found, automagic insertion not supported yet")
+			logrus.Warn("no existing label found, cannot continue...")
-			logrus.Fatalf("add '- \"%s\"' manually to the 'app' service in %s", label, composeFile)
+			logrus.Fatalf("add '%s' manually, automagic insertion not supported yet", label)
 		}
+
 	}
 
 	return nil

pkg/config/app.go

@@ -1,14 +1,14 @@
 package config
 
 import (
+	"errors"
 	"fmt"
-	"html/template"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
@@ -36,33 +36,25 @@ type AppFiles map[AppName]AppFile
 // App reprents an app with its env file read into memory
 type App struct {
 	Name   AppName
-	Recipe string
+	Type   string
 	Domain string
 	Env    AppEnv
 	Server string
 	Path   string
 }
 
-// StackName gets what the docker safe stack name is for the app. This should
+// StackName gets what the docker safe stack name is for the app
-// not not shown to the user, use a.Name for that. Give the output of this
-// command to Docker only.
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
 	}
-
 	stackName := SanitiseAppName(a.Name)
-
-	if len(stackName) > 45 {
-		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:45])
-		stackName = stackName[:45]
-	}
-
 	a.Env["STACK_NAME"] = stackName
-
 	return stackName
 }
 
+// SORTING TYPES
+
 // ByServer sort a slice of Apps
 type ByServer []App
 
@@ -72,25 +64,25 @@ func (a ByServer) Less(i, j int) bool {
 	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
 }
 
-// ByServerAndRecipe sort a slice of Apps
+// ByServerAndType sort a slice of Apps
-type ByServerAndRecipe []App
+type ByServerAndType []App
 
-func (a ByServerAndRecipe) Len() int      { return len(a) }
+func (a ByServerAndType) Len() int      { return len(a) }
-func (a ByServerAndRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServerAndType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a ByServerAndRecipe) Less(i, j int) bool {
+func (a ByServerAndType) Less(i, j int) bool {
 	if a[i].Server == a[j].Server {
-		return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe)
+		return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
 	}
 	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
 }
 
-// ByRecipe sort a slice of Apps
+// ByType sort a slice of Apps
-type ByRecipe []App
+type ByType []App
 
-func (a ByRecipe) Len() int      { return len(a) }
+func (a ByType) Len() int      { return len(a) }
-func (a ByRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a ByRecipe) Less(i, j int) bool {
+func (a ByType) Less(i, j int) bool {
-	return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe)
+	return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
 }
 
 // ByName sort a slice of Apps
@@ -105,14 +97,14 @@ func (a ByName) Less(i, j int) bool {
 func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	env, err := ReadEnv(appFile.Path)
 	if err != nil {
-		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
+		return App{}, fmt.Errorf("env file for '%s' couldn't be read: %s", name, err.Error())
 	}
 
-	logrus.Debugf("read env %s from %s", env, appFile.Path)
+	logrus.Debugf("read env '%s' from '%s'", env, appFile.Path)
 
 	app, err := newApp(env, name, appFile)
 	if err != nil {
-		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
+		return App{}, fmt.Errorf("env file for '%s' has issues: %s", name, err.Error())
 	}
 
 	return app, nil
@@ -120,20 +112,17 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 
 // newApp creates new App object
 func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+	// Checking for type as it is required - apps wont work without it
 	domain := env["DOMAIN"]
-
+	apptype, ok := env["TYPE"]
-	recipe, exists := env["RECIPE"]
+	if !ok {
-	if !exists {
+		return App{}, errors.New("missing TYPE variable")
-		recipe, exists = env["TYPE"]
-		if !exists {
-			return App{}, fmt.Errorf("%s is missing the RECIPE env var", name)
-		}
 	}
 
 	return App{
 		Name:   name,
 		Domain: domain,
-		Recipe: recipe,
+		Type:   apptype,
 		Env:    env,
 		Server: appFile.Server,
 		Path:   appFile.Path,
@@ -147,24 +136,24 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 		if servers[0] == "" {
 			// Empty servers flag, one string will always be passed
 			var err error
-			servers, err = GetAllFoldersInDirectory(SERVERS_DIR)
+			servers, err = getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
 			if err != nil {
 				return nil, err
 			}
 		}
 	}
 
-	logrus.Debugf("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", "))
+	logrus.Debugf("collecting metadata from '%v' servers: '%s'", len(servers), strings.Join(servers, ", "))
 
 	for _, server := range servers {
-		serverDir := path.Join(SERVERS_DIR, server)
+		serverDir := path.Join(ABRA_SERVER_FOLDER, server)
 		files, err := getAllFilesInDirectory(serverDir)
 		if err != nil {
-			return nil, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server)
+			return nil, err
 		}
 		for _, file := range files {
 			appName := strings.TrimSuffix(file.Name(), ".env")
-			appFilePath := path.Join(SERVERS_DIR, server, file.Name())
+			appFilePath := path.Join(ABRA_SERVER_FOLDER, server, file.Name())
 			appFiles[appName] = AppFile{
 				Path:   appFilePath,
 				Server: server,
@@ -180,7 +169,7 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
 func GetApp(apps AppFiles, name AppName) (App, error) {
 	appFile, exists := apps[name]
 	if !exists {
-		return App{}, fmt.Errorf("cannot find app with name %s", name)
+		return App{}, fmt.Errorf("cannot find app with name '%s'", name)
 	}
 
 	app, err := readAppEnvFile(appFile, name)
@@ -220,13 +209,13 @@ func GetAppServiceNames(appName string) ([]string, error) {
 		return serviceNames, err
 	}
 
-	composeFiles, err := GetAppComposeFiles(app.Recipe, app.Env)
+	composeFiles, err := GetAppComposeFiles(app.Type, app.Env)
 	if err != nil {
 		return serviceNames, err
 	}
 
 	opts := stack.Deploy{Composefiles: composeFiles}
-	compose, err := GetAppComposeConfig(app.Recipe, opts, app.Env)
+	compose, err := GetAppComposeConfig(app.Type, opts, app.Env)
 	if err != nil {
 		return serviceNames, err
 	}
@@ -260,39 +249,27 @@ func GetAppNames() ([]string, error) {
 }
 
 // TemplateAppEnvSample copies the example env file for the app into the users env files
-func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
+func TemplateAppEnvSample(appType, appName, server, domain, recipe string) error {
-	envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample")
+	envSamplePath := path.Join(ABRA_DIR, "apps", appType, ".env.sample")
 	envSample, err := ioutil.ReadFile(envSamplePath)
 	if err != nil {
 		return err
 	}
 
 	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
-	if _, err := os.Stat(appEnvPath); os.IsExist(err) {
+	if _, err := os.Stat(appEnvPath); err == nil {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
-	err = ioutil.WriteFile(appEnvPath, envSample, 0664)
+	envSample = []byte(strings.Replace(string(envSample), fmt.Sprintf("%s.example.com", recipe), domain, -1))
+	envSample = []byte(strings.Replace(string(envSample), "example.com", domain, -1))
+
+	err = ioutil.WriteFile(appEnvPath, envSample, 0755)
 	if err != nil {
 		return err
 	}
 
-	file, err := os.OpenFile(appEnvPath, os.O_RDWR, 0664)
+	logrus.Debugf("copied '%s' to '%s'", envSamplePath, appEnvPath)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	tpl, err := template.ParseFiles(appEnvPath)
-	if err != nil {
-		return err
-	}
-
-	if err := tpl.Execute(file, struct{ Name string }{recipeName}); err != nil {
-		return err
-	}
-
-	logrus.Debugf("copied & templated %s to %s", envSamplePath, appEnvPath)
 
 	return nil
 }
@@ -338,6 +315,9 @@ func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
 			if version, ok := service.Spec.Labels[labelKey]; ok {
 				result["version"] = version
 			} else {
+				//FIXME: we only need to check containers with the version label not
+				//       every single container and then skip when we see no label perf gains
+				//       to be had here
 				continue
 			}
 
@@ -345,7 +325,7 @@ func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
 		}
 	}
 
-	logrus.Debugf("retrieved app statuses: %s", statuses)
+	logrus.Debugf("retrieved app statuses: '%s'", statuses)
 
 	return statuses, nil
 }
@@ -357,20 +337,20 @@ func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
 
 	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
 		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
-		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
+		path := fmt.Sprintf("%s/%s/compose.yml", APPS_DIR, recipe)
 		composeFiles = append(composeFiles, path)
 		return composeFiles, nil
 	}
 
 	composeFileEnvVar := appEnv["COMPOSE_FILE"]
 	envVars := strings.Split(composeFileEnvVar, ":")
-	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
+	logrus.Debugf("COMPOSE_FILE detected ('%s'), loading '%s'", composeFileEnvVar, strings.Join(envVars, ", "))
 	for _, file := range strings.Split(composeFileEnvVar, ":") {
-		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
+		path := fmt.Sprintf("%s/%s/%s", APPS_DIR, recipe, file)
 		composeFiles = append(composeFiles, path)
 	}
 
-	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
+	logrus.Debugf("retrieved '%s' configs for '%s'", strings.Join(composeFiles, ", "), recipe)
 
 	return composeFiles, nil
 }
@@ -384,7 +364,7 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp
 		return &composetypes.Config{}, err
 	}
 
-	logrus.Debugf("retrieved %s for %s", compose.Filename, recipe)
+	logrus.Debugf("retrieved '%s' for '%s'", compose.Filename, recipe)
 
 	return compose, nil
 }

pkg/config/app_test.go

@@ -26,6 +26,7 @@ func TestReadAppEnvFile(t *testing.T) {
 }
 
 func TestGetApp(t *testing.T) {
+	// TODO: Test failures as well as successes
 	app, err := GetApp(expectedAppFiles, appName)
 	if err != nil {
 		t.Fatal(err)

pkg/config/env.go

@@ -15,24 +15,21 @@ import (
 )
 
 var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
-var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
+var ABRA_SERVER_FOLDER = path.Join(ABRA_DIR, "servers")
-var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
+var APPS_JSON = path.Join(ABRA_DIR, "apps.json")
-var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
+var APPS_DIR = path.Join(ABRA_DIR, "apps")
-var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json")
 var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
-var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
-var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
 // GetServers retrieves all servers.
 func GetServers() ([]string, error) {
 	var servers []string
 
-	servers, err := GetAllFoldersInDirectory(SERVERS_DIR)
+	servers, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
 	if err != nil {
 		return servers, err
 	}
 
-	logrus.Debugf("retrieved %v servers: %s", len(servers), servers)
+	logrus.Debugf("retrieved '%v' servers: '%s'", len(servers), servers)
 
 	return servers, nil
 }
@@ -46,20 +43,20 @@ func ReadEnv(filePath string) (AppEnv, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("read %s from %s", envFile, filePath)
+	logrus.Debugf("read '%s' from '%s'", envFile, filePath)
 
 	return envFile, nil
 }
 
 // ReadServerNames retrieves all server names.
 func ReadServerNames() ([]string, error) {
-	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
+	serverNames, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
 
 	if err != nil {
 		return nil, err
 	}
 
-	logrus.Debugf("read %s from %s", strings.Join(serverNames, ","), SERVERS_DIR)
+	logrus.Debugf("read '%s' from '%s'", strings.Join(serverNames, ","), ABRA_SERVER_FOLDER)
 
 	return serverNames, nil
 }
@@ -83,7 +80,7 @@ func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
 
 		realPath, err := filepath.EvalSymlinks(filePath)
 		if err != nil {
-			logrus.Warningf("broken symlink in your abra config folders: %s", filePath)
+			logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
 		} else {
 			realFile, err := os.Stat(realPath)
 			if err != nil {
@@ -98,8 +95,8 @@ func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
 	return realFiles, nil
 }
 
-// GetAllFoldersInDirectory returns both folder and symlink paths
+// getAllFoldersInDirectory returns both folder and symlink paths
-func GetAllFoldersInDirectory(directory string) ([]string, error) {
+func getAllFoldersInDirectory(directory string) ([]string, error) {
 	var folders []string
 
 	files, err := ioutil.ReadDir(directory)
@@ -107,7 +104,7 @@ func GetAllFoldersInDirectory(directory string) ([]string, error) {
 		return nil, err
 	}
 	if len(files) == 0 {
-		return nil, fmt.Errorf("directory is empty: %s", directory)
+		return nil, fmt.Errorf("directory is empty: '%s'", directory)
 	}
 
 	for _, file := range files {
@@ -116,7 +113,7 @@ func GetAllFoldersInDirectory(directory string) ([]string, error) {
 			filePath := path.Join(directory, file.Name())
 			realDir, err := filepath.EvalSymlinks(filePath)
 			if err != nil {
-				logrus.Warningf("broken symlink in your abra config folders: %s", filePath)
+				logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
 			} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
 				// path is a directory
 				folders = append(folders, file.Name())
@@ -127,6 +124,17 @@ func GetAllFoldersInDirectory(directory string) ([]string, error) {
 	return folders, nil
 }
 
+// EnsureAbraDirExists checks for the abra config folder and throws error if not
+func EnsureAbraDirExists() error {
+	if _, err := os.Stat(ABRA_DIR); os.IsNotExist(err) {
+		logrus.Debugf("'%s' does not exist, creating it", ABRA_DIR)
+		if err := os.Mkdir(ABRA_DIR, 0777); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // ReadAbraShEnvVars reads env vars from an abra.sh recipe file.
 func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 	envVars := make(map[string]string)
@@ -153,7 +161,7 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 		}
 	}
 
-	logrus.Debugf("read %s from %s", envVars, abraSh)
+	logrus.Debugf("read '%s' from '%s'", envVars, abraSh)
 
 	return envVars, nil
 }

pkg/config/env_test.go

@@ -20,12 +20,12 @@ var serverName = "evil.corp"
 
 var expectedAppEnv = AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
-	"RECIPE": "ecloud",
+	"TYPE":   "ecloud",
 }
 
 var expectedApp = App{
 	Name:   appName,
-	Recipe: expectedAppEnv["RECIPE"],
+	Type:   expectedAppEnv["TYPE"],
 	Domain: expectedAppEnv["DOMAIN"],
 	Env:    expectedAppEnv,
 	Path:   expectedAppFile.Path,
@@ -44,7 +44,7 @@ var expectedAppFiles = map[string]AppFile{
 // var expectedServerNames = []string{"evil.corp"}
 
 func TestGetAllFoldersInDirectory(t *testing.T) {
-	folders, err := GetAllFoldersInDirectory(testFolder)
+	folders, err := getAllFoldersInDirectory(testFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -74,11 +74,11 @@ func TestReadEnv(t *testing.T) {
 	}
 	if !reflect.DeepEqual(env, expectedAppEnv) {
 		t.Fatalf(
-			"did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s",
+			"did not get expected application settings. Expected: DOMAIN=%s TYPE=%s; Got: DOMAIN=%s TYPE=%s",
 			expectedAppEnv["DOMAIN"],
-			expectedAppEnv["RECIPE"],
+			expectedAppEnv["TYPE"],
 			env["DOMAIN"],
-			env["RECIPE"],
+			env["TYPE"],
 		)
 	}
 }

pkg/container/container.go

@@ -1,70 +0,0 @@
-package container
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-)
-
-// GetContainer retrieves a container. If noInput is false and the retrievd
-// count of containers does not match 1, then a prompt is presented to let the
-// user choose. A count of 0 is handled gracefully.
-func GetContainer(c context.Context, cl *client.Client, filters filters.Args, noInput bool) (types.Container, error) {
-	containerOpts := types.ContainerListOptions{Filters: filters}
-	containers, err := cl.ContainerList(c, containerOpts)
-	if err != nil {
-		return types.Container{}, err
-	}
-
-	if len(containers) == 0 {
-		filter := filters.Get("name")[0]
-		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
-	}
-
-	if len(containers) != 1 {
-		var containersRaw []string
-		for _, container := range containers {
-			containerName := strings.Join(container.Names, " ")
-			trimmed := strings.TrimPrefix(containerName, "/")
-			created := formatter.HumanDuration(container.Created)
-			containersRaw = append(containersRaw, fmt.Sprintf("%s (created %v)", trimmed, created))
-		}
-
-		if noInput {
-			err := fmt.Errorf("expected 1 container but found %v: %s", len(containers), strings.Join(containersRaw, " "))
-			return types.Container{}, err
-		}
-
-		logrus.Warnf("ambiguous container list received, prompting for input")
-
-		var response string
-		prompt := &survey.Select{
-			Message: "which container are you looking for?",
-			Options: containersRaw,
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return types.Container{}, err
-		}
-
-		chosenContainer := strings.TrimSpace(strings.Split(response, " ")[0])
-		for _, container := range containers {
-			containerName := strings.TrimSpace(strings.Join(container.Names, " "))
-			trimmed := strings.TrimPrefix(containerName, "/")
-			if trimmed == chosenContainer {
-				return container, nil
-			}
-		}
-
-		logrus.Panic("failed to match chosen container")
-	}
-
-	return containers[0], nil
-}

pkg/dns/common.go

@@ -47,6 +47,8 @@ func EnsureIPv4(domainName string) (string, error) {
 		},
 	}
 
+	logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
+
 	ctx := context.Background()
 	ips, err := resolver.LookupIPAddr(ctx, domainName)
 	if err != nil {
@@ -58,7 +60,7 @@ func EnsureIPv4(domainName string) (string, error) {
 	}
 
 	ipv4 = ips[0].IP.To4().String()
-	logrus.Debugf("%s points to %s (resolver: %s)", domainName, ipv4, freifunkDNS)
+	logrus.Debugf("discovered the following ipv4 addr: %s", ipv4)
 
 	return ipv4, nil
 }
@@ -92,12 +94,3 @@ func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
 
 	return ipv4, nil
 }
-
-// GetTTL parses a ttl string into a duration
-func GetTTL(ttl string) (time.Duration, error) {
-	val, err := time.ParseDuration(ttl)
-	if err != nil {
-		return val, err
-	}
-	return val, nil
-}

pkg/git/branch.go

@@ -1,35 +0,0 @@
-package git
-
-import (
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-)
-
-// GetCurrentBranch retrieves the current branch of a repository
-func GetCurrentBranch(repository *git.Repository) (string, error) {
-	branchRefs, err := repository.Branches()
-	if err != nil {
-		return "", err
-	}
-
-	headRef, err := repository.Head()
-	if err != nil {
-		return "", err
-	}
-
-	var currentBranchName string
-	err = branchRefs.ForEach(func(branchRef *plumbing.Reference) error {
-		if branchRef.Hash() == headRef.Hash() {
-			currentBranchName = branchRef.Name().String()
-
-			return nil
-		}
-
-		return nil
-	})
-	if err != nil {
-		return "", err
-	}
-
-	return currentBranchName, nil
-}

pkg/git/clone.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
@@ -14,10 +15,10 @@ import (
 // Clone runs a git clone which accounts for different default branches.
 func Clone(dir, url string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		logrus.Debugf("%s does not exist, attempting to git clone from %s", dir, url)
+		logrus.Debugf("'%s' does not exist, attempting to git clone from '%s'", dir, url)
 		_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
 		if err != nil {
-			logrus.Debugf("cloning %s default branch failed, attempting from main branch", url)
+			logrus.Debugf("cloning '%s' default branch failed, attempting from main branch", url)
 			_, err := git.PlainClone(dir, false, &git.CloneOptions{
 				URL:           url,
 				Tags:          git.AllTags,
@@ -31,10 +32,77 @@ func Clone(dir, url string) error {
 				return err
 			}
 		}
-		logrus.Debugf("%s has been git cloned successfully", dir)
+		logrus.Debugf("'%s' has been git cloned successfully", dir)
 	} else {
-		logrus.Debugf("%s already exists", dir)
+		logrus.Debugf("'%s' already exists, doing nothing", dir)
 	}
 
 	return nil
 }
+
+// EnsureUpToDate ensures that a git repo on disk has the latest changes (git-fetch).
+func EnsureUpToDate(dir string) error {
+	repo, err := git.PlainOpen(dir)
+	if err != nil {
+		return err
+	}
+
+	recipeName := filepath.Base(dir)
+	isClean, err := IsClean(recipeName)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
+	}
+
+	branch := "master"
+	if _, err := repo.Branch("master"); err != nil {
+		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", dir)
+			return err
+		}
+		branch = "main"
+	}
+
+	logrus.Debugf("choosing '%s' as main git branch in '%s'", branch, dir)
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	refName := fmt.Sprintf("refs/heads/%s", branch)
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Keep:   true,
+		Branch: plumbing.ReferenceName(refName),
+	}
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", refName, dir)
+		return err
+	}
+
+	logrus.Debugf("successfully checked out '%s' in '%s'", branch, dir)
+
+	remote, err := repo.Remote("origin")
+	if err != nil {
+		return err
+	}
+
+	fetchOpts := &git.FetchOptions{
+		RemoteName: "origin",
+		RefSpecs:   []config.RefSpec{"refs/heads/*:refs/remotes/origin/*"},
+		Force:      true,
+	}
+	if err := remote.Fetch(fetchOpts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return err
+		}
+	}
+
+	logrus.Debugf("successfully fetched all changes in '%s'", dir)
+
+	return nil
+}

pkg/git/commit.go

@@ -1,56 +0,0 @@
-package git
-
-import (
-	"fmt"
-
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-)
-
-// Commit runs a git commit
-func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
-	if commitMessage == "" {
-		return fmt.Errorf("no commit message specified?")
-	}
-
-	commitRepo, err := git.PlainOpen(repoPath)
-	if err != nil {
-		return err
-	}
-
-	commitWorktree, err := commitRepo.Worktree()
-	if err != nil {
-		return err
-	}
-
-	patterns, err := GetExcludesFiles()
-	if err != nil {
-		return err
-	}
-
-	if len(patterns) > 0 {
-		commitWorktree.Excludes = append(patterns, commitWorktree.Excludes...)
-	}
-
-	if !dryRun {
-		err = commitWorktree.AddGlob(glob)
-		if err != nil {
-			return err
-		}
-		logrus.Debugf("staged %s for commit", glob)
-	} else {
-		logrus.Debugf("dry run: did not stage %s for commit", glob)
-	}
-
-	if !dryRun {
-		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
-		if err != nil {
-			return err
-		}
-		logrus.Debug("git changes commited")
-	} else {
-		logrus.Debug("dry run: no changes commited")
-	}
-
-	return nil
-}

pkg/git/init.go

@@ -24,7 +24,7 @@ func Init(repoPath string, commit bool) error {
 			logrus.Fatal(err)
 		}
 
-		if err := commitWorktree.AddWithOptions(&git.AddOptions{All: true}); err != nil {
+		if err := commitWorktree.AddGlob("**"); err != nil {
 			return err
 		}
 

pkg/git/push.go

@@ -1,43 +0,0 @@
-package git
-
-import (
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/config"
-	"github.com/sirupsen/logrus"
-)
-
-// Push pushes the latest changes & optionally tags to the default remote
-func Push(repoDir string, remote string, tags bool, dryRun bool) error {
-	if dryRun {
-		logrus.Debugf("dry run: no git changes pushed in %s", repoDir)
-		return nil
-	}
-
-	commitRepo, err := git.PlainOpen(repoDir)
-	if err != nil {
-		return err
-	}
-
-	opts := &git.PushOptions{}
-	if remote != "" {
-		opts.RemoteName = remote
-	}
-
-	if err := commitRepo.Push(opts); err != nil {
-		return err
-	}
-
-	logrus.Debugf("git changes pushed")
-
-	if tags {
-		opts.RefSpecs = append(opts.RefSpecs, config.RefSpec("+refs/tags/*:refs/tags/*"))
-
-		if err := commitRepo.Push(opts); err != nil {
-			return err
-		}
-
-		logrus.Debugf("git tags pushed")
-	}
-
-	return nil
-}

pkg/git/read.go

@@ -2,7 +2,6 @@ package git
 
 import (
 	"io/ioutil"
-	"os"
 	"os/user"
 	"path"
 	"path/filepath"
@@ -18,7 +17,7 @@ import (
 
 // GetRecipeHead retrieves latest HEAD metadata.
 func GetRecipeHead(recipeName string) (*plumbing.Reference, error) {
-	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
 
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
@@ -34,8 +33,10 @@ func GetRecipeHead(recipeName string) (*plumbing.Reference, error) {
 }
 
 // IsClean checks if a repo has unstaged changes
-func IsClean(repoPath string) (bool, error) {
+func IsClean(recipeName string) (bool, error) {
-	repo, err := git.PlainOpen(repoPath)
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return false, err
 	}
@@ -49,10 +50,7 @@ func IsClean(repoPath string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-
-	if len(patterns) > 0 {
 	worktree.Excludes = append(patterns, worktree.Excludes...)
-	}
 
 	status, err := worktree.Status()
 	if err != nil {
@@ -60,15 +58,15 @@ func IsClean(repoPath string) (bool, error) {
 	}
 
 	if status.String() != "" {
-		logrus.Debugf("discovered git status in %s: %s", repoPath, status.String())
+		logrus.Debugf("discovered git status for %s repository: %s", recipeName, status.String())
 	} else {
-		logrus.Debugf("discovered clean git status in %s", repoPath)
+		logrus.Debugf("discovered clean git status for %s repository", recipeName)
 	}
 
 	return status.IsClean(), nil
 }
 
-// GetExcludesFiles reads the exlude files from a global gitignore
+// GetExcludesFiles reads the exlude files from a global git ignore
 func GetExcludesFiles() ([]gitignore.Pattern, error) {
 	var err error
 	var patterns []gitignore.Pattern
@@ -95,16 +93,7 @@ func parseGitConfig() (*gitConfigPkg.Config, error) {
 		return nil, err
 	}
 
-	globalGitConfig := filepath.Join(usr.HomeDir, ".gitconfig")
+	b, err := ioutil.ReadFile(usr.HomeDir + "/.gitconfig")
-	if _, err := os.Stat(globalGitConfig); err != nil {
-		if os.IsNotExist(err) {
-			logrus.Debugf("no %s exists, not reading any global gitignore config", globalGitConfig)
-			return cfg, nil
-		}
-		return cfg, err
-	}
-
-	b, err := ioutil.ReadFile(globalGitConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -126,41 +115,27 @@ func getExcludesFile(cfg *gitConfigPkg.Config) string {
 			}
 		}
 	}
-
+	return ""
-	return "~/.gitignore"
 }
 
 func parseExcludesFile(excludesfile string) ([]gitignore.Pattern, error) {
-	var ps []gitignore.Pattern
-
 	excludesfile, err := expandTilde(excludesfile)
 	if err != nil {
 		return nil, err
 	}
 
-	if _, err := os.Stat(excludesfile); err != nil {
-		if os.IsNotExist(err) {
-			logrus.Debugf("no %s exists, skipping reading gitignore paths", excludesfile)
-			return ps, nil
-		}
-		return ps, err
-	}
-
 	data, err := ioutil.ReadFile(excludesfile)
 	if err != nil {
 		return nil, err
 	}
 
-	var pathsRaw []string
+	var ps []gitignore.Pattern
 	for _, s := range strings.Split(string(data), "\n") {
 		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
-			pathsRaw = append(pathsRaw, s)
 			ps = append(ps, gitignore.ParsePattern(s, nil))
 		}
 	}
 
-	logrus.Debugf("read global ignore paths: %s", strings.Join(pathsRaw, " "))
-
 	return ps, nil
 }
 
@@ -168,13 +143,11 @@ func expandTilde(path string) (string, error) {
 	if !strings.HasPrefix(path, "~") {
 		return path, nil
 	}
-
 	var paths []string
 	u, err := user.Current()
 	if err != nil {
 		return "", err
 	}
-
 	for _, p := range strings.Split(path, string(filepath.Separator)) {
 		if p == "~" {
 			paths = append(paths, u.HomeDir)
@@ -182,6 +155,5 @@ func expandTilde(path string) (string, error) {
 			paths = append(paths, p)
 		}
 	}
-
+	return "/" + filepath.Join(paths...), nil
-	return filepath.Join(paths...), nil
 }

pkg/git/remote.go

@@ -1,28 +0,0 @@
-package git
-
-import (
-	"strings"
-
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/config"
-	"github.com/sirupsen/logrus"
-)
-
-// CreateRemote creates a new git remote in a repository
-func CreateRemote(repo *git.Repository, name, url string, dryRun bool) error {
-	if dryRun {
-		logrus.Debugf("dry run: remote %s (%s) not created", name, url)
-		return nil
-	}
-
-	if _, err := repo.CreateRemote(&config.RemoteConfig{
-		Name: name,
-		URLs: []string{url},
-	}); err != nil {
-		if !strings.Contains(err.Error(), "remote already exists") {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/integration/integration.go

@@ -1,12 +0,0 @@
-package integration
-
-import (
-	"os"
-	"testing"
-)
-
-func skipIfNotIntegration(t *testing.T) {
-	if os.Getenv("ABRA_INTEGRATION") == "" {
-		t.Skip("missing 'ABRA_INTEGRATION', not running integration tests")
-	}
-}

pkg/lint/recipe.go

@@ -1,338 +0,0 @@
-package lint
-
-import (
-	"fmt"
-	"net/http"
-	"os"
-	"path"
-
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/tagcmp"
-	"github.com/docker/distribution/reference"
-	"github.com/sirupsen/logrus"
-)
-
-var Warn = "warn"
-var Critical = "critical"
-
-type LintFunction func(recipe.Recipe) (bool, error)
-
-type LintRule struct {
-	Ref          string
-	Level        string
-	Description  string
-	HowToResolve string
-	Function     LintFunction
-}
-
-var LintRules = map[string][]LintRule{
-	"warn": {
-		{
-			Ref:          "R001",
-			Level:        "warn",
-			Description:  "compose config has expected version",
-			HowToResolve: "ensure 'version: \"3.8\"' in compose configs",
-			Function:     LintComposeVersion,
-		},
-		{
-			Ref:          "R002",
-			Level:        "warn",
-			Description:  "healthcheck enabled for all services",
-			HowToResolve: "wire up healthchecks",
-			Function:     LintHealthchecks,
-		},
-		{
-			Ref:          "R003",
-			Level:        "warn",
-			Description:  "all images use a tag",
-			HowToResolve: "use a tag for all images",
-			Function:     LintAllImagesTagged,
-		},
-		{
-			Ref:          "R004",
-			Level:        "warn",
-			Description:  "no unstable tags",
-			HowToResolve: "tag all images with stable tags",
-			Function:     LintNoUnstableTags,
-		},
-		{
-			Ref:          "R005",
-			Level:        "warn",
-			Description:  "tags use semver-like format",
-			HowToResolve: "use semver-like tags",
-			Function:     LintSemverLikeTags,
-		},
-		{
-			Ref:          "R006",
-			Level:        "warn",
-			Description:  "has published catalogue version",
-			HowToResolve: "publish a recipe version to the catalogue",
-			Function:     LintHasPublishedVersion,
-		},
-		{
-			Ref:          "R007",
-			Level:        "warn",
-			Description:  "README.md metadata filled in",
-			HowToResolve: "fill out all the metadata",
-			Function:     LintMetadataFilledIn,
-		},
-		{
-			Ref:          "R013",
-			Level:        "warn",
-			Description:  "git.coopcloud.tech repo exists",
-			HowToResolve: "upload your recipe to git.coopcloud.tech/coop-cloud/...",
-			Function:     LintHasRecipeRepo,
-		},
-	},
-	"error": {
-		{
-			Ref:          "R008",
-			Level:        "error",
-			Description:  ".env.sample provided",
-			HowToResolve: "create an example .env.sample",
-			Function:     LintEnvConfigPresent,
-		},
-		{
-			Ref:          "R009",
-			Level:        "error",
-			Description:  "one service named 'app'",
-			HowToResolve: "name a servce 'app'",
-			Function:     LintAppService,
-		},
-		{
-			Ref:          "R010",
-			Level:        "error",
-			Description:  "traefik routing enabled",
-			HowToResolve: "include \"traefik.enable=true\" deploy label",
-			Function:     LintTraefikEnabled,
-		},
-		{
-			Ref:          "R011",
-			Level:        "error",
-			Description:  "all services have images",
-			HowToResolve: "ensure \"image: ...\" set on all services",
-			Function:     LintImagePresent,
-		},
-		{
-			Ref:          "R012",
-			Level:        "error",
-			Description:  "config version are vendored",
-			HowToResolve: "vendor config versions in an abra.sh",
-			Function:     LintAbraShVendors,
-		},
-	},
-}
-
-func LintForErrors(recipe recipe.Recipe) error {
-	logrus.Debugf("linting for critical errors in %s configs", recipe.Name)
-
-	for level := range LintRules {
-		if level != "error" {
-			continue
-		}
-		for _, rule := range LintRules[level] {
-			ok, err := rule.Function(recipe)
-			if err != nil {
-				return err
-			}
-			if !ok {
-				return fmt.Errorf("lint error in %s configs: \"%s\" failed lint checks (%s)", recipe.Name, rule.Description, rule.Ref)
-			}
-		}
-	}
-
-	logrus.Debugf("linting successful, %s is well configured", recipe.Name)
-
-	return nil
-}
-
-func LintComposeVersion(recipe recipe.Recipe) (bool, error) {
-	if recipe.Config.Version == "3.8" {
-		return true, nil
-	}
-
-	return true, nil
-}
-
-func LintEnvConfigPresent(recipe recipe.Recipe) (bool, error) {
-	envSample := fmt.Sprintf("%s/%s/.env.sample", config.RECIPES_DIR, recipe.Name)
-	if _, err := os.Stat(envSample); !os.IsNotExist(err) {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func LintAppService(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		if service.Name == "app" {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
-func LintTraefikEnabled(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		for label := range service.Deploy.Labels {
-			if label == "traefik.enable" {
-				if service.Deploy.Labels[label] == "true" {
-					return true, nil
-				}
-			}
-		}
-	}
-
-	return false, nil
-}
-
-func LintHealthchecks(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		if service.HealthCheck == nil {
-			return false, nil
-		}
-	}
-
-	return true, nil
-}
-
-func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		img, err := reference.ParseNormalizedNamed(service.Image)
-		if err != nil {
-			return false, err
-		}
-		if reference.IsNameOnly(img) {
-			return false, nil
-		}
-	}
-
-	return true, nil
-}
-
-func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		img, err := reference.ParseNormalizedNamed(service.Image)
-		if err != nil {
-			return false, err
-		}
-
-		var tag string
-		switch img.(type) {
-		case reference.NamedTagged:
-			tag = img.(reference.NamedTagged).Tag()
-		case reference.Named:
-			return false, nil
-		}
-
-		if tag == "latest" {
-			return false, nil
-		}
-	}
-
-	return true, nil
-}
-
-func LintSemverLikeTags(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		img, err := reference.ParseNormalizedNamed(service.Image)
-		if err != nil {
-			return false, err
-		}
-
-		var tag string
-		switch img.(type) {
-		case reference.NamedTagged:
-			tag = img.(reference.NamedTagged).Tag()
-		case reference.Named:
-			return false, nil
-		}
-
-		if !tagcmp.IsParsable(tag) {
-			return false, nil
-		}
-	}
-
-	return true, nil
-}
-
-func LintImagePresent(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		if service.Image == "" {
-			return false, nil
-		}
-	}
-	return true, nil
-}
-
-func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
-	catl, err := recipePkg.ReadRecipeCatalogue()
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	versions, err := recipePkg.GetRecipeCatalogueVersions(recipe.Name, catl)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if len(versions) == 0 {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-func LintMetadataFilledIn(r recipe.Recipe) (bool, error) {
-	features, category, err := recipe.GetRecipeFeaturesAndCategory(r.Name)
-	if err != nil {
-		return false, err
-	}
-
-	if category == "" {
-		return false, nil
-	}
-
-	if features.Backups == "" ||
-		features.Email == "" ||
-		features.Healthcheck == "" ||
-		features.Image.Image == "" ||
-		features.SSO == "" {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-func LintAbraShVendors(recipe recipe.Recipe) (bool, error) {
-	for _, service := range recipe.Config.Services {
-		if len(service.Configs) > 0 {
-			abraSh := path.Join(config.RECIPES_DIR, recipe.Name, "abra.sh")
-			if _, err := os.Stat(abraSh); err != nil {
-				if os.IsNotExist(err) {
-					return false, err
-				}
-				return false, err
-			}
-		}
-	}
-	return true, nil
-}
-
-func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
-	url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe.Name)
-
-	res, err := http.Get(url)
-	if err != nil {
-		return false, err
-	}
-
-	if res.StatusCode != 200 {
-		return false, err
-	}
-
-	return true, nil
-}

pkg/secret/pass.go

@@ -19,13 +19,13 @@ func PassInsertSecret(secretValue, secretName, appName, server string) error {
 		secretValue, server, appName, secretName,
 	)
 
-	logrus.Debugf("attempting to run %s", cmd)
+	logrus.Debugf("attempting to run '%s'", cmd)
 
 	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
 		return err
 	}
 
-	logrus.Infof("%s inserted into pass store", secretName)
+	logrus.Infof("'%s' inserted into pass store", secretName)
 
 	return nil
 }
@@ -41,13 +41,13 @@ func PassRmSecret(secretName, appName, server string) error {
 		server, appName, secretName,
 	)
 
-	logrus.Debugf("attempting to run %s", cmd)
+	logrus.Debugf("attempting to run '%s'", cmd)
 
 	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
 		return err
 	}
 
-	logrus.Infof("%s removed from pass store", secretName)
+	logrus.Infof("'%s' removed from pass store", secretName)
 
 	return nil
 }

pkg/secret/secret.go

@@ -8,7 +8,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -35,7 +34,7 @@ func GeneratePasswords(count, length uint) ([]string, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("generated %s", strings.Join(passwords, ", "))
+	logrus.Debugf("generated '%s'", strings.Join(passwords, ", "))
 
 	return passwords, nil
 }
@@ -54,7 +53,7 @@ func GeneratePassphrases(count uint) ([]string, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("generated %s", strings.Join(passphrases, ", "))
+	logrus.Debugf("generated '%s'", strings.Join(passphrases, ", "))
 
 	return passphrases, nil
 }
@@ -70,32 +69,35 @@ func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
 		}
 	}
 
-	logrus.Debugf("read %s as secrets from %s", secretEnvVars, appEnv)
+	logrus.Debugf("read '%s' as secrets from '%s'", secretEnvVars, appEnv)
 
 	return secretEnvVars
 }
 
+// TODO: should probably go in the config/app package?
 func ParseSecretEnvVarName(secretEnvVar string) string {
 	withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_")
 	withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION")
 	name := strings.ToLower(withoutSuffix)
-	logrus.Debugf("parsed %s as name from %s", name, secretEnvVar)
+	logrus.Debugf("parsed '%s' as name from '%s'", name, secretEnvVar)
 	return name
 }
 
+// TODO: should probably go in the config/app package?
 func ParseGeneratedSecretName(secret string, appEnv config.App) string {
 	name := fmt.Sprintf("%s_", appEnv.StackName())
 	withoutAppName := strings.TrimPrefix(secret, name)
 	idx := strings.LastIndex(withoutAppName, "_")
 	parsed := withoutAppName[:idx]
-	logrus.Debugf("parsed %s as name from %s", parsed, secret)
+	logrus.Debugf("parsed '%s' as name from '%s'", parsed, secret)
 	return parsed
 }
 
+// TODO: should probably go in the config/app package?
 func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 	values := strings.Split(secret, "#")
 	if len(values) == 0 {
-		return secretValue{}, fmt.Errorf("unable to parse %s", secret)
+		return secretValue{}, fmt.Errorf("unable to parse '%s'", secret)
 	}
 
 	if len(values) == 1 {
@@ -111,7 +113,7 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 	}
 	version := strings.ReplaceAll(values[0], " ", "")
 
-	logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
+	logrus.Debugf("parsed version '%s' and length '%v' from '%s'", version, length, secret)
 
 	return secretValue{Version: version, Length: length}, nil
 }
@@ -120,44 +122,27 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
 func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
 	secrets := make(map[string]string)
 
-	var mutex sync.Mutex
-	var wg sync.WaitGroup
 	ch := make(chan error, len(secretEnvVars))
 	for secretEnvVar := range secretEnvVars {
-		wg.Add(1)
-
 		go func(s string) {
-			defer wg.Done()
-
 			secretName := ParseSecretEnvVarName(s)
 			secretValue, err := ParseSecretEnvVarValue(secretEnvVars[s])
 			if err != nil {
 				ch <- err
 				return
 			}
-
 			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version)
-			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
+			logrus.Debugf("attempting to generate and store '%s' on '%s'", secretRemoteName, server)
-
 			if secretValue.Length > 0 {
 				passwords, err := GeneratePasswords(1, uint(secretValue.Length))
 				if err != nil {
 					ch <- err
 					return
 				}
-
 				if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
-					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
-						ch <- nil
-					} else {
 					ch <- err
-					}
 					return
 				}
-
-				mutex.Lock()
-				defer mutex.Unlock()
 				secrets[secretName] = passwords[0]
 			} else {
 				passphrases, err := GeneratePassphrases(1)
@@ -165,27 +150,15 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 					ch <- err
 					return
 				}
-
 				if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
-					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
-						ch <- nil
-					} else {
 					ch <- err
 				}
-					return
-				}
-
-				mutex.Lock()
-				defer mutex.Unlock()
 				secrets[secretName] = passphrases[0]
 			}
 			ch <- nil
 		}(secretEnvVar)
 	}
 
-	wg.Wait()
-
 	for range secretEnvVars {
 		err := <-ch
 		if err != nil {
@@ -193,7 +166,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
 		}
 	}
 
-	logrus.Debugf("generated and stored %s on %s", secrets, server)
+	logrus.Debugf("generated and stored '%s' on '%s'", secrets, server)
 
 	return secrets, nil
 }

pkg/server/server.go

@@ -12,7 +12,7 @@ import (
 func CreateServerDir(serverName string) error {
 	serverPath := path.Join(config.ABRA_DIR, "servers", serverName)
 
-	if err := os.Mkdir(serverPath, 0764); err != nil {
+	if err := os.Mkdir(serverPath, 0755); err != nil {
 		if !os.IsExist(err) {
 			return err
 		}

pkg/service/service.go

@@ -1,78 +0,0 @@
-package service
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-)
-
-// GetService retrieves a service container. If prompt is true and the retrievd
-// count of service containers does not match 1, then a prompt is presented to
-// let the user choose. A count of 0 is handled gracefully.
-func GetService(c context.Context, cl *client.Client, filters filters.Args, prompt bool) (swarm.Service, error) {
-	serviceOpts := types.ServiceListOptions{Filters: filters}
-	services, err := cl.ServiceList(c, serviceOpts)
-	if err != nil {
-		return swarm.Service{}, err
-	}
-
-	if len(services) == 0 {
-		filter := filters.Get("name")[0]
-		return swarm.Service{}, fmt.Errorf("no services matching the %v filter found?", filter)
-	}
-
-	if len(services) != 1 {
-		var servicesRaw []string
-		for _, service := range services {
-			serviceName := service.Spec.Name
-			created := formatter.HumanDuration(service.CreatedAt.Unix())
-			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
-		}
-
-		if !prompt {
-			err := fmt.Errorf("expected 1 service but found %v: %s", len(services), strings.Join(servicesRaw, " "))
-			return swarm.Service{}, err
-		}
-
-		logrus.Warnf("ambiguous service list received, prompting for input")
-
-		var response string
-		prompt := &survey.Select{
-			Message: "which service are you looking for?",
-			Options: servicesRaw,
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return swarm.Service{}, err
-		}
-
-		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
-		for _, service := range services {
-			serviceName := strings.ToLower(service.Spec.Name)
-			if serviceName == chosenService {
-				return service, nil
-			}
-		}
-
-		logrus.Panic("failed to match chosen service")
-	}
-
-	return services[0], nil
-}
-
-// ContainerToServiceName converts a container name to a service name.
-func ContainerToServiceName(containerNames []string, stackName string) string {
-	containerName := strings.Join(containerNames, "")
-	trimmed := strings.TrimPrefix(containerName, "/")
-	stackNameServiceName := strings.Split(trimmed, ".")[0]
-	splitter := fmt.Sprintf("%s_", stackName)
-	return strings.Split(stackNameServiceName, splitter)[1]
-}

pkg/ssh/ssh.go

@@ -111,7 +111,7 @@ type sudoWriter struct {
 
 // Write satisfies the write interface for sudoWriter
 func (w *sudoWriter) Write(p []byte) (int, error) {
-	if strings.Contains(string(p), "sudo_password") {
+	if string(p) == "sudo_password" {
 		w.stdin.Write([]byte(w.pw + "\n"))
 		w.pw = ""
 		return len(p), nil
@@ -131,9 +131,11 @@ func RunSudoCmd(cmd, passwd string, cl *Client) error {
 	}
 	defer session.Close()
 
-	sudoCmd := fmt.Sprintf("SSH_ASKPASS=/usr/bin/ssh-askpass; sudo -p sudo_password -S %s", cmd)
+	cmd = "sudo -p " + "sudo_password" + " -S " + cmd
 
-	w := &sudoWriter{pw: passwd}
+	w := &sudoWriter{
+		pw: passwd,
+	}
 	w.stdin, err = session.StdinPipe()
 	if err != nil {
 		return err
@@ -142,19 +144,79 @@ func RunSudoCmd(cmd, passwd string, cl *Client) error {
 	session.Stdout = w
 	session.Stderr = w
 
-	modes := ssh.TerminalModes{
+	done := make(chan struct{})
-		ssh.ECHO:          0,
+	scanner := bufio.NewScanner(session.Stdin)
-		ssh.TTY_OP_ISPEED: 14400,
+
-		ssh.TTY_OP_OSPEED: 14400,
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			fmt.Println(line)
+		}
+		done <- struct{}{}
+	}()
+
+	if err := session.Start(cmd); err != nil {
+		return err
 	}
 
-	err = session.RequestPty("xterm", 80, 40, modes)
+	<-done
+
+	if err := session.Wait(); err != nil {
+		return err
+	}
+
+	return err
+}
+
+// Exec runs a command on a remote and streams output
+func Exec(cmd string, cl *Client) error {
+	session, err := cl.SSHClient.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	stdout, err := session.StdoutPipe()
 	if err != nil {
 		return err
 	}
 
-	if err := session.Run(sudoCmd); err != nil {
+	stderr, err := session.StdoutPipe()
-		return fmt.Errorf("%s", string(w.b.Bytes()))
+	if err != nil {
+		return err
+	}
+
+	stdoutDone := make(chan struct{})
+	stdoutScanner := bufio.NewScanner(stdout)
+
+	go func() {
+		for stdoutScanner.Scan() {
+			line := stdoutScanner.Text()
+			fmt.Println(line)
+		}
+		stdoutDone <- struct{}{}
+	}()
+
+	stderrDone := make(chan struct{})
+	stderrScanner := bufio.NewScanner(stderr)
+
+	go func() {
+		for stderrScanner.Scan() {
+			line := stderrScanner.Text()
+			fmt.Println(line)
+		}
+		stderrDone <- struct{}{}
+	}()
+
+	if err := session.Start(cmd); err != nil {
+		return err
+	}
+
+	<-stdoutDone
+	<-stderrDone
+
+	if err := session.Wait(); err != nil {
+		return err
 	}
 
 	return nil
@@ -258,7 +320,7 @@ func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.Publ
 
 	if exists {
 		hostname := strings.Split(hostnameAndPort, ":")[0]
-		logrus.Debugf("server SSH host key found for %s", hostname)
+		logrus.Debugf("server SSH host key found for %s, moving on", hostname)
 		return nil
 	}
 
@@ -325,7 +387,7 @@ func connect(username, host, port string, authMethod ssh.AuthMethod, timeout tim
 	conn, err = net.DialTimeout("tcp", hostnameAndPort, timeout)
 	if err != nil {
 		logrus.Debugf("tcp dialing %s failed, trying via ~/.ssh/config", hostnameAndPort)
-		hostConfig, err := GetHostConfig(host, username, port, true)
+		hostConfig, err := GetHostConfig(host, username, port)
 		if err != nil {
 			return nil, err
 		}
@@ -452,7 +514,7 @@ func GetContextConnDetails(serverName string) (*dockerSSHPkg.Spec, error) {
 		}
 	}
 
-	hostConfig, err := GetHostConfig(serverName, "", "", false)
+	hostConfig, err := GetHostConfig(serverName, "", "")
 	if err != nil {
 		return &dockerSSHPkg.Spec{}, err
 	}
@@ -472,51 +534,45 @@ func GetContextConnDetails(serverName string) (*dockerSSHPkg.Spec, error) {
 }
 
 // GetHostConfig retrieves a ~/.ssh/config config for a host.
-func GetHostConfig(hostname, username, port string, override bool) (HostConfig, error) {
+func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 	var hostConfig HostConfig
 
-	if hostname == "" || override {
+	var host, idf string
-		if sshHost := ssh_config.Get(hostname, "Hostname"); sshHost != "" {
+
-			hostname = sshHost
+	if host = ssh_config.Get(hostname, "Hostname"); host == "" {
-		}
+		logrus.Debugf("no hostname found in SSH config, assuming %s", hostname)
+		host = hostname
 	}
 
-	if username == "" || override {
+	if username == "" {
-		if sshUser := ssh_config.Get(hostname, "User"); sshUser != "" {
+		if username = ssh_config.Get(hostname, "User"); username == "" {
-			username = sshUser
-		} else {
 			systemUser, err := user.Current()
 			if err != nil {
 				return hostConfig, err
 			}
+			logrus.Debugf("no username found in SSH config or passed on command-line, assuming %s", username)
 			username = systemUser.Username
 		}
 	}
 
-	if port == "" || override {
+	if port == "" {
-		if sshPort := ssh_config.Get(hostname, "Port"); sshPort != "" {
+		if port = ssh_config.Get(hostname, "Port"); port == "" {
-			// skip override probably correct port with dummy default value from
+			logrus.Debugf("no port found in SSH config or passed on command-line, assuming 22")
-			// ssh_config which is 22. only when the original port number is empty
+			port = "22"
-			// should we try this default. this might not cover all cases
-			// unfortunately.
-			if port != "" && sshPort != "22" {
-				port = sshPort
-			}
 		}
 	}
 
-	if idf := ssh_config.Get(hostname, "IdentityFile"); idf != "" && idf != "~/.ssh/identity" {
+	idf = ssh_config.Get(hostname, "IdentityFile")
+	if idf != "" {
 		var err error
 		idf, err = identityFileAbsPath(idf)
 		if err != nil {
 			return hostConfig, err
 		}
 		hostConfig.IdentityFile = idf
-	} else {
-		hostConfig.IdentityFile = ""
 	}
 
-	hostConfig.Host = hostname
+	hostConfig.Host = host
 	hostConfig.Port = port
 	hostConfig.User = username
 

pkg/upstream/commandconn/commandconn.go

@@ -188,14 +188,14 @@ func ignorableCloseError(err error) bool {
 func (c *commandConn) CloseRead() error {
 	// NOTE: maybe already closed here
 	if err := c.stdout.Close(); err != nil && !ignorableCloseError(err) {
-		// muted because https://github.com/docker/compose/issues/8544
+		// TODO: muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseRead: %v", err)
 	}
 	c.stdioClosedMu.Lock()
 	c.stdoutClosed = true
 	c.stdioClosedMu.Unlock()
 	if err := c.killIfStdioClosed(); err != nil {
-		// muted because https://github.com/docker/compose/issues/8544
+		// TODO: muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseRead: %v", err)
 	}
 	return nil
@@ -212,14 +212,14 @@ func (c *commandConn) Read(p []byte) (int, error) {
 func (c *commandConn) CloseWrite() error {
 	// NOTE: maybe already closed here
 	if err := c.stdin.Close(); err != nil && !ignorableCloseError(err) {
-		// muted because https://github.com/docker/compose/issues/8544
+		// TODO: muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseWrite: %v", err)
 	}
 	c.stdioClosedMu.Lock()
 	c.stdinClosed = true
 	c.stdioClosedMu.Unlock()
 	if err := c.killIfStdioClosed(); err != nil {
-		// muted because https://github.com/docker/compose/issues/8544
+		// TODO: muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.CloseWrite: %v", err)
 	}
 	return nil
@@ -239,7 +239,7 @@ func (c *commandConn) Close() error {
 		logrus.Warnf("commandConn.Close: CloseRead: %v", err)
 	}
 	if err = c.CloseWrite(); err != nil {
-		// muted because https://github.com/docker/compose/issues/8544
+		// TODO: muted because https://github.com/docker/compose/issues/8544
 		// logrus.Warnf("commandConn.Close: CloseWrite: %v", err)
 	}
 	return err

pkg/upstream/commandconn/connection.go

@@ -44,7 +44,6 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 			ctxConnDetails.Host,
 			ctxConnDetails.User,
 			ctxConnDetails.Port,
-			false,
 		)
 		if err != nil {
 			return nil, err
@@ -67,13 +66,13 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 	return nil, err
 }
 
-func NewConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
+func NewConnectionHelper(daemonURL string) *connhelper.ConnectionHelper {
 	helper, err := GetConnectionHelper(daemonURL)
 	if err != nil {
-		return nil, err
+		logrus.Fatal(err)
 	}
 
-	return helper, nil
+	return helper
 }
 
 func getDockerEndpoint(host string) (docker.Endpoint, error) {

pkg/upstream/container/hijack.go

@@ -16,6 +16,7 @@ import (
 )
 
 // The default escape key sequence: ctrl-p, ctrl-q
+// TODO: This could be moved to `pkg/term`.
 var defaultEscapeKeys = []byte{16, 17}
 
 // A hijackedIOStreamer handles copying input to and output from streams to the

pkg/upstream/convert/service.go

@@ -399,6 +399,7 @@ func convertServiceNetworks(
 	return nets, nil
 }
 
+// TODO: fix secrets API so that SecretAPIClient is not required here
 func convertServiceSecrets(
 	client client.SecretAPIClient,
 	namespace Namespace,
@@ -420,12 +421,6 @@ func convertServiceSecrets(
 			return nil, err
 		}
 
-		// NOTE(d1): strip # length=... modifiers
-		if strings.Contains(obj.Name, "#") {
-			vals := strings.Split(obj.Name, "#")
-			obj.Name = strings.TrimSpace(vals[0])
-		}
-
 		file := swarm.SecretReferenceFileTarget(obj.File)
 		refs = append(refs, &swarm.SecretReference{
 			File:       &file,
@@ -447,6 +442,8 @@ func convertServiceSecrets(
 // required by the serivce. Unlike convertServiceSecrets, this takes the whole
 // ServiceConfig, because some Configs may be needed as a result of other
 // fields (like CredentialSpecs).
+//
+// TODO: fix configs API so that ConfigsAPIClient is not required here
 func convertServiceConfigObjs(
 	client client.ConfigAPIClient,
 	namespace Namespace,
@@ -629,6 +626,7 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 }
 
 func convertRestartPolicy(restart string, source *composetypes.RestartPolicy) (*swarm.RestartPolicy, error) {
+	// TODO: log if restart is being ignored
 	if source == nil {
 		policy, err := opts.ParseRestartPolicy(restart)
 		if err != nil {

pkg/upstream/service/scale.go

@@ -1,44 +0,0 @@
-package upstream
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-)
-
-// RunServiceScale scales a service (useful for restart action)
-func RunServiceScale(ctx context.Context, cl *client.Client, serviceID string, scale uint64) error {
-	service, _, err := cl.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
-	if err != nil {
-		return err
-	}
-
-	serviceMode := &service.Spec.Mode
-	if serviceMode.Replicated != nil {
-		serviceMode.Replicated.Replicas = &scale
-	} else if serviceMode.ReplicatedJob != nil {
-		serviceMode.ReplicatedJob.TotalCompletions = &scale
-	} else {
-		return fmt.Errorf("scale can only be used with replicated or replicated-job mode")
-	}
-
-	response, err := cl.ServiceUpdate(
-		ctx,
-		service.ID,
-		service.Version,
-		service.Spec,
-		types.ServiceUpdateOptions{},
-	)
-	if err != nil {
-		return err
-	}
-
-	for _, warning := range response.Warnings {
-		logrus.Warn(warning)
-	}
-
-	return nil
-}