.dockerignore

@@ -1,4 +0,0 @@
-Dockerfile
-.dockerignore
-*.swp
-*.swo

.drone.yml

@@ -3,23 +3,45 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.20
+    image: golang:1.17
     commands:
       - make check
 
+  - name: make static
+    image: golang:1.17
+    ignore: true # until we decide we all want this check
+    environment:
+      STATIC_CHECK_URL: honnef.co/go/tools/cmd/staticcheck
+      STATIC_CHECK_VERSION: v0.2.0
+    commands:
+      - go install $STATIC_CHECK_URL@$STATIC_CHECK_VERSION
+      - make static
+
   - name: make build
-    image: golang:1.20
+    image: golang:1.17
     commands:
       - make build
-    depends_on:
-      - make check
 
   - name: make test
-    image: golang:1.20
+    image: golang:1.17
     commands:
       - make test
+
+  - name: notify on failure
+    image: plugins/matrix
+    settings:
+      homeserver: https://matrix.autonomic.zone
+      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
+      userid: "@autono-bot:autonomic.zone"
+      accesstoken:
+        from_secret: autono_bot_access_token
     depends_on:
       - make check
+      - make build
+      - make test
+    when:
+      status:
+        - failure
 
   - name: fetch
     image: docker:git
@@ -33,7 +55,7 @@ steps:
       event: tag
 
   - name: release
-    image: golang:1.20
+    image: golang:1.17
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token
@@ -47,23 +69,6 @@ steps:
     when:
       event: tag
 
-  - name: publish image
-    image: plugins/docker
-    settings:
-      auto_tag: true
-      username: 3wordchant
-      password:
-        from_secret: git_coopcloud_tech_token_3wc
-      repo: git.coopcloud.tech/coop-cloud/abra
-      tags: dev
-      registry: git.coopcloud.tech
-    when:
-      event:
-        exclude:
-          - pull_request
-    depends_on:
-      - make check
-
 volumes:
   - name: deps
     temp: {}

.e2e.env.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

.gitignore

@@ -1,9 +1,6 @@
-*fmtcoverage.html
-.e2e.env
-.envrc
-.vscode/
-/kadabra
 abra
-dist/
+.vscode/
-tests/integration/.abra/catalogue
 vendor/
+.envrc
+dist/
+*fmtcoverage.html

.goreleaser.yml

@@ -1,76 +1,40 @@
 ---
+project_name: abra
 gitea_urls:
   api: https://git.coopcloud.tech/api/v1
   download: https://git.coopcloud.tech/
   skip_tls_verify: false
-
 before:
   hooks:
     - go mod tidy
-
+    - go generate ./...
 builds:
-  - id: abra
+  - env:
-    binary: abra
+      - CGO_ENABLED=0
     dir: cmd/abra
-    env:
-      - CGO_ENABLED=0
     goos:
       - linux
       - darwin
-    goarch:
-      - 386
-      - amd64
-      - arm
-      - arm64
-    goarm:
-      - 5
-      - 6
-      - 7
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
-
-  - id: kadabra
-    binary: kadabra
-    dir: cmd/kadabra
-    env:
-      - CGO_ENABLED=0
-    goos:
-      - linux
-      - darwin
-    goarch:
-      - 386
-      - amd64
-      - arm
-      - arm64
-    goarm:
-      - 5
-      - 6
-      - 7
-    ldflags:
-      - "-X 'main.Commit={{ .Commit }}'"
-      - "-X 'main.Version={{ .Version }}'"
-
 archives:
   - replacements:
       386: i386
       amd64: x86_64
     format: binary
-
 checksum:
   name_template: "checksums.txt"
-
 snapshot:
   name_template: "{{ incpatch .Version }}-next"
-
 changelog:
   sort: desc
   filters:
     exclude:
-      - "^Merge"
-      - "^Revert"
       - "^WIP:"
-      - "^chore(deps):"
+      - "^chore:"
+      - "^docs:"
+      - "^refactor:"
       - "^style:"
       - "^test:"
       - "^tests:"

AUTHORS.md

@@ -1,13 +0,0 @@
-# authors
-
-> If you're looking at this and you hack on `abra` and you're not listed here,
-> please do add yourself! This is a community project, let's show some :heart:
-
-- 3wordchant
-- cassowary
-- decentral1se
-- frando
-- kawaiipunk
-- knoflook
-- moritz
-- roxxers

Dockerfile

@@ -1,17 +0,0 @@
-FROM golang:1.20-alpine AS build
-
-ENV GOPRIVATE coopcloud.tech
-
-RUN apk add --no-cache make git gcc musl-dev
-
-COPY . /app
-
-WORKDIR /app
-
-RUN CGO_ENABLED=0 make build
-
-FROM scratch
-
-COPY --from=build /app/abra /abra
-
-ENTRYPOINT ["/abra"]

LICENSE

@@ -1,15 +0,0 @@
-Abra: The Co-op Cloud utility belt
-Copyright (C) 2022 Co-op Cloud <helo@coopcloud.tech>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.

Makefile

@@ -1,13 +1,11 @@
-ABRA         := ./cmd/abra
+ABRA    := ./cmd/abra
-KADABRA      := ./cmd/kadabra
+COMMIT  := $(shell git rev-list -1 HEAD)
-COMMIT       := $(shell git rev-list -1 HEAD)
+GOPATH  := $(shell go env GOPATH)
-GOPATH       := $(shell go env GOPATH)
+LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
-LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
-
 export GOPRIVATE=coopcloud.tech
 
-all: format check build test
+all: run test install build clean format check static
 
 run:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
@@ -16,32 +14,25 @@ install:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
 build-dev:
-	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
+	@go build -ldflags=$(LDFLAGS) $(ABRA)
 
 build:
-	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
-	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
 clean:
 	@rm '$(GOPATH)/bin/abra'
-	@rm '$(GOPATH)/bin/kadabra'
 
 format:
 	@gofmt -s -w .
 
 check:
-	@test -z $$(gofmt -l .) || \
+	@test -z $$(gofmt -l .) || (echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
-	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+
+static:
+	@staticcheck $(ABRA)
 
 test:
-	@go test ./... -cover -v
+	@go test ./... -cover
 
 loc:
 	@find . -name "*.go" | xargs wc -l
-
-loc-author:
-	@git ls-files -z | \
-		xargs -0rn 1 -P "$$(nproc)" -I{} sh -c 'git blame -w -M -C -C --line-porcelain -- {} | grep -I --line-buffered "^author "' | \
-   	sort -f | \
-   	uniq -ic | \
-   	sort -n

README.md

@@ -1,12 +1,111 @@
-# `abra`
+# abra
+
+> https://coopcloud.tech
 
 [![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
 [![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
 
 The Co-op Cloud utility belt 🎩🐇
 
-<a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
+`abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create applications, deploy them, run backup and restore operations and a whole lot of other things. It is the go-to tool for day-to-day operations when managing a Co-op Cloud instance.
 
-`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
+## Install
 
-Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!
+### Arch-based Linux Distros
+
+[abra (coming-soon)](https://aur.archlinux.org/packages/abra/) or for the latest version on git [abra-git](https://aur.archlinux.org/packages/abra-git/)
+
+```sh
+yay -S abra-git # or abra
+```
+
+### Debian-based Linux Distros
+
+**Coming Soon**
+
+### Homebrew
+
+**Coming Soon**
+
+### Build from source
+
+```sh
+git clone https://git.coopcloud.tech/coop-cloud/abra
+cd abra
+go env -w GOPRIVATE=coopcloud.tech
+make install
+```
+
+The abra binary will be in `$GOPATH/bin`.
+
+## Autocompletion
+
+**bash**
+
+Copy `scripts/autocomplete/bash` into `/etc/bash_completion.d/` and rename
+it to abra.
+
+```
+sudo cp scripts/autocomplete/bash /etc/bash_completion.d/abra
+source /etc/bash_completion.d/abra
+```
+
+In development, you can source the script in your git checkout, just make sure
+to set `PROG=abra`, otherwise it'll add completion to the wrong command:
+
+```
+PROG=abra source /path/to/abra/scripts/autocomplete/bash
+```
+
+**(fi)zsh**
+
+(fi)zsh doesn't have an autocompletion folder by default but you can create one, then copy `scripts/autocomplete/zsh` into it and add a couple lines to your `~/.zshrc` or `~/.fizsh/.fizshrc`
+
+```
+sudo mkdir /etc/zsh/completion.d/
+sudo cp scripts/autocomplete/zsh /etc/zsh/completion.d/abra
+echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
+```
+
+(replace .zshrc with ~/.fizsh/.fizshrc if you use fizsh)
+
+## Hacking
+
+Install direnv, run `cp .envrc.sample .envrc`, then run `direnv allow` in this directory. This will set coopcloud repos as private due to [this bug.](https://git.coopcloud.tech/coop-cloud/coopcloud.tech/issues/20#issuecomment-8201). Or you can run `go env -w GOPRIVATE=coopcloud.tech` but I'm not sure how persistent this is.
+
+Install [Go >= 1.16](https://golang.org/doc/install) and then:
+
+- `make build` to build
+- `./abra` to run commands
+- `make test` will run tests
+
+Our [Drone CI configuration](.drone.yml) runs a number of sanity on each pushed commit. See the [Makefile](./Makefile) for more handy targets.
+
+Please use the [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/) for your commits so we can automate our change log.
+
+## Versioning
+
+We use [goreleaser](https://goreleaser.com) to help us automate releases. We use [semver](https://semver.org) for versioning all releases of the tool. While we are still in the public alpha release phase, we will maintain a `0.y.z-alpha` format. Change logs are generated from our commit logs. We are still working this out and aim to refine our release praxis as we go.
+
+For developers, while using this `-alpha` format, the `y` part is the "major" version part. So, if you make breaking changes, you increment that and _not_ the `x` part. So, if you're on `0.1.0-alpha`, then you'd go to `0.1.1-alpha` for a backwards compatible change and `0.2.0-alpha` for a backwards incompatible change.
+
+## Making a new release
+
+- Change `ABRA_VERSION` to match the new tag in [`scripts`](./scripts/installer/installer) (use [semver](https://semver.org))
+- Commit that change (e.g. `git commit -m 'chore: publish next tag 0.3.1-alpha'`)
+- Make a new tag (e.g. `git tag 0.y.z-alpha`)
+- Push the new tag (e.g. `git push && git push --tags`)
+- Wait until the build finishes on [build.coopcloud.tech](https://build.coopcloud.tech/coop-cloud/abra)
+- Deploy the new installer script (e.g. `cd ./scripts/installer && make`)
+- Check the release worked, (e.g. `abra upgrade; abra version`)
+
+## Fork maintenance
+
+We maintain a fork of [godotenv](https://github.com/Autonomic-Cooperative/godotenv) for two features:
+
+1. multi-line env var support
+2. inline comment parsing
+
+You can upgrade the version here by running `go get github.com/Autonomic-Cooperative/godotenv@<commit>` where `<commit>` is the
+latest commit you want to pin to. We are aiming to migrate to YAML format for the environment configuration, so this should only
+be a temporary thing.

cli/app/app.go

@@ -1,37 +1,38 @@
 package app
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var AppCommand = cli.Command{
+// AppCommand defines the `abra app` command and ets subcommands
-	Name:        "app",
+var AppCommand = &cli.Command{
-	Aliases:     []string{"a"},
+	Name:      "app",
-	Usage:       "Manage apps",
+	Usage:     "Manage deployed apps",
-	ArgsUsage:   "<domain>",
+	Aliases:   []string{"a"},
-	Description: "Functionality for managing the life cycle of your apps",
+	ArgsUsage: "<app>",
-	Subcommands: []cli.Command{
+	Description: `
-		appBackupCommand,
+This command provides all the functionality you need to manage the life cycle
-		appCheckCommand,
+of your apps. From initial deployment, day-2 operations (e.g. backup/restore)
-		appCmdCommand,
+to scaling apps up and spinning them down.
-		appConfigCommand,
+`,
-		appCpCommand,
+	Subcommands: []*cli.Command{
-		appDeployCommand,
-		appErrorsCommand,
-		appListCommand,
-		appLogsCommand,
 		appNewCommand,
-		appPsCommand,
+		appConfigCommand,
-		appRemoveCommand,
+		appDeployCommand,
-		appRestartCommand,
-		appRestoreCommand,
-		appRollbackCommand,
-		appRunCommand,
-		appSecretCommand,
-		appServicesCommand,
-		appUndeployCommand,
 		appUpgradeCommand,
-		appVersionCommand,
+		appUndeployCommand,
+		appBackupCommand,
+		appRestoreCommand,
+		appRemoveCommand,
+		appCheckCommand,
+		appListCommand,
+		appPsCommand,
+		appLogsCommand,
+		appCpCommand,
+		appRunCommand,
+		appRollbackCommand,
+		appSecretCommand,
 		appVolumeCommand,
+		appVersionCommand,
 	},
 }

cli/app/backup.go

@@ -1,399 +1,87 @@
 package app
 
 import (
-	"archive/tar"
+	"errors"
-	"context"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os"
-	"path/filepath"
+	"os/exec"
+	"path"
 	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/system"
-	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-type backupConfig struct {
+var backupAllServices bool
-	preHookCmd  string
+var backupAllServicesFlag = &cli.BoolFlag{
-	postHookCmd string
+	Name:        "all",
-	backupPaths []string
+	Value:       false,
+	Destination: &backupAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Backup all services",
 }
 
-var appBackupCommand = cli.Command{
+var appBackupCommand = &cli.Command{
 	Name:      "backup",
-	Aliases:   []string{"bk"},
+	Usage:     "Backup an app",
-	Usage:     "Run app backup",
+	Aliases:   []string{"b"},
-	ArgsUsage: "<domain> [<service>]",
+	Flags:     []cli.Flag{backupAllServicesFlag},
-	Flags: []cli.Flag{
+	ArgsUsage: "<service>",
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app backup.
-
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		conf := runtime.New()
 
-		cl, err := client.New(app.Server)
+		if c.Args().Get(1) != "" && backupAllServices {
-		if err != nil {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<service>' and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
 			logrus.Fatal(err)
 		}
 
-		recipe, err := recipe.Get(app.Recipe, conf)
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+
+		execCmd := "abra_backup"
+		if !backupAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_backup_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
+		if !strings.Contains(string(bytes), execCmd) {
-		backupConfigs := make(map[string]backupConfig)
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
 		}
 
-		serviceName := c.Args().Get(1)
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
-		if serviceName != "" {
+		cmd := exec.Command("bash", "-c", sourceAndExec)
-			backupConfig, ok := backupConfigs[serviceName]
+		if err := internal.RunCmd(cmd); err != nil {
-			if !ok {
+			logrus.Fatal(err)
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
-
-			logrus.Infof("running backup for the %s service", serviceName)
-
-			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if len(backupConfigs) == 0 {
-				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
-			}
-
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
-
-				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
 		}
 
 		return nil
 	},
-}
+	BashComplete: func(c *cli.Context) {
-
+		appNames, err := config.GetAppNames()
-// TimeStamp generates a file name friendly timestamp.
-func TimeStamp() string {
-	ts := time.Now().UTC().Format(time.RFC3339)
-	return strings.Replace(ts, ":", "-", -1)
-}
-
-// runBackup does the actual backup logic.
-func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+			logrus.Warn(err)
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
 		}
-		defer content.Close()
+		if c.NArg() > 0 {
-
+			return
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
 		}
-
+		for _, a := range appNames {
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
+			fmt.Println(a)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
 		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
+	},
-	}
-
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
-
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
-
-	return nil
-}
-
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
-
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
-		}
-
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
-
-	return nil
-}
-
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
-
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
-		}
-	}
-
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
-		}
-	}
-
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
-
-	return nil
-}
-
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
-
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
-
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
-			}
-			break
-		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
-		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
-}
-
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
-
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
-
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
-
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
-
-	rc = fin
-	tr = tar.NewReader(rc)
-
-	return tr, rc, nil
 }

cli/app/check.go

@@ -1,33 +1,29 @@
 package app
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appCheckCommand = cli.Command{
+var appCheckCommand = &cli.Command{
 	Name:      "check",
-	Aliases:   []string{"chk"},
 	Usage:     "Check if app is configured correctly",
-	ArgsUsage: "<domain>",
+	Aliases:   []string{"c"},
-	Flags: []cli.Flag{
+	ArgsUsage: "<service>",
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", app.Type, ".env.sample")
 		if _, err := os.Stat(envSamplePath); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
+				logrus.Fatalf("'%s' does not exist?", envSamplePath)
 			}
 			logrus.Fatal(err)
 		}
@@ -49,9 +45,20 @@ var appCheckCommand = cli.Command{
 			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
 		}
 
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+		logrus.Infof("all necessary environment variables defined for '%s'", app.Name)
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/cmd.go

@@ -1,245 +0,0 @@
-package app
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var appCmdCommand = cli.Command{
-	Name:    "command",
-	Aliases: []string{"cmd"},
-	Usage:   "Run app commands",
-	Description: `
-Run an app specific command.
-
-These commands are bash functions, defined in the abra.sh of the recipe itself.
-They can be run within the context of a service (e.g. app) or locally on your
-work station by passing "--local". Arguments can be passed into these functions
-using the "-- <args>" syntax.
-
-Example:
-
-  abra app cmd example.com app create_user -- me@example.com
-`,
-	ArgsUsage: "<domain> [<service>] <command> [-- <args>]",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.LocalCmdFlag,
-		internal.RemoteUserFlag,
-	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if internal.LocalCmd && internal.RemoteUser != "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
-		}
-
-		hasCmdArgs, parsedCmdArgs := parseCmdArgs(c.Args(), internal.LocalCmd)
-
-		abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
-		if _, err := os.Stat(abraSh); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist for %s?", abraSh, app.Name)
-			}
-			logrus.Fatal(err)
-		}
-
-		if internal.LocalCmd {
-			cmdName := c.Args().Get(1)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
-				logrus.Fatal(err)
-			}
-
-			logrus.Debugf("--local detected, running %s on local work station", cmdName)
-
-			var exportEnv string
-			for k, v := range app.Env {
-				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
-			}
-			var sourceAndExec string
-			if hasCmdArgs {
-				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName, parsedCmdArgs)
-			} else {
-				logrus.Debug("did not detect any command arguments")
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName)
-			}
-
-			shell := "/bin/bash"
-			if _, err := os.Stat(shell); errors.Is(err, os.ErrNotExist) {
-				logrus.Debugf("%s does not exist locally, use /bin/sh as fallback", shell)
-				shell = "/bin/sh"
-			}
-			cmd := exec.Command(shell, "-c", sourceAndExec)
-
-			if err := internal.RunCmd(cmd); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			targetServiceName := c.Args().Get(1)
-
-			cmdName := c.Args().Get(2)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
-				logrus.Fatal(err)
-			}
-
-			serviceNames, err := config.GetAppServiceNames(app.Name)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			matchingServiceName := false
-			for _, serviceName := range serviceNames {
-				if serviceName == targetServiceName {
-					matchingServiceName = true
-				}
-			}
-
-			if !matchingServiceName {
-				logrus.Fatalf("no service %s for %s?", targetServiceName, app.Name)
-			}
-
-			logrus.Debugf("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName)
-
-			if hasCmdArgs {
-				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-			} else {
-				logrus.Debug("did not detect any command arguments")
-			}
-
-			if err := runCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		return nil
-	},
-}
-
-func parseCmdArgs(args []string, isLocal bool) (bool, string) {
-	var (
-		parsedCmdArgs string
-		hasCmdArgs    bool
-	)
-
-	if isLocal {
-		if len(args) > 2 {
-			return true, fmt.Sprintf("%s ", strings.Join(args[2:], " "))
-		}
-	} else {
-		if len(args) > 3 {
-			return true, fmt.Sprintf("%s ", strings.Join(args[3:], " "))
-		}
-	}
-
-	return hasCmdArgs, parsedCmdArgs
-}
-
-func ensureCommand(abraSh, recipeName, execCmd string) error {
-	bytes, err := ioutil.ReadFile(abraSh)
-	if err != nil {
-		return err
-	}
-
-	if !strings.Contains(string(bytes), execCmd) {
-		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
-	}
-
-	return nil
-}
-
-func runCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
-
-	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-	content, err := archive.TarWithOptions(abraSh, toTarOpts)
-	if err != nil {
-		return err
-	}
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
-		return err
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	shell := "/bin/bash"
-	findShell := []string{"test", "-e", shell}
-	execCreateOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          findShell,
-		Detach:       false,
-		Tty:          false,
-	}
-
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
-		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
-		shell = "/bin/sh"
-	}
-
-	var cmd []string
-	if cmdArgs != "" {
-		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.Name, app.StackName(), cmdName, cmdArgs)}
-	} else {
-		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.Name, app.StackName(), cmdName)}
-	}
-
-	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
-
-	if internal.RemoteUser != "" {
-		logrus.Debugf("running command with user %s", internal.RemoteUser)
-		execCreateOpts.User = internal.RemoteUser
-	}
-
-	execCreateOpts.Cmd = cmd
-	execCreateOpts.Tty = true
-
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
-		return err
-	}
-
-	return nil
-}

cli/app/cmd_test.go

@@ -1,31 +0,0 @@
-package app
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestParseCmdArgs(t *testing.T) {
-	tests := []struct {
-		input          []string
-		shouldParse    bool
-		expectedOutput string
-	}{
-		// `--` is not parsed when passed in from the command-line e.g. -- foo bar baz
-		// so we need to eumlate that as missing when testing if bash args are passed in
-		// see https://git.coopcloud.tech/coop-cloud/organising/issues/336 for more
-		{[]string{"foo.com", "app", "test"}, false, ""},
-		{[]string{"foo.com", "app", "test", "foo"}, true, "foo "},
-		{[]string{"foo.com", "app", "test", "foo", "bar", "baz"}, true, "foo bar baz "},
-	}
-
-	for _, test := range tests {
-		ok, parsed := parseCmdArgs(test.input, false)
-		if ok != test.shouldParse {
-			t.Fatalf("[%s] should not parse", strings.Join(test.input, " "))
-		}
-		if parsed != test.expectedOutput {
-			t.Fatalf("%s does not match %s", parsed, test.expectedOutput)
-		}
-	}
-}

cli/app/config.go

@@ -1,43 +1,23 @@
 package app
 
 import (
-	"errors"
+	"fmt"
 	"os"
 	"os/exec"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appConfigCommand = cli.Command{
+var appConfigCommand = &cli.Command{
-	Name:      "config",
+	Name:    "config",
-	Aliases:   []string{"cfg"},
+	Aliases: []string{"c"},
-	Usage:     "Edit app config",
+	Usage:   "Edit app config",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		appName := c.Args().First()
+		app := internal.ValidateApp(c)
-
-		if appName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no app provided"))
-		}
-
-		files, err := config.LoadAppFiles("")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		appFile, exists := files[appName]
-		if !exists {
-			logrus.Fatalf("cannot find app with name %s", appName)
-		}
 
 		ed, ok := os.LookupEnv("EDITOR")
 		if !ok {
@@ -50,7 +30,7 @@ var appConfigCommand = cli.Command{
 			}
 		}
 
-		cmd := exec.Command(ed, appFile.Path)
+		cmd := exec.Command(ed, app.Path)
 		cmd.Stdin = os.Stdin
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
@@ -60,5 +40,16 @@ var appConfigCommand = cli.Command{
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/cp.go

@@ -1,54 +1,29 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"strings"
 
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appCpCommand = cli.Command{
+var appCpCommand = &cli.Command{
 	Name:      "cp",
 	Aliases:   []string{"c"},
-	ArgsUsage: "<domain> <src> <dst>",
+	ArgsUsage: "<src> <dst>",
-	Flags: []cli.Flag{
+	Usage:     "Copy files to/from a running app service",
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Copy files to/from a running app service",
-	Description: `
-Copy files to and from any app service file system.
-
-If you want to copy a myfile.txt to the root of the app service:
-
-    abra app cp <domain> myfile.txt app:/
-
-And if you want to copy that file back to your current working directory locally:
-
-		abra app cp <domain> app:/myfile.txt .
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		src := c.Args().Get(1)
 		dst := c.Args().Get(2)
 		if src == "" {
@@ -89,65 +64,61 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Debugf("assuming transfer is going TO the container")
 		}
 
-		if !isToContainer {
+		appFiles, err := config.LoadAppFiles("")
-			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
+		if err != nil {
-				logrus.Fatalf("%s does not exist locally?", dstPath)
+			logrus.Fatal(err)
+		}
+
+		appEnv, err := config.GetApp(appFiles, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
+		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(containers) != 1 {
+			logrus.Fatalf("expected 1 container but got %v", len(containers))
+		}
+		container := containers[0]
+
+		logrus.Debugf("retrieved '%s' as target container on '%s'", formatter.ShortenID(container.ID), app.Server)
+
+		if isToContainer {
+			if _, err := os.Stat(srcPath); err != nil {
+				logrus.Fatalf("'%s' does not exist?", srcPath)
+			}
+
+			toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+			content, err := archive.TarWithOptions(srcPath, toTarOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+			if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer content.Close()
+			fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+			if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+				logrus.Fatal(err)
 			}
 		}
-
-		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
-			logrus.Fatal(err)
-		}
-
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
-}
-
-func configureAndCp(
-	c *cli.Context,
-	cl *dockerClient.Client,
-	app config.App,
-	srcPath string,
-	dstPath string,
-	service string,
-	isToContainer bool) error {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
-
-	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
-
-	if isToContainer {
-		if _, err := os.Stat(srcPath); err != nil {
-			logrus.Fatalf("%s does not exist?", srcPath)
-		}
-
-		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		content, err := archive.TarWithOptions(srcPath, toTarOpts)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	} else {
-		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		defer content.Close()
-		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	return nil
 }

cli/app/deploy.go

@@ -1,28 +1,33 @@
 package app
 
 import (
+	"fmt"
+	"strings"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/client"
+	stack "coopcloud.tech/abra/pkg/client/stack"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
 )
 
-var appDeployCommand = cli.Command{
+var appDeployCommand = &cli.Command{
-	Name:      "deploy",
+	Name:    "deploy",
-	Aliases:   []string{"d"},
+	Aliases: []string{"d"},
-	Usage:     "Deploy an app",
+	Usage:   "Deploy an app",
-	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
-Deploy an app. It does not support incrementing the version of a deployed app,
+This command deploys a new instance of an app. It does not support changing the
-for this you need to look at the "abra app upgrade <domain>" command.
+version of an existing deployed app, for this you need to look at the "abra app
+upgrade <app>" command.
 
 You may pass "--force" to re-deploy the same version again. This can be useful
 if the container runtime has gotten into a weird state.
@@ -31,6 +36,181 @@ Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	Action:       internal.DeployAction,
+	Action: func(c *cli.Context) error {
-	BashComplete: autocomplete.AppNameComplete,
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			if internal.Force {
+				logrus.Warnf("'%s' already deployed but continuing (--force)", stackName)
+			} else if internal.Chaos {
+				logrus.Warnf("'%s' already deployed but continuing (--chaos)", stackName)
+			} else {
+				logrus.Fatalf("'%s' is already deployed", stackName)
+			}
+		}
+
+		version := deployedVersion
+		if version == "" && !internal.Chaos {
+			versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if len(versions) > 0 {
+				version = versions[len(versions)-1]
+				logrus.Debugf("choosing '%s' as version to deploy", version)
+				if err := recipe.EnsureVersion(app.Type, version); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				version = "latest commit"
+				logrus.Warn("no versions detected, using latest commit")
+				if err := recipe.EnsureLatest(app.Type); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		if version == "" && !internal.Chaos {
+			logrus.Debugf("choosing '%s' as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Type, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warnf("chaos mode engaged")
+			var err error
+			version, err = recipe.ChaosVersion(app.Type)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}
+
+// DeployOverview shows a deployment overview
+func DeployOverview(app config.App, version, message string) error {
+	tableCol := []string{"server", "compose", "domain", "stack", "version"}
+	table := abraFormatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), version})
+	table.Render()
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: message,
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}
+
+// NewVersionOverview shows an upgrade or downgrade overview
+func NewVersionOverview(app config.App, currentVersion, newVersion string) error {
+	tableCol := []string{"server", "compose", "domain", "stack", "current version", "to be deployed"}
+	table := abraFormatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), currentVersion, newVersion})
+	table.Render()
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with deployment?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
 }

cli/app/errors.go

@@ -1,145 +0,0 @@
-package app
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var appErrorsCommand = cli.Command{
-	Name:      "errors",
-	Usage:     "List errors for a deployed app",
-	ArgsUsage: "<domain>",
-	Description: `
-List errors for a deployed app.
-
-This is a best-effort implementation and an attempt to gather a number of tips
-& tricks for finding errors together into one convenient command. When an app
-is failing to deploy or having issues, it could be a lot of things.
-
-This command currently takes into account:
-
-    Is the service deployed?
-    Is the service killed by an OOM error?
-    Is the service reporting an error (like in "ps --no-trunc" output)
-    Is the service healthcheck failing? what are the healthcheck logs?
-
-Got any more ideas? Please let us know:
-
-    https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose
-
-This command is best accompanied by "abra app logs <domain>" which may reveal
-further information which can help you debug the cause of an app failure via
-the logs.
-`,
-	Aliases: []string{"e"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.WatchFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-		conf := runtime.New()
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
-		}
-
-		if !internal.Watch {
-			if err := checkErrors(c, cl, app, conf); err != nil {
-				logrus.Fatal(err)
-			}
-			return nil
-		}
-
-		for {
-			if err := checkErrors(c, cl, app, conf); err != nil {
-				logrus.Fatal(err)
-			}
-			time.Sleep(2 * time.Second)
-		}
-
-		return nil
-	},
-}
-
-func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App, conf *runtime.Config) error {
-	recipe, err := recipe.Get(app.Recipe, conf)
-	if err != nil {
-		return err
-	}
-
-	for _, service := range recipe.Config.Services {
-		filters := filters.NewArgs()
-		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
-
-		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
-		if err != nil {
-			return err
-		}
-
-		if len(containers) == 0 {
-			logrus.Warnf("%s is not up, something seems wrong", service.Name)
-			continue
-		}
-
-		container := containers[0]
-		containerState, err := cl.ContainerInspect(context.Background(), container.ID)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if containerState.State.OOMKilled {
-			logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
-		}
-
-		if containerState.State.Error != "" {
-			logrus.Warnf("%s reports this error: %s", service.Name, containerState.State.Error)
-		}
-
-		if containerState.State.Health != nil {
-			if containerState.State.Health.Status != "healthy" {
-				logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
-				logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
-				for _, log := range containerState.State.Health.Log {
-					logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func getServiceName(names []string) string {
-	containerName := strings.Join(names, " ")
-	trimmed := strings.TrimPrefix(containerName, "/")
-	return strings.Split(trimmed, ".")[0]
-}

cli/app/list.go

@@ -1,163 +1,120 @@
 package app
 
 import (
-	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
 
-	"coopcloud.tech/abra/cli/internal"
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var status bool
 var statusFlag = &cli.BoolFlag{
-	Name:        "status, S",
+	Name:        "status",
+	Aliases:     []string{"S"},
+	Value:       false,
 	Usage:       "Show app deployment status",
 	Destination: &status,
 }
 
-var recipeFilter string
+var appType string
-var recipeFlag = &cli.StringFlag{
+var typeFlag = &cli.StringFlag{
-	Name:        "recipe, r",
+	Name:        "type",
+	Aliases:     []string{"t"},
 	Value:       "",
-	Usage:       "Show apps of a specific recipe",
+	Usage:       "Show apps of a specific type",
-	Destination: &recipeFilter,
+	Destination: &appType,
 }
 
 var listAppServer string
 var listAppServerFlag = &cli.StringFlag{
-	Name:        "server, s",
+	Name:        "server",
+	Aliases:     []string{"s"},
 	Value:       "",
 	Usage:       "Show apps of a specific server",
 	Destination: &listAppServer,
 }
 
-type appStatus struct {
+var appListCommand = &cli.Command{
-	Server  string `json:"server"`
+	Name:  "list",
-	Recipe  string `json:"recipe"`
+	Usage: "List all managed apps",
-	AppName string `json:"appName"`
-	Domain  string `json:"domain"`
-	Status  string `json:"status"`
-	Version string `json:"version"`
-	Upgrade string `json:"upgrade"`
-}
-
-type serverStatus struct {
-	Apps             []appStatus `json:"apps"`
-	AppCount         int         `json:"appCount"`
-	VersionCount     int         `json:"versionCount"`
-	UnversionedCount int         `json:"unversionedCount"`
-	LatestCount      int         `json:"latestCount"`
-	UpgradeCount     int         `json:"upgradeCount"`
-}
-
-var appListCommand = cli.Command{
-	Name:    "list",
-	Aliases: []string{"ls"},
-	Usage:   "List all managed apps",
 	Description: `
-Read the local file system listing of apps and servers (e.g. ~/.abra/) to
+This command looks at your local file system listing of apps and servers (e.g.
-generate a report of all your apps.
+in ~/.abra/) to generate a report of all your apps.
 
 By passing the "--status/-S" flag, you can query all your servers for the
 actual live deployment status. Depending on how many servers you manage, this
 can take some time.
-`,
+	`,
+	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.MachineReadableFlag,
 		statusFlag,
 		listAppServerFlag,
-		recipeFlag,
+		typeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		appFiles, err := config.LoadAppFiles(listAppServer)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		apps, err := config.GetApps(appFiles, recipeFilter)
+		apps, err := config.GetApps(appFiles)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
+		sort.Sort(config.ByServerAndType(apps))
-		sort.Sort(config.ByServerAndRecipe(apps))
 
 		statuses := make(map[string]map[string]string)
-		var catl recipe.RecipeCatalogue
+		tableCol := []string{"Server", "Type", "Domain"}
 		if status {
-			alreadySeen := make(map[string]bool)
+			tableCol = append(tableCol, "Status", "Version", "Updates")
-			for _, app := range apps {
+			statuses, err = config.GetAppStatuses(appFiles)
-				if _, ok := alreadySeen[app.Server]; !ok {
-					alreadySeen[app.Server] = true
-				}
-			}
-
-			statuses, err = config.GetAppStatuses(apps, internal.MachineReadable)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			catl, err = recipe.ReadRecipeCatalogue()
 			if err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		var totalServersCount int
+		table := abraFormatter.CreateTable(tableCol)
-		var totalAppsCount int
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
-		allStats := make(map[string]serverStatus)
+
+		var (
+			versionedAppsCount   int
+			unversionedAppsCount int
+			onLatestCount        int
+			canUpgradeCount      int
+		)
+
 		for _, app := range apps {
-			var stats serverStatus
+			var tableRow []string
-			var ok bool
+			if app.Type == appType || appType == "" {
-			if stats, ok = allStats[app.Server]; !ok {
+				// If type flag is set, check for it, if not, Type == ""
-				stats = serverStatus{}
+				tableRow = []string{app.Server, app.Type, app.Domain}
-				if recipeFilter == "" {
-					// count server, no filtering
-					totalServersCount++
-				}
-			}
-
-			if app.Recipe == recipeFilter || recipeFilter == "" {
-				if recipeFilter != "" {
-					// only count server if matches filter
-					totalServersCount++
-				}
-
-				appStats := appStatus{}
-				stats.AppCount++
-				totalAppsCount++
-
 				if status {
+					stackName := app.StackName()
 					status := "unknown"
 					version := "unknown"
-					if statusMeta, ok := statuses[app.StackName()]; ok {
+					if statusMeta, ok := statuses[stackName]; ok {
 						if currentVersion, exists := statusMeta["version"]; exists {
-							if currentVersion != "" {
+							version = currentVersion
-								version = currentVersion
-							}
 						}
 						if statusMeta["status"] != "" {
 							status = statusMeta["status"]
 						}
-						stats.VersionCount++
+						tableRow = append(tableRow, status, version)
+						versionedAppsCount++
 					} else {
-						stats.UnversionedCount++
+						tableRow = append(tableRow, status, version)
+						unversionedAppsCount++
 					}
 
-					appStats.Status = status
-					appStats.Version = version
-
 					var newUpdates []string
 					if version != "unknown" {
-						updates, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type)
 						if err != nil {
 							logrus.Fatal(err)
 						}
@@ -181,89 +138,35 @@ can take some time.
 
 					if len(newUpdates) == 0 {
 						if version == "unknown" {
-							appStats.Upgrade = "unknown"
+							tableRow = append(tableRow, "unknown")
 						} else {
-							appStats.Upgrade = "latest"
+							tableRow = append(tableRow, "on latest")
-							stats.LatestCount++
+							onLatestCount++
 						}
 					} else {
-						newUpdates = internal.ReverseStringList(newUpdates)
+						// FIXME: jeezus golang why do you not have a list reverse function
-						appStats.Upgrade = strings.Join(newUpdates, "\n")
+						for i, j := 0, len(newUpdates)-1; i < j; i, j = i+1, j-1 {
-						stats.UpgradeCount++
+							newUpdates[i], newUpdates[j] = newUpdates[j], newUpdates[i]
+						}
+						tableRow = append(tableRow, strings.Join(newUpdates, "\n"))
+						canUpgradeCount++
 					}
 				}
-
-				appStats.Server = app.Server
-				appStats.Recipe = app.Recipe
-				appStats.AppName = app.Name
-				appStats.Domain = app.Domain
-
-				stats.Apps = append(stats.Apps, appStats)
 			}
-			allStats[app.Server] = stats
+			table.Append(tableRow)
 		}
 
-		if internal.MachineReadable {
+		stats := fmt.Sprintf(
-			jsonstring, err := json.Marshal(allStats)
+			"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
-			if err != nil {
+			len(apps),
-				logrus.Fatal(err)
+			versionedAppsCount,
-			} else {
+			unversionedAppsCount,
-				fmt.Println(string(jsonstring))
+			onLatestCount,
-			}
+			canUpgradeCount,
-			return nil
+		)
-		}
 
-		alreadySeen := make(map[string]bool)
+		table.SetCaption(true, stats)
-		for _, app := range apps {
+		table.Render()
-			if _, ok := alreadySeen[app.Server]; ok {
-				continue
-			}
-
-			serverStat := allStats[app.Server]
-
-			tableCol := []string{"recipe", "domain"}
-			if status {
-				tableCol = append(tableCol, []string{"status", "version", "upgrade"}...)
-			}
-
-			table := formatter.CreateTable(tableCol)
-
-			for _, appStat := range serverStat.Apps {
-				tableRow := []string{appStat.Recipe, appStat.Domain}
-				if status {
-					tableRow = append(tableRow, []string{appStat.Status, appStat.Version, appStat.Upgrade}...)
-				}
-				table.Append(tableRow)
-			}
-
-			if table.NumLines() > 0 {
-				table.Render()
-
-				if status {
-					fmt.Println(fmt.Sprintf(
-						"server: %s | total apps: %v | versioned: %v | unversioned: %v | latest: %v | upgrade: %v",
-						app.Server,
-						serverStat.AppCount,
-						serverStat.VersionCount,
-						serverStat.UnversionedCount,
-						serverStat.LatestCount,
-						serverStat.UpgradeCount,
-					))
-				} else {
-					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.AppCount))
-				}
-			}
-
-			if len(allStats) > 1 && table.NumLines() > 0 {
-				fmt.Println() // newline separator for multiple servers
-			}
-
-			alreadySeen[app.Server] = true
-		}
-
-		if len(allStats) > 1 {
-			fmt.Println(fmt.Sprintf("total servers: %v | total apps: %v ", totalServersCount, totalAppsCount))
-		}
 
 		return nil
 	},

cli/app/logs.go

@@ -1,45 +1,27 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
 	"sync"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/service"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var logOpts = types.ContainerLogsOptions{
-	ShowStderr: true,
-	ShowStdout: true,
-	Since:      "",
-	Until:      "",
-	Timestamps: true,
-	Follow:     true,
-	Tail:       "20",
-	Details:    false,
-}
-
 // stackLogs lists logs for all stack services
-func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
+func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
-	filters, err := app.Filters(true, false)
+	filters := filters.NewArgs()
-	if err != nil {
+	filters.Add("name", stackName)
-		logrus.Fatal(err)
-	}
-
 	serviceOpts := types.ServiceListOptions{Filters: filters}
-	services, err := client.ServiceList(context.Background(), serviceOpts)
+	services, err := client.ServiceList(c.Context, serviceOpts)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -48,14 +30,19 @@ func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
 	for _, service := range services {
 		wg.Add(1)
 		go func(s string) {
-			if internal.StdErrOnly {
+			logOpts := types.ContainerLogsOptions{
-				logOpts.ShowStdout = false
+				Details:    true,
+				Follow:     true,
+				ShowStderr: true,
+				ShowStdout: true,
+				Tail:       "20",
+				Timestamps: true,
 			}
-
+			logs, err := client.ServiceLogs(c.Context, s, logOpts)
-			logs, err := client.ServiceLogs(context.Background(), s, logOpts)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+			// defer after err check as any err returns a nil io.ReadCloser
 			defer logs.Close()
 
 			_, err = io.Copy(os.Stdout, logs)
@@ -64,72 +51,74 @@ func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
 			}
 		}(service.ID)
 	}
-
 	wg.Wait()
-
 	os.Exit(0)
 }
 
-var appLogsCommand = cli.Command{
+var appLogsCommand = &cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
-	ArgsUsage: "<domain> [<service>]",
+	ArgsUsage: "[<service>]",
 	Usage:     "Tail app logs",
-	Flags: []cli.Flag{
-		internal.StdErrOnlyFlag,
-		internal.SinceLogsFlag,
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c, runtime.WithEnsureRecipeExists(false))
+		app := internal.ValidateApp(c)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logOpts.Since = internal.SinceLogs
-
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
-			logrus.Debugf("tailing logs for all %s services", app.Recipe)
+			logrus.Debug("tailing logs for all app services")
-			stackLogs(c, app, cl)
+			stackLogs(c, app.StackName(), cl)
-		} else {
+		}
-			logrus.Debugf("tailing logs for %s", serviceName)
+		logrus.Debugf("tailing logs for '%s'", serviceName)
-			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
+
-				logrus.Fatal(err)
+		service := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-			}
+		filters := filters.NewArgs()
+		filters.Add("name", service)
+		serviceOpts := types.ServiceListOptions{Filters: filters}
+		services, err := cl.ServiceList(c.Context, serviceOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if len(services) != 1 {
+			logrus.Fatalf("expected 1 service but got %v", len(services))
+		}
+
+		logOpts := types.ContainerLogsOptions{
+			Details:    true,
+			Follow:     true,
+			ShowStderr: true,
+			ShowStdout: true,
+			Tail:       "20",
+			Timestamps: true,
+		}
+		logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		// defer after err check as any err returns a nil io.ReadCloser
+		defer logs.Close()
+
+		_, err = io.Copy(os.Stdout, logs)
+		if err != nil && err != io.EOF {
+			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-}
+	BashComplete: func(c *cli.Context) {
-
+		appNames, err := config.GetAppNames()
-func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
+		if err != nil {
-	filters := filters.NewArgs()
+			logrus.Warn(err)
-	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
+		}
-
+		if c.NArg() > 0 {
-	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
+			return
-	if err != nil {
+		}
-		logrus.Fatal(err)
+		for _, a := range appNames {
-	}
+			fmt.Println(a)
-
+		}
-	if internal.StdErrOnly {
+	},
-		logOpts.ShowStdout = false
-	}
-
-	logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	defer logs.Close()
-
-	_, err = io.Copy(os.Stdout, logs)
-	if err != nil && err != io.EOF {
-		logrus.Fatal(err)
-	}
-
-	return nil
 }

cli/app/new.go

@@ -1,17 +1,54 @@
 package app
 
 import (
+	"fmt"
+	"path"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
 )
 
+type secrets map[string]string
+
+var domain string
+var domainFlag = &cli.StringFlag{
+	Name:        "domain",
+	Aliases:     []string{"d"},
+	Value:       "",
+	Usage:       "Choose a domain name",
+	Destination: &domain,
+}
+
+var newAppServer string
+var newAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &newAppServer,
+}
+
+var newAppName string
+var newAppNameFlag = &cli.StringFlag{
+	Name:        "app-name",
+	Aliases:     []string{"a"},
+	Value:       "",
+	Usage:       "Choose an app name",
+	Destination: &newAppName,
+}
+
 var appNewDescription = `
-Take a recipe and uses it to create a new app. This new app configuration is
+This command takes a recipe and uses it to create a new app. This new app
-stored in your ~/.abra directory under the appropriate server.
+configuration is stored in your ~/.abra directory under the appropriate server.
 
 This command does not deploy your app for you. You will need to run "abra app
-deploy <domain>" to do so.
+deploy <app>" to do so.
 
 You can see what recipes are available (i.e. values for the <recipe> argument)
 by running "abra recipe ls".
@@ -26,21 +63,172 @@ pass store (see passwordstore.org for more). The pass command must be available
 on your $PATH.
 `
 
-var appNewCommand = cli.Command{
+var appNewCommand = &cli.Command{
 	Name:        "new",
-	Aliases:     []string{"n"},
 	Usage:       "Create a new app",
+	Aliases:     []string{"n"},
 	Description: appNewDescription,
 	Flags: []cli.Flag{
-		internal.DebugFlag,
+		newAppServerFlag,
-		internal.NoInputFlag,
+		domainFlag,
-		internal.NewAppServerFlag,
+		newAppNameFlag,
-		internal.DomainFlag,
 		internal.PassFlag,
 		internal.SecretsFlag,
 	},
-	Before:       internal.SubCommandBefore,
+	ArgsUsage: "<recipe>",
-	ArgsUsage:    "[<recipe>]",
+	Action:    action,
-	Action:       internal.NewAction,
+	BashComplete: func(c *cli.Context) {
-	BashComplete: autocomplete.RecipeNameComplete,
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag() error {
+	if domain == "" {
+		prompt := &survey.Input{
+			Message: "Specify app domain",
+		}
+		if err := survey.AskOne(prompt, &domain); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	servers, err := config.GetServers()
+	if err != nil {
+		return err
+	}
+
+	if newAppServer == "" {
+		prompt := &survey.Select{
+			Message: "Select app server:",
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &newAppServer); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the AppName flag was used. if not, asks the user for it.
+func ensureAppNameFlag() error {
+	if newAppName == "" {
+		prompt := &survey.Input{
+			Message: "Specify app name:",
+			Default: config.SanitiseAppName(domain),
+		}
+		if err := survey.AskOne(prompt, &newAppName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(sanitisedAppName string) (secrets, error) {
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", newAppServer, fmt.Sprintf("%s.env", sanitisedAppName))
+	appEnv, err := config.ReadEnv(appEnvPath)
+	if err != nil {
+		return nil, err
+	}
+
+	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
+	secrets, err := secret.GenerateSecrets(secretEnvVars, sanitisedAppName, newAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if internal.Pass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(secretValue, secretName, sanitisedAppName, newAppServer); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return secrets, nil
+}
+
+// action is the main command-line action for this package
+func action(c *cli.Context) error {
+	recipe := internal.ValidateRecipe(c)
+
+	if err := config.EnsureAbraDirExists(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureServerFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureDomainFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureAppNameFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	sanitisedAppName := config.SanitiseAppName(newAppName)
+	if len(sanitisedAppName) > 45 {
+		logrus.Fatalf("'%s' cannot be longer than 45 characters", sanitisedAppName)
+	}
+	logrus.Debugf("'%s' sanitised as '%s' for new app", newAppName, sanitisedAppName)
+
+	if err := config.TemplateAppEnvSample(recipe.Name, newAppName, newAppServer, domain, recipe.Name); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if internal.Secrets {
+		secrets, err := createSecrets(sanitisedAppName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretCols := []string{"Name", "Value"}
+		secretTable := abraFormatter.CreateTable(secretCols)
+		for secret := range secrets {
+			secretTable.Append([]string{secret, secrets[secret]})
+		}
+
+		if len(secrets) > 0 {
+			defer secretTable.Render()
+		}
+	}
+
+	if newAppServer == "default" {
+		newAppServer = "local"
+	}
+
+	tableCol := []string{"Name", "Domain", "Type", "Server"}
+	table := abraFormatter.CreateTable(tableCol)
+	table.Append([]string{sanitisedAppName, domain, recipe.Name, newAppServer})
+
+	fmt.Println("")
+	fmt.Println(fmt.Sprintf("New '%s' created! Here is your new app overview:", recipe.Name))
+	fmt.Println("")
+	table.Render()
+	fmt.Println("")
+	fmt.Println("You can configure this app by running the following:")
+	fmt.Println(fmt.Sprintf("\n    abra app config %s", sanitisedAppName))
+	fmt.Println("")
+	fmt.Println("You can deploy this app by running the following:")
+	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", sanitisedAppName))
+	fmt.Println("")
+
+	return nil
 }

cli/app/ps.go

@@ -1,83 +1,82 @@
 package app
 
 import (
-	"context"
+	"fmt"
 	"strings"
 	"time"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/docker/cli/cli/command/formatter"
-	"coopcloud.tech/abra/pkg/service"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/buger/goterm"
-	dockerFormatter "github.com/docker/cli/cli/command/formatter"
 	"github.com/docker/docker/api/types"
-	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appPsCommand = cli.Command{
+var watch bool
-	Name:        "ps",
+var watchFlag = &cli.BoolFlag{
-	Aliases:     []string{"p"},
+	Name:        "watch",
-	Usage:       "Check app status",
+	Aliases:     []string{"w"},
-	ArgsUsage:   "<domain>",
+	Value:       false,
-	Description: "Show a more detailed status output of a specific deployed app",
+	Usage:       "Watch status by polling repeatedly",
+	Destination: &watch,
+}
+
+var appPsCommand = &cli.Command{
+	Name:    "ps",
+	Usage:   "Check app status",
+	Aliases: []string{"p"},
 	Flags: []cli.Flag{
-		internal.WatchFlag,
+		watchFlag,
-		internal.DebugFlag,
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+		if !watch {
-
+			showPSOutput(c)
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
-		}
-
-		if !internal.Watch {
-			showPSOutput(c, app, cl)
 			return nil
 		}
 
-		goterm.Clear()
+		// TODO: how do we make this update in-place in an x-platform way?
 		for {
-			goterm.MoveCursor(1, 1)
+			showPSOutput(c)
-			showPSOutput(c, app, cl)
-			goterm.Flush()
 			time.Sleep(2 * time.Second)
 		}
 	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }
 
 // showPSOutput renders ps output.
-func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
+func showPSOutput(c *cli.Context) {
-	filters, err := app.Filters(true, true)
+	app := internal.ValidateApp(c)
+
+	cl, err := client.New(app.Server)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+	filters := filters.NewArgs()
+	filters.Add("name", app.StackName())
+
+	containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
+	tableCol := []string{"image", "created", "status", "ports", "names"}
-	table := formatter.CreateTable(tableCol)
+	table := abraFormatter.CreateTable(tableCol)
 
 	for _, container := range containers {
 		var containerNames []string
@@ -87,12 +86,11 @@ func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
 		}
 
 		tableRow := []string{
-			service.ContainerToServiceName(container.Names, app.StackName()),
+			abraFormatter.RemoveSha(container.Image),
-			formatter.RemoveSha(container.Image),
+			abraFormatter.HumanDuration(container.Created),
-			formatter.HumanDuration(container.Created),
 			container.Status,
-			container.State,
+			formatter.DisplayablePorts(container.Ports),
-			dockerFormatter.DisplayablePorts(container.Ports),
+			strings.Join(containerNames, "\n"),
 		}
 		table.Append(tableRow)
 	}

cli/app/remove.go

@@ -1,84 +1,76 @@
 package app
 
 import (
-	"context"
 	"fmt"
 	"os"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/abra/pkg/config"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appRemoveCommand = cli.Command{
+// Volumes stores the variable from VolumesFlag
-	Name:      "remove",
+var Volumes bool
-	Aliases:   []string{"rm"},
-	ArgsUsage: "<domain>",
-	Usage:     "Remove all app data, locally and remotely",
-	Description: `
-This command removes everything related to an app which is already undeployed.
 
-By default, it will prompt for confirmation before proceeding. All secrets,
+// VolumesFlag is used to specify if volumes should be deleted when deleting an app
-volumes and the local app env file will be deleted.
+var VolumesFlag = &cli.BoolFlag{
+	Name:        "volumes",
+	Value:       false,
+	Destination: &Volumes,
+}
 
-Only run this command when you are sure you want to completely remove the app
+var appRemoveCommand = &cli.Command{
-and all associated app data. This is a destructive action, Be Careful!
+	Name:    "remove",
-
+	Usage:   "Remove an already undeployed app",
-If you would like to delete specific volumes or secrets, please use removal
+	Aliases: []string{"rm"},
-sub-commands under "app volume" and "app secret" instead.
-
-Please note, if you delete the local app env file without removing volumes and
-secrets first, Abra will *not* be able to help you remove them afterwards.
-
-To delete everything without prompt, use the "--force/-f" or the "--no-input/n"
-flag.
-`,
 	Flags: []cli.Flag{
+		VolumesFlag,
 		internal.ForceFlag,
-		internal.DebugFlag,
-		internal.NoInputFlag,
 	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if !internal.Force && !internal.NoInput {
+		if !internal.Force {
 			response := false
-			msg := "ALERTA ALERTA: this will completely remove %s data and configurations locally and remotely, are you sure?"
+			prompt := &survey.Confirm{
-			prompt := &survey.Confirm{Message: fmt.Sprintf(msg, app.Name)}
+				Message: fmt.Sprintf("about to delete %s, are you sure?", app.Name),
+			}
 			if err := survey.AskOne(prompt, &response); err != nil {
 				logrus.Fatal(err)
 			}
 			if !response {
-				logrus.Fatal("aborting as requested")
+				logrus.Fatal("user aborted app removal")
 			}
 		}
 
+		appFiles, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
+		if !internal.Force {
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+			// FIXME: only query for app we are interested in, not all of them!
-		if err != nil {
+			statuses, err := config.GetAppStatuses(appFiles)
-			logrus.Fatal(err)
+			if err != nil {
-		}
+				logrus.Fatal(err)
-		if isDeployed {
+			}
-			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+			if statuses[app.Name]["status"] == "deployed" {
+				logrus.Fatalf("'%s' is still deployed. Run \"abra app %s undeploy\" or pass --force", app.Name, app.Name)
+			}
 		}
 
-		fs, err := app.Filters(false, false)
+		fs := filters.NewArgs()
-		if err != nil {
+		fs.Add("name", app.Name)
-			logrus.Fatal(err)
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: fs})
-		}
-
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: fs})
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -92,8 +84,20 @@ flag.
 		}
 
 		if len(secrets) > 0 {
-			for _, name := range secretNames {
+			var secretNamesToRemove []string
-				err := cl.SecretRemove(context.Background(), secrets[name])
+			if !internal.Force {
+				secretsPrompt := &survey.MultiSelect{
+					Message: "which secrets do you want to remove?",
+					Options: secretNames,
+					Default: secretNames,
+				}
+				if err := survey.AskOne(secretsPrompt, &secretNamesToRemove); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			for _, name := range secretNamesToRemove {
+				err := cl.SecretRemove(c.Context, secrets[name])
 				if err != nil {
 					logrus.Fatal(err)
 				}
@@ -103,12 +107,7 @@ flag.
 			logrus.Info("no secrets to remove")
 		}
 
-		fs, err = app.Filters(false, true)
+		volumeListOKBody, err := cl.VolumeList(c.Context, fs)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		volumeListOKBody, err := cl.VolumeList(context.Background(), fs)
 		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
@@ -120,24 +119,50 @@ flag.
 		}
 
 		if len(vols) > 0 {
-			var removeVols []string
+			if Volumes {
-			for _, vol := range removeVols {
+				var removeVols []string
-				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				if !internal.Force {
-				if err != nil {
+					volumesPrompt := &survey.MultiSelect{
-					logrus.Fatal(err)
+						Message: "which volumes do you want to remove?",
+						Options: vols,
+						Default: vols,
+					}
+					if err := survey.AskOne(volumesPrompt, &removeVols); err != nil {
+						logrus.Fatal(err)
+					}
 				}
-				logrus.Info(fmt.Sprintf("volume %s removed", vol))
+				for _, vol := range removeVols {
+					err := cl.VolumeRemove(c.Context, vol, internal.Force) // last argument is for force removing
+					if err != nil {
+						logrus.Fatal(err)
+					}
+					logrus.Info(fmt.Sprintf("volume %s removed", vol))
+				}
+			} else {
+				logrus.Info("no volumes were removed")
 			}
 		} else {
 			logrus.Info("no volumes to remove")
 		}
 
-		if err = os.Remove(app.Path); err != nil {
+		err = os.Remove(app.Path)
+		if err != nil {
 			logrus.Fatal(err)
 		}
-
 		logrus.Info(fmt.Sprintf("file: %s removed", app.Path))
 
 		return nil
 	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/restart.go

@@ -1,70 +0,0 @@
-package app
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
-	upstream "coopcloud.tech/abra/pkg/upstream/service"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var appRestartCommand = cli.Command{
-	Name:      "restart",
-	Aliases:   []string{"re"},
-	Usage:     "Restart an app",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Description:  `This command restarts a service within a deployed app.`,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		serviceNameShort := c.Args().Get(1)
-		if serviceNameShort == "" {
-			err := errors.New("missing service?")
-			internal.ShowSubcommandHelpAndError(c, err)
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
-
-		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)
-		if err := upstream.RunServiceScale(context.Background(), cl, serviceName, 0); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := stack.WaitOnService(context.Background(), cl, serviceName, app.Name); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("%s has been scaled to 0 (restart logic)", serviceName)
-
-		logrus.Debugf("attempting to scale %s to 1 (restart logic)", serviceName)
-		if err := upstream.RunServiceScale(context.Background(), cl, serviceName, 1); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := stack.WaitOnService(context.Background(), cl, serviceName, app.Name); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("%s has been scaled to 1 (restart logic)", serviceName)
-
-		logrus.Infof("%s service successfully restarted", serviceNameShort)
-
-		return nil
-	},
-}

cli/app/restore.go

@@ -1,205 +1,79 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"os/exec"
+	"path"
+	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-type restoreConfig struct {
+var restoreAllServices bool
-	preHookCmd  string
+var restoreAllServicesFlag = &cli.BoolFlag{
-	postHookCmd string
+	Name:        "all",
+	Value:       false,
+	Destination: &restoreAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Restore all services",
 }
 
-var appRestoreCommand = cli.Command{
+var appRestoreCommand = &cli.Command{
 	Name:      "restore",
-	Aliases:   []string{"rs"},
+	Usage:     "Restore an app from a backup",
-	Usage:     "Run app restore",
+	Aliases:   []string{"r"},
-	ArgsUsage: "<domain> <service> <file>",
+	Flags:     []cli.Flag{restoreAllServicesFlag},
-	Flags: []cli.Flag{
+	ArgsUsage: "<service> [<backup file>]",
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app restore.
-
-Pre/post hook commands are defined in the recipe configuration. Abra reads this
-configuration and run the comands in the context of the service before
-restoring the backup.
-
-Unlike "abra app backup", restore must be run on a per-service basis. You can
-not restore all services in one go. Backup files produced by Abra are
-compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic.
-
-Example:
-
-    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		conf := runtime.New()
 
-		cl, err := client.New(app.Server)
+		if c.Args().Len() > 1 && restoreAllServices {
-		if err != nil {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use <service>/<backup file> and '--all' together"))
-			logrus.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
-		if serviceName == "" {
+		if _, err := os.Stat(abraSh); err != nil {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
-		}
-
-		backupPath := c.Args().Get(2)
-		if backupPath == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
-		}
-
-		if _, err := os.Stat(backupPath); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("%s doesn't exist?", backupPath)
+				logrus.Fatalf("'%s' does not exist?", abraSh)
 			}
-		}
-
-		recipe, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		restoreConfigs := make(map[string]restoreConfig)
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
-		for _, service := range recipe.Config.Services {
+		execCmd := "abra_restore"
-			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
+		if !restoreAllServices {
-				if restoreEnabled == "true" {
+			serviceName := c.Args().Get(1)
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+			if serviceName == "" {
-					rsConfig := restoreConfig{}
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
-
-					logrus.Debugf("restore config detected for %s", fullServiceName)
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						rsConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						rsConfig.postHookCmd = postHookCmd
-					}
-
-					restoreConfigs[service.Name] = rsConfig
-				}
 			}
+			execCmd = fmt.Sprintf("abra_restore_%s", serviceName)
 		}
 
-		rsConfig, ok := restoreConfigs[serviceName]
+		bytes, err := ioutil.ReadFile(abraSh)
-		if !ok {
+		if err != nil {
-			rsConfig = restoreConfig{}
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
 		}
 
-		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
+		backupFile := c.Args().Get(2)
+		if backupFile != "" {
+			execCmd = fmt.Sprintf("%s %s", execCmd, backupFile)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		if err := internal.RunCmd(cmd); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
-
-// runRestore does the actual restore logic.
-func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if rsConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
-	}
-
-	backupReader, err := os.Open(backupPath)
-	if err != nil {
-		return err
-	}
-
-	content, err := archive.DecompressStream(backupReader)
-	if err != nil {
-		return err
-	}
-
-	// we use absolute paths so tar knows what to do. it will restore files
-	// according to the paths set in the compresed archive
-	restorePath := "/"
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
-		return err
-	}
-
-	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
-
-	if rsConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
-	}
-
-	return nil
-}

cli/app/rollback.go

@@ -1,38 +1,30 @@
 package app
 
 import (
-	"context"
 	"fmt"
 
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
+	stack "coopcloud.tech/abra/pkg/client/stack"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appRollbackCommand = cli.Command{
+var appRollbackCommand = &cli.Command{
 	Name:      "rollback",
-	Aliases:   []string{"rl"},
 	Usage:     "Roll an app back to a previous version",
-	ArgsUsage: "<domain>",
+	Aliases:   []string{"r", "downgrade"},
+	ArgsUsage: "<app>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
 This command rolls an app back to a previous version if one exists.
 
@@ -40,67 +32,54 @@ You may pass "--force/-f" to downgrade to the same version again. This can be
 useful if the container runtime has gotten into a weird state.
 
 This action could be destructive, please ensure you have a copy of your app
-data beforehand.
+data beforehand - see "abra app backup <app>" for more.
 
 Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
 recipes.
 `,
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
-		conf := runtime.New()
-
-		if !internal.Chaos {
-			if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		r, err := recipe.Get(app.Recipe, conf)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
-		}
-
 		var availableDowngrades []string
-		if deployedVersion == "unknown" {
+		if deployedVersion == "" {
+			deployedVersion = "unknown"
 			availableDowngrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
 		}
 
 		if deployedVersion != "unknown" && !internal.Chaos {
@@ -119,21 +98,23 @@ recipes.
 			}
 
 			if len(availableDowngrades) == 0 {
-				logrus.Info("no available downgrades, you're on oldest ✌️")
+				logrus.Fatal("no available downgrades, you're on latest")
-				return nil
 			}
 		}
 
-		availableDowngrades = internal.ReverseStringList(availableDowngrades)
+		// FIXME: jeezus golang why do you not have a list reverse function
+		for i, j := 0, len(availableDowngrades)-1; i < j; i, j = i+1, j-1 {
+			availableDowngrades[i], availableDowngrades[j] = availableDowngrades[j], availableDowngrades[i]
+		}
 
 		var chosenDowngrade string
 		if !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force {
 				chosenDowngrade = availableDowngrades[0]
-				logrus.Debugf("choosing %s as version to downgrade to (--force)", chosenDowngrade)
+				logrus.Debugf("choosing '%s' as version to downgrade to (--force)", chosenDowngrade)
 			} else {
 				prompt := &survey.Select{
-					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
+					Message: fmt.Sprintf("Please select a downgrade (current version: '%s'):", deployedVersion),
 					Options: availableDowngrades,
 				}
 				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
@@ -143,7 +124,7 @@ recipes.
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenDowngrade); err != nil {
+			if err := recipe.EnsureVersion(app.Type, chosenDowngrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -151,13 +132,13 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenDowngrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenDowngrade, err = recipe.ChaosVersion(app.Type)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
 		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
@@ -166,7 +147,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -180,18 +161,14 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		config.ExposeAllEnv(stackName, compose, app.Env)
-		config.SetRecipeLabel(compose, stackName, app.Recipe)
-		config.SetChaosLabel(compose, stackName, internal.Chaos)
-		config.SetUpdateLabel(compose, stackName, app.Env)
 
 		if !internal.Force {
-			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
+			if err := NewVersionOverview(app, deployedVersion, chosenDowngrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/run.go

@@ -1,55 +1,51 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/client/container"
-	"coopcloud.tech/abra/pkg/upstream/container"
+	"coopcloud.tech/abra/pkg/config"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var user string
 var userFlag = &cli.StringFlag{
-	Name:        "user, u",
+	Name:        "user",
 	Value:       "",
 	Destination: &user,
 }
 
 var noTTY bool
 var noTTYFlag = &cli.BoolFlag{
-	Name:        "no-tty, t",
+	Name:        "no-tty",
+	Value:       false,
 	Destination: &noTTY,
 }
 
-var appRunCommand = cli.Command{
+var appRunCommand = &cli.Command{
-	Name:    "run",
+	Name: "run",
-	Aliases: []string{"r"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
 		noTTYFlag,
 		userFlag,
 	},
-	Before:       internal.SubCommandBefore,
+	Aliases:   []string{"r"},
-	ArgsUsage:    "<domain> <service> <args>...",
+	ArgsUsage: "<service> <args>...",
-	Usage:        "Run a command in a service container",
+	Usage:     "Run a command in a service container",
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if len(c.Args()) < 2 {
+		if c.Args().Len() < 2 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no <service> provided?"))
 		}
 
-		if len(c.Args()) < 3 {
+		if c.Args().Len() < 3 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no <args> provided?"))
 		}
 
@@ -59,16 +55,23 @@ var appRunCommand = cli.Command{
 		}
 
 		serviceName := c.Args().Get(1)
-		stackAndServiceName := fmt.Sprintf("^%s_%s", app.StackName(), serviceName)
+		stackAndServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
 		filters := filters.NewArgs()
 		filters.Add("name", stackAndServiceName)
 
-		targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		cmd := c.Args()[2:]
+		if len(containers) == 0 {
+			logrus.Fatalf("no containers matching '%s' found?", stackAndServiceName)
+		}
+		if len(containers) > 1 {
+			logrus.Fatalf("expected 1 container matching '%s' but got %d", stackAndServiceName, len(containers))
+		}
+
+		cmd := c.Args().Slice()[2:]
 		execCreateOpts := types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
@@ -85,16 +88,41 @@ var appRunCommand = cli.Command{
 			execCreateOpts.Tty = false
 		}
 
-		// FIXME: avoid instantiating a new CLI
+		// FIXME: an absolutely monumental hack to instantiate another command-line
+		// client withing our command-line client so that we pass something down
+		// the tubes that satisfies the necessary interface requirements. We should
+		// refactor our vendored container code to not require all this cruft.  For
+		// now, It Works.
 		dcli, err := command.NewDockerCli()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if err := container.RunExec(dcli, cl, containers[0].ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
+	BashComplete: func(c *cli.Context) {
+		switch c.NArg() {
+		case 0:
+			appNames, err := config.GetAppNames()
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, a := range appNames {
+				fmt.Println(a)
+			}
+		case 1:
+			appName := c.Args().First()
+			serviceNames, err := config.GetAppServiceNames(appName)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, s := range serviceNames {
+				fmt.Println(s)
+			}
+		}
+	},
 }

cli/app/secret.go

@@ -1,59 +1,41 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"os"
 	"strconv"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/docker/docker/api/types"
-	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var allSecrets bool
 var allSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
+	Name:        "all",
+	Aliases:     []string{"A"},
+	Value:       false,
 	Destination: &allSecrets,
 	Usage:       "Generate all secrets",
 }
 
-var rmAllSecrets bool
+var appSecretGenerateCommand = &cli.Command{
-var rmAllSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &rmAllSecrets,
-	Usage:       "Remove all secrets",
-}
-
-var appSecretGenerateCommand = cli.Command{
 	Name:      "generate",
 	Aliases:   []string{"g"},
 	Usage:     "Generate secrets",
-	ArgsUsage: "<domain> <secret> <version>",
+	ArgsUsage: "<secret> <version>",
-	Flags: []cli.Flag{
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
-		internal.DebugFlag,
-		allSecretsFlag,
-		internal.PassFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
+		if c.Args().Len() == 1 && !allSecrets {
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(c.Args()) == 1 && !allSecrets {
 			err := errors.New("missing arguments <secret>/<version> or '--all'")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
@@ -75,23 +57,21 @@ var appSecretGenerateCommand = cli.Command{
 				parsed := secret.ParseSecretEnvVarName(sec)
 				if secretName == parsed {
 					secretsToCreate[sec] = secretVersion
-					matches = true
 				}
 			}
-
 			if !matches {
-				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
+				logrus.Fatalf("'%s' doesn't exist in the env config?", secretName)
 			}
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
+		secretVals, err := secret.GenerateSecrets(secretsToCreate, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if internal.Pass {
 			for name, data := range secretVals {
-				if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+				if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
 					logrus.Fatal(err)
 				}
 			}
@@ -103,7 +83,7 @@ var appSecretGenerateCommand = cli.Command{
 		}
 
 		tableCol := []string{"name", "value"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 		for name, val := range secretVals {
 			table.Append([]string{name, val})
 		}
@@ -114,38 +94,16 @@ var appSecretGenerateCommand = cli.Command{
 	},
 }
 
-var appSecretInsertCommand = cli.Command{
+var appSecretInsertCommand = &cli.Command{
-	Name:    "insert",
+	Name:      "insert",
-	Aliases: []string{"i"},
+	Aliases:   []string{"i"},
-	Usage:   "Insert secret",
+	Usage:     "Insert secret",
-	Flags: []cli.Flag{
+	Flags:     []cli.Flag{internal.PassFlag},
-		internal.DebugFlag,
+	ArgsUsage: "<secret> <version> <data>",
-		internal.PassFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	ArgsUsage:    "<domain> <secret-name> <version> <data>",
-	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-This command inserts a secret into an app environment.
-
-This can be useful when you want to manually generate secrets for an app
-environment. Typically, you can let Abra generate them for you on app creation
-(see "abra app new --secrets" for more).
-
-Example:
-
-    abra app secret insert myapp db_pass v1 mySecretPassword
-
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
+		if c.Args().Len() != 4 {
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(c.Args()) != 4 {
 			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
 		}
 
@@ -154,14 +112,12 @@ Example:
 		data := c.Args().Get(3)
 
 		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
-		if err := client.StoreSecret(cl, secretName, data, app.Server); err != nil {
+		if err := client.StoreSecret(secretName, data, app.Server); err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Infof("%s successfully stored on server", secretName)
-
 		if internal.Pass {
-			if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+			if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -170,54 +126,20 @@ Example:
 	},
 }
 
-// secretRm removes a secret.
+var appSecretRmCommand = &cli.Command{
-func secretRm(cl *dockerClient.Client, app config.App, secretName, parsed string) error {
+	Name:      "remove",
-	if err := cl.SecretRemove(context.Background(), secretName); err != nil {
+	Usage:     "Remove a secret",
-		return err
+	Aliases:   []string{"rm"},
-	}
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
-
+	ArgsUsage: "<secret>",
-	logrus.Infof("deleted %s successfully from server", secretName)
-
-	if internal.PassRemove {
-		if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
-			return err
-		}
-
-		logrus.Infof("deleted %s successfully from local pass store", secretName)
-	}
-
-	return nil
-}
-
-var appSecretRmCommand = cli.Command{
-	Name:    "remove",
-	Aliases: []string{"rm"},
-	Usage:   "Remove a secret",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		rmAllSecretsFlag,
-		internal.PassRemoveFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	ArgsUsage:    "<domain> [<secret-name>]",
-	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-This command removes app secrets.
-
-Example:
-
-    abra app secret remove myapp db_pass
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
-		secrets := secret.ReadSecretEnvVars(app.Env)
 
-		if c.Args().Get(1) != "" && rmAllSecrets {
+		if c.Args().Get(1) != "" && allSecrets {
-			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret-name>' and '--all' together"))
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret>' and '--all' together"))
 		}
 
-		if c.Args().Get(1) == "" && !rmAllSecrets {
+		if c.Args().Get(1) == "" && !allSecrets {
 			internal.ShowSubcommandHelpAndError(c, errors.New("no secret(s) specified?"))
 		}
 
@@ -226,89 +148,63 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, false)
+		filters := filters.NewArgs()
+		filters.Add("name", app.StackName())
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteSecretNames := make(map[string]bool)
-		for _, cont := range secretList {
-			remoteSecretNames[cont.Spec.Annotations.Name] = true
-		}
-
-		match := false
 		secretToRm := c.Args().Get(1)
-		for sec := range secrets {
+		for _, cont := range secretList {
-			secretName := secret.ParseSecretEnvVarName(sec)
+			secretName := cont.Spec.Annotations.Name
-
+			parsed := secret.ParseGeneratedSecretName(secretName, app)
-			secVal, err := secret.ParseSecretEnvVarValue(secrets[sec])
+			if allSecrets {
-			if err != nil {
+				if err := cl.SecretRemove(c.Context, secretName); err != nil {
-				logrus.Fatal(err)
+					logrus.Fatal(err)
-			}
+				}
-
+				if internal.Pass {
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version)
+					if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
-			if _, ok := remoteSecretNames[secretRemoteName]; ok {
-				if secretToRm != "" {
-					if secretName == secretToRm {
-						if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
-							logrus.Fatal(err)
-						}
-
-						return nil
-					}
-				} else {
-					match = true
-
-					if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
 						logrus.Fatal(err)
 					}
 				}
+			} else {
+				if parsed == secretToRm {
+					if err := cl.SecretRemove(c.Context, secretName); err != nil {
+						logrus.Fatal(err)
+					}
+					if internal.Pass {
+						if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+							logrus.Fatal(err)
+						}
+					}
+				}
 			}
 		}
 
-		if !match && secretToRm != "" {
-			logrus.Fatalf("%s doesn't exist on server?", secretToRm)
-		}
-
-		if !match {
-			logrus.Fatal("no secrets to remove?")
-		}
-
 		return nil
 	},
 }
 
-var appSecretLsCommand = cli.Command{
+var appSecretLsCommand = &cli.Command{
 	Name:    "list",
+	Usage:   "List all secrets",
 	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Usage:  "List all secrets",
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		secrets := secret.ReadSecretEnvVars(app.Env)
 
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, false)
+		filters := filters.NewArgs()
-		if err != nil {
+		filters.Add("name", app.StackName())
-			logrus.Fatal(err)
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: filters})
-		}
-
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -333,23 +229,29 @@ var appSecretLsCommand = cli.Command{
 			table.Append(tableRow)
 		}
 
-		if table.NumLines() > 0 {
+		table.Render()
-			table.Render()
-		} else {
-			logrus.Warnf("no secrets stored for %s", app.Name)
-		}
-
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }
 
-var appSecretCommand = cli.Command{
+var appSecretCommand = &cli.Command{
 	Name:      "secret",
 	Aliases:   []string{"s"},
 	Usage:     "Manage app secrets",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<command>",
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		appSecretGenerateCommand,
 		appSecretInsertCommand,
 		appSecretRmCommand,

cli/app/services.go

@@ -1,80 +0,0 @@
-package app
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/service"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var appServicesCommand = cli.Command{
-	Name:      "services",
-	Aliases:   []string{"sr"},
-	Usage:     "Display all services of an app",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
-		}
-
-		filters, err := app.Filters(true, true)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		tableCol := []string{"service name", "image"}
-		table := formatter.CreateTable(tableCol)
-
-		for _, container := range containers {
-			var containerNames []string
-			for _, containerName := range container.Names {
-				trimmed := strings.TrimPrefix(containerName, "/")
-				containerNames = append(containerNames, trimmed)
-			}
-
-			serviceShortName := service.ContainerToServiceName(container.Names, app.StackName())
-			serviceLongName := fmt.Sprintf("%s_%s", app.StackName(), serviceShortName)
-
-			tableRow := []string{
-				serviceLongName,
-				formatter.RemoveSha(container.Image),
-			}
-			table.Append(tableRow)
-		}
-
-		table.Render()
-
-		return nil
-	},
-}

cli/app/undeploy.go

@@ -1,26 +1,20 @@
 package app
 
 import (
-	"context"
+	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	stack "coopcloud.tech/abra/pkg/client/stack"
+	"coopcloud.tech/abra/pkg/config"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appUndeployCommand = cli.Command{
+var appUndeployCommand = &cli.Command{
-	Name:      "undeploy",
+	Name:    "undeploy",
-	Aliases:   []string{"un"},
+	Aliases: []string{"u"},
-	ArgsUsage: "<domain>",
+	Usage:   "Undeploy an app",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Undeploy an app",
 	Description: `
 This does not destroy any of the application data. However, you should remain
 vigilant, as your swarm installation will consider any previously attached
@@ -35,27 +29,38 @@ volumes as eligiblef or pruning once undeployed.
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", stackName)
 		}
 
-		if err := internal.DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
+		if err := DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
 			logrus.Fatal(err)
 		}
 
 		rmOpts := stack.Remove{Namespaces: []string{app.StackName()}}
-		if err := stack.RunRemove(context.Background(), cl, rmOpts); err != nil {
+		if err := stack.RunRemove(c.Context, cl, rmOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/upgrade.go

@@ -1,50 +1,42 @@
 package app
 
 import (
-	"context"
 	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
+	stack "coopcloud.tech/abra/pkg/client/stack"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appUpgradeCommand = cli.Command{
+var appUpgradeCommand = &cli.Command{
 	Name:      "upgrade",
-	Aliases:   []string{"up"},
+	Aliases:   []string{"u"},
 	Usage:     "Upgrade an app",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<app>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
-Upgrade an app. You can use it to choose and roll out a new upgrade to an
+This command supports upgrading an app. You can use it to choose and roll out a
-existing app.
+new upgrade to an existing app.
 
-This command specifically supports incrementing the version of running apps, as
+This command specifically supports changing the version of running apps, as
-opposed to "abra app deploy <domain>" which will not change the version of a
+opposed to "abra app deploy <app>" which will not change the version of a
 deployed app.
 
 You may pass "--force/-f" to upgrade to the same version again. This can be
 useful if the container runtime has gotten into a weird state.
 
 This action could be destructive, please ensure you have a copy of your app
-data beforehand.
+data beforehand - see "abra app backup <app>" for more.
 
 Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
 including unstaged changes and can be useful for live hacking and testing new
@@ -53,57 +45,37 @@ recipes.
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
-		conf := runtime.New()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if !internal.Chaos {
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
-			if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
 
-		r, err := recipe.Get(app.Recipe, conf)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("checking whether %s is already deployed", stackName)
-
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		catl, err := recipe.ReadRecipeCatalogue()
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if len(versions) == 0 && !internal.Chaos {
-			logrus.Fatalf("no published releases for %s in the recipe catalogue?", app.Recipe)
+			logrus.Fatalf("no versions available '%s' in recipe catalogue?", app.Type)
 		}
 
 		var availableUpgrades []string
-		if deployedVersion == "unknown" {
+		if deployedVersion == "" {
+			deployedVersion = "unknown"
 			availableUpgrades = versions
-			logrus.Warnf("failed to determine version of deployed %s", app.Name)
+			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
 		}
 
 		if deployedVersion != "unknown" && !internal.Chaos {
@@ -122,21 +94,19 @@ recipes.
 			}
 
 			if len(availableUpgrades) == 0 && !internal.Force {
-				logrus.Infof("no available upgrades, you're on latest (%s) ✌️", deployedVersion)
+				logrus.Fatal("no available upgrades, you're on latest")
-				return nil
+				availableUpgrades = versions
 			}
 		}
 
-		availableUpgrades = internal.ReverseStringList(availableUpgrades)
-
 		var chosenUpgrade string
 		if len(availableUpgrades) > 0 && !internal.Chaos {
-			if internal.Force || internal.NoInput {
+			if internal.Force {
 				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
-				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
+				logrus.Debugf("choosing '%s' as version to upgrade to", chosenUpgrade)
 			} else {
 				prompt := &survey.Select{
-					Message: fmt.Sprintf("Please select an upgrade (current version: %s):", deployedVersion),
+					Message: fmt.Sprintf("Please select an upgrade (current version: '%s'):", deployedVersion),
 					Options: availableUpgrades,
 				}
 				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
@@ -145,16 +115,8 @@ recipes.
 			}
 		}
 
-		// if release notes written after git tag published, read them before we
-		// check out the tag and then they'll appear to be missing. this covers
-		// when we obviously will forget to write release notes before publishing
-		releaseNotes, err := internal.GetReleaseNotes(app.Recipe, chosenUpgrade)
-		if err != nil {
-			return err
-		}
-
 		if !internal.Chaos {
-			if err := recipe.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
+			if err := recipe.EnsureVersion(app.Type, chosenUpgrade); err != nil {
 				logrus.Fatal(err)
 			}
 		}
@@ -162,13 +124,13 @@ recipes.
 		if internal.Chaos {
 			logrus.Warn("chaos mode engaged")
 			var err error
-			chosenUpgrade, err = recipe.ChaosVersion(app.Recipe)
+			chosenUpgrade, err = recipe.ChaosVersion(app.Type)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
 		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
@@ -177,7 +139,7 @@ recipes.
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -191,20 +153,27 @@ recipes.
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		config.ExposeAllEnv(stackName, compose, app.Env)
-		config.SetRecipeLabel(compose, stackName, app.Recipe)
-		config.SetChaosLabel(compose, stackName, internal.Chaos)
-		config.SetUpdateLabel(compose, stackName, app.Env)
 
-		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
+		if err := NewVersionOverview(app, deployedVersion, chosenUpgrade); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.StackName(), internal.DontWaitConverge); err != nil {
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/version.go

@@ -1,18 +1,18 @@
 package app
 
 import (
-	"context"
+	"fmt"
+	"strings"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/client/stack"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // getImagePath returns the image name
@@ -21,62 +21,53 @@ func getImagePath(image string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-
 	path := reference.Path(img)
-
+	if strings.Contains(path, "library") {
-	path = formatter.StripTagMeta(path)
+		path = strings.Split(path, "/")[1]
-
+	}
-	logrus.Debugf("parsed %s from %s", path, image)
+	logrus.Debugf("parsed '%s' from '%s'", path, image)
-
 	return path, nil
 }
 
-var appVersionCommand = cli.Command{
+var appVersionCommand = &cli.Command{
-	Name:      "version",
+	Name:    "version",
-	Aliases:   []string{"v"},
+	Aliases: []string{"v"},
-	ArgsUsage: "<domain>",
+	Usage:   "Show app versions",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Show app versions",
 	Description: `
-Show all information about versioning related to a deployed app. This includes
+This command shows all information about versioning related to a deployed app.
-the individual image names, tags and digests. But also the Co-op Cloud recipe
+This includes the individual image names, tags and digests. But also the Co-op
-version.
+Cloud recipe version.
 `,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
-		conf := runtime.New()
 
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("checking whether %s is already deployed", stackName)
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if deployedVersion == "unknown" {
+		if deployedVersion == "" {
-			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
+			logrus.Fatalf("failed to determine version of deployed '%s'", app.Name)
 		}
 
 		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, conf)
+		recipeMeta, err := catalogue.GetRecipeMeta(app.Type)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		versionsMeta := make(map[string]recipe.ServiceMeta)
+		versionsMeta := make(map[string]catalogue.ServiceMeta)
 		for _, recipeVersion := range recipeMeta.Versions {
 			if currentVersion, exists := recipeVersion[deployedVersion]; exists {
 				versionsMeta = currentVersion
@@ -84,20 +75,30 @@ version.
 		}
 
 		if len(versionsMeta) == 0 {
-			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
+			logrus.Fatalf("PANIC: could not retrieve deployed version ('%s') from recipe catalogue?", deployedVersion)
 		}
 
-		tableCol := []string{"version", "service", "image"}
+		tableCol := []string{"name", "image", "version", "tag", "digest"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
-		table.SetAutoMergeCellsByColumnIndex([]int{0})
 
 		for serviceName, versionMeta := range versionsMeta {
-			table.Append([]string{deployedVersion, serviceName, versionMeta.Image})
+			table.Append([]string{serviceName, versionMeta.Image, deployedVersion, versionMeta.Tag, versionMeta.Digest})
 		}
 
 		table.Render()
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }

cli/app/volume.go

@@ -1,125 +1,77 @@
 package app
 
 import (
-	"context"
+	"fmt"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/config"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var appVolumeListCommand = cli.Command{
+var appVolumeListCommand = &cli.Command{
-	Name:      "list",
+	Name:    "list",
-	Aliases:   []string{"ls"},
+	Usage:   "List volumes associated with an app",
-	ArgsUsage: "<domain>",
+	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List volumes associated with an app",
-	BashComplete: autocomplete.AppNameComplete,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
+		volumeList, err := client.GetVolumes(c.Context, app.Server, app.Name)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, true)
+		table := abraFormatter.CreateTable([]string{"driver", "volume name"})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		table := formatter.CreateTable([]string{"name", "created", "mounted"})
 		var volTable [][]string
 		for _, volume := range volumeList {
-			volRow := []string{volume.Name, volume.CreatedAt, volume.Mountpoint}
+			volRow := []string{
+				volume.Driver,
+				volume.Name,
+			}
 			volTable = append(volTable, volRow)
 		}
 
 		table.AppendBulk(volTable)
-
+		table.Render()
-		if table.NumLines() > 0 {
-			table.Render()
-		} else {
-			logrus.Warnf("no volumes created for %s", app.Name)
-		}
 
 		return nil
 	},
 }
 
-var appVolumeRemoveCommand = cli.Command{
+var appVolumeRemoveCommand = &cli.Command{
-	Name:  "remove",
+	Name:    "remove",
-	Usage: "Remove volume(s) associated with an app",
+	Usage:   "Remove volume(s) associated with an app",
-	Description: `
+	Aliases: []string{"rm"},
-This command supports removing volumes associated with an app. The app in
-question must be undeployed before you try to remove volumes. See "abra app
-undeploy <domain>" for more.
-
-The command is interactive and will show a multiple select input which allows
-you to make a seclection. Use the "?" key to see more help on navigating this
-interface.
-
-Passing "--force/-f" will select all volumes for removal. Be careful.
-`,
-	ArgsUsage: "<domain>",
-	Aliases:   []string{"rm"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		internal.ForceFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		cl, err := client.New(app.Server)
+		volumeList, err := client.GetVolumes(c.Context, app.Server, app.Name)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		filters, err := app.Filters(false, true)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		volumeNames := client.GetVolumeNames(volumeList)
 
 		var volumesToRemove []string
-		if !internal.Force && !internal.NoInput {
+		if !internal.Force {
 			volumesPrompt := &survey.MultiSelect{
 				Message: "which volumes do you want to remove?",
-				Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
-				VimMode: true,
 				Options: volumeNames,
 				Default: volumeNames,
 			}
 			if err := survey.AskOne(volumesPrompt, &volumesToRemove); err != nil {
 				logrus.Fatal(err)
 			}
-		}
+		} else {
-
-		if internal.Force || internal.NoInput {
 			volumesToRemove = volumeNames
 		}
 
-		err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
+		err = client.RemoveVolumes(c.Context, app.Server, volumesToRemove, internal.Force)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -128,15 +80,26 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 
 		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
 }
 
-var appVolumeCommand = cli.Command{
+var appVolumeCommand = &cli.Command{
 	Name:      "volume",
 	Aliases:   []string{"vl"},
 	Usage:     "Manage app volumes",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<command>",
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		appVolumeListCommand,
 		appVolumeRemoveCommand,
 	},

cli/catalogue/catalogue.go

@@ -1,223 +1,17 @@
 package catalogue
 
 import (
-	"encoding/json"
+	"github.com/urfave/cli/v2"
-	"fmt"
-	"io/ioutil"
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	gitPkg "coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 )
 
-var catalogueGenerateCommand = cli.Command{
-	Name:    "generate",
-	Aliases: []string{"g"},
-	Usage:   "Generate the recipe catalogue",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.PublishFlag,
-		internal.DryFlag,
-		internal.SkipUpdatesFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Generate a new copy of the recipe catalogue which can be found on:
-
-    https://recipes.coopcloud.tech (website that humans read)
-    https://recipes.coopcloud.tech/recipes.json (JSON that Abra reads)
-
-It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
-listing, parses README.md and git tags to produce recipe metadata which is
-loaded into the catalogue JSON file.
-
-It is possible to generate new metadata for a single recipe by passing
-<recipe>. The existing local catalogue will be updated, not overwritten.
-
-It is quite easy to get rate limited by Docker Hub when running this command.
-If you have a Hub account you can have Abra log you in to avoid this. Pass
-"--user" and "--pass".
-
-Push your new release to git.coopcloud.tech with "-p/--publish". This requires
-that you have permission to git push to these repositories and have your SSH
-keys configured on your account.
-`,
-	ArgsUsage: "[<recipe>]",
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-		if recipeName != "" {
-			internal.ValidateRecipe(c)
-		}
-
-		if err := catalogue.EnsureUpToDate(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		repos, err := recipe.ReadReposMetadata()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var barLength int
-		var logMsg string
-		if recipeName != "" {
-			barLength = 1
-			logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
-		} else {
-			barLength = len(repos)
-			logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
-		}
-
-		if !internal.SkipUpdates {
-			logrus.Warn(logMsg)
-			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		catl := make(recipe.RecipeCatalogue)
-		catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
-		for _, recipeMeta := range repos {
-			if recipeName != "" && recipeName != recipeMeta.Name {
-				catlBar.Add(1)
-				continue
-			}
-
-			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
-			versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
-			if err != nil {
-				logrus.Warn(err)
-			}
-
-			features, category, err := recipe.GetRecipeFeaturesAndCategory(recipeMeta.Name)
-			if err != nil {
-				logrus.Warn(err)
-			}
-
-			catl[recipeMeta.Name] = recipe.RecipeMeta{
-				Name:          recipeMeta.Name,
-				Repository:    recipeMeta.CloneURL,
-				SSHURL:        recipeMeta.SSHURL,
-				Icon:          recipeMeta.AvatarURL,
-				DefaultBranch: recipeMeta.DefaultBranch,
-				Description:   recipeMeta.Description,
-				Website:       recipeMeta.Website,
-				Versions:      versions,
-				Category:      category,
-				Features:      features,
-			}
-
-			catlBar.Add(1)
-		}
-
-		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if recipeName == "" {
-			if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			catlFS, err := recipe.ReadRecipeCatalogue()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			catlFS[recipeName] = catl[recipeName]
-
-			updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		logrus.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)
-
-		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
-		if internal.Publish {
-
-			isClean, err := gitPkg.IsClean(cataloguePath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if isClean {
-				if !internal.Dry {
-					logrus.Fatalf("no changes discovered in %s, nothing to publish?", cataloguePath)
-				}
-			}
-
-			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
-				logrus.Fatal(err)
-			}
-
-			repo, err := git.PlainOpen(cataloguePath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			sshURL := fmt.Sprintf(config.SSH_URL_TEMPLATE, config.CATALOGUE_JSON_REPO_NAME)
-			if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		repo, err := git.PlainOpen(cataloguePath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		head, err := repo.Head()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Dry && internal.Publish {
-			url := fmt.Sprintf("%s/%s/commit/%s", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME, head.Hash())
-			logrus.Infof("new changes published: %s", url)
-		}
-
-		if internal.Dry {
-			logrus.Info("dry run: no changes published")
-		}
-
-		return nil
-	},
-	BashComplete: autocomplete.RecipeNameComplete,
-}
-
 // CatalogueCommand defines the `abra catalogue` command and sub-commands.
-var CatalogueCommand = cli.Command{
+var CatalogueCommand = &cli.Command{
 	Name:        "catalogue",
-	Usage:       "Manage the recipe catalogue",
+	Usage:       "Manage the recipe catalogue (for maintainers)",
 	Aliases:     []string{"c"},
 	ArgsUsage:   "<recipe>",
 	Description: "This command helps recipe packagers interact with the recipe catalogue",
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		catalogueGenerateCommand,
 	},
 }

cli/catalogue/generate.go

@@ -0,0 +1,173 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                  true,
+	"abra-bash":             true,
+	"abra-apps":             true,
+	"abra-aur":              true,
+	"abra-capsul":           true,
+	"abra-gandi":            true,
+	"abra-hetzner":          true,
+	"apps":                  true,
+	"aur-abra-git":          true,
+	"auto-apps-json":        true,
+	"auto-mirror":           true,
+	"backup-bot":            true,
+	"coopcloud.tech":        true,
+	"coturn":                true,
+	"docker-cp-deploy":      true,
+	"docker-dind-bats-kcov": true,
+	"docs.coopcloud.tech":   true,
+	"example":               true,
+	"gardening":             true,
+	"go-abra":               true,
+	"organising":            true,
+	"pyabra":                true,
+	"radicle-seed-node":     true,
+	"stack-ssh-deploy":      true,
+	"swarm-cronjob":         true,
+	"tagcmp":                true,
+	"tyop":                  true,
+}
+
+var catalogueGenerateCommand = &cli.Command{
+	Name:      "generate",
+	Aliases:   []string{"g"},
+	Usage:     "Generate a new copy of the catalogue",
+	ArgsUsage: "[<recipe>]",
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		repos, err := catalogue.ReadReposMetadata()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
+
+		retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
+		ch := make(chan string, len(repos))
+		for _, repoMeta := range repos {
+			go func(rm catalogue.RepoMeta) {
+				if recipeName != "" && recipeName != rm.Name {
+					ch <- rm.Name
+					retrieveBar.Add(1)
+					return
+				}
+				if _, exists := CatalogueSkipList[rm.Name]; exists {
+					ch <- rm.Name
+					retrieveBar.Add(1)
+					return
+				}
+
+				recipeDir := path.Join(config.ABRA_DIR, "apps", rm.Name)
+
+				if err := git.Clone(recipeDir, rm.SSHURL); err != nil {
+					logrus.Fatal(err)
+				}
+
+				if err := git.EnsureUpToDate(recipeDir); err != nil {
+					logrus.Fatal(err)
+				}
+
+				ch <- rm.Name
+				retrieveBar.Add(1)
+			}(repoMeta)
+		}
+
+		for range repos {
+			<-ch // wait for everything
+		}
+
+		catl := make(catalogue.RecipeCatalogue)
+		catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...")
+		for _, recipeMeta := range repos {
+			if recipeName != "" && recipeName != recipeMeta.Name {
+				catlBar.Add(1)
+				continue
+			}
+
+			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
+				catlBar.Add(1)
+				continue
+			}
+
+			versions, err := catalogue.GetRecipeVersions(recipeMeta.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			catl[recipeMeta.Name] = catalogue.RecipeMeta{
+				Name:          recipeMeta.Name,
+				Repository:    recipeMeta.CloneURL,
+				Icon:          recipeMeta.AvatarURL,
+				DefaultBranch: recipeMeta.DefaultBranch,
+				Description:   recipeMeta.Description,
+				Website:       recipeMeta.Website,
+				Versions:      versions,
+				// Category:      ..., // FIXME: once we sort out the machine-readable catalogue interface
+				// Features:      ..., // FIXME: once we figure out the machine-readable catalogue interface
+			}
+			catlBar.Add(1)
+		}
+
+		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if _, err := os.Stat(config.APPS_JSON); err != nil && os.IsNotExist(err) {
+			if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if recipeName != "" {
+				catlFS, err := catalogue.ReadRecipeCatalogue()
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				catlFS[recipeName] = catl[recipeName]
+
+				updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if err := ioutil.WriteFile(config.APPS_JSON, updatedRecipesJSON, 0644); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		logrus.Infof("generated new recipe catalogue in '%s'", config.APPS_JSON)
+
+		return nil
+	},
+}

cli/cli.go

@@ -2,194 +2,110 @@
 package cli
 
 import (
-	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path"
 
 	"coopcloud.tech/abra/cli/app"
 	"coopcloud.tech/abra/cli/catalogue"
-	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/domain"
 	"coopcloud.tech/abra/cli/recipe"
-	"coopcloud.tech/abra/cli/record"
 	"coopcloud.tech/abra/cli/server"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/web"
+	logrusStack "github.com/Gurpartap/logrus-stack"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// AutoCompleteCommand helps people set up auto-complete in their shells
+// Verbose stores the variable from VerboseFlag.
-var AutoCompleteCommand = cli.Command{
+var Verbose bool
-	Name:    "autocomplete",
-	Aliases: []string{"ac"},
-	Usage:   "Configure shell autocompletion (recommended)",
-	Description: `
-Set up auto-completion in your shell by downloading the relevant files and
-laying out what additional information must be loaded. Supported shells are as
-follows: bash, fish, fizsh & zsh.
 
-Example:
+// VerboseFlag turns on/off verbose logging down to the INFO level.
-
+var VerboseFlag = &cli.BoolFlag{
-    abra autocomplete bash
+	Name:        "verbose",
-`,
+	Aliases:     []string{"V"},
-	ArgsUsage: "<shell>",
+	Value:       false,
-	Flags: []cli.Flag{
+	Destination: &Verbose,
-		internal.DebugFlag,
+	Usage:       "Show INFO messages",
-	},
-	Action: func(c *cli.Context) error {
-		shellType := c.Args().First()
-
-		if shellType == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
-		}
-
-		supportedShells := map[string]bool{
-			"bash":  true,
-			"zsh":   true,
-			"fizsh": true,
-			"fish":  true,
-		}
-
-		if _, ok := supportedShells[shellType]; !ok {
-			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
-		}
-
-		if shellType == "fizsh" {
-			shellType = "zsh" // handled the same on the autocompletion side
-		}
-
-		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
-		if err := os.Mkdir(autocompletionDir, 0764); err != nil {
-			if !os.IsExist(err) {
-				logrus.Fatal(err)
-			}
-			logrus.Debugf("%s already created", autocompletionDir)
-		}
-
-		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
-		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
-			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
-			logrus.Infof("fetching %s", url)
-			if err := web.GetFile(autocompletionFile, url); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		switch shellType {
-		case "bash":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install auto-completion
-sudo mkdir /etc/bash_completion.d/
-sudo cp %s /etc/bash_completion.d/abra
-echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
-# To test, run the following: "abra app <hit tab key>" - you should see command completion!
-`, autocompletionFile))
-		case "zsh":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install auto-completion
-sudo mkdir /etc/zsh/completion.d/
-sudo cp %s /etc/zsh/completion.d/abra
-echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
-# To test, run the following: "abra app <hit tab key>" - you should see command completion!
-`, autocompletionFile))
-		case "fish":
-			fmt.Println(fmt.Sprintf(`
-# Run the following commands to install auto-completion
-sudo mkdir -p /etc/fish/completions
-sudo cp %s /etc/fish/completions/abra
-echo "source /etc/fish/completions/abra" >> ~/.config/fish/config.fish
-# To test, run the following: "abra app <hit tab key>" - you should see command completion!
-`, autocompletionFile))
-		}
-
-		return nil
-	},
 }
 
-// UpgradeCommand upgrades abra in-place.
+// Debug stores the variable from DebugFlag.
-var UpgradeCommand = cli.Command{
+var Debug bool
-	Name:    "upgrade",
-	Aliases: []string{"u"},
-	Usage:   "Upgrade Abra itself",
-	Description: `
-Upgrade Abra in-place with the latest stable or release candidate.
 
-Pass "-r/--rc" to install the latest release candidate. Please bear in mind
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
-that it may contain catastrophic bugs. Thank you very much for the testing
+var DebugFlag = &cli.BoolFlag{
-efforts!
+	Name:        "debug",
-`,
+	Aliases:     []string{"d"},
-	Flags: []cli.Flag{internal.RCFlag},
+	Value:       false,
-	Action: func(c *cli.Context) error {
+	Destination: &Debug,
-		mainURL := "https://install.abra.coopcloud.tech"
+	Usage:       "Show DEBUG messages",
-		cmd := exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash", mainURL))
-
-		if internal.RC {
-			releaseCandidateURL := "https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer"
-			cmd = exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash -s -- --rc", releaseCandidateURL))
-		}
-
-		logrus.Debugf("attempting to run %s", cmd)
-
-		if err := internal.RunCmd(cmd); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
 }
 
 func newAbraApp(version, commit string) *cli.App {
 	app := &cli.App{
 		Name: "abra",
 		Usage: `The Co-op Cloud command-line utility belt 🎩🐇
+
     ____                           ____ _                 _
    / ___|___         ___  _ __    / ___| | ___  _   _  __| |
   | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
   | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
    \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
                          |_|
-`,
+		`,
 		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
-		Commands: []cli.Command{
+		Commands: []*cli.Command{
 			app.AppCommand,
 			server.ServerCommand,
 			recipe.RecipeCommand,
 			catalogue.CatalogueCommand,
-			record.RecordCommand,
+			domain.DomainCommand,
 			UpgradeCommand,
-			AutoCompleteCommand,
 		},
-		BashComplete: autocomplete.SubcommandComplete,
+		Flags: []cli.Flag{
+			VerboseFlag,
+			DebugFlag,
+		},
+		Authors: []*cli.Author{
+			{
+				Name:  "Autonomic Co-op",
+				Email: "helo@autonomic.zone",
+			},
+		},
 	}
 
 	app.EnableBashCompletion = true
 
 	app.Before = func(c *cli.Context) error {
+		if Debug {
+			logrus.SetLevel(logrus.DebugLevel)
+			logrus.SetFormatter(&logrus.TextFormatter{})
+			logrus.SetOutput(os.Stderr)
+			logrus.AddHook(logrusStack.StandardHook())
+		}
+
 		paths := []string{
 			config.ABRA_DIR,
-			path.Join(config.SERVERS_DIR),
+			path.Join(config.ABRA_DIR, "servers"),
-			path.Join(config.RECIPES_DIR),
+			path.Join(config.ABRA_DIR, "apps"),
-			path.Join(config.VENDOR_DIR),
+			path.Join(config.ABRA_DIR, "vendor"),
-			path.Join(config.BACKUP_DIR),
 		}
 
 		for _, path := range paths {
-			if err := os.Mkdir(path, 0764); err != nil {
+			if err := os.Mkdir(path, 0755); err != nil {
 				if !os.IsExist(err) {
 					logrus.Fatal(err)
 				}
+				logrus.Debugf("'%s' already created, moving on...", path)
 				continue
 			}
+			logrus.Debugf("'%s' is missing, creating...", path)
 		}
 
-		logrus.Debugf("abra version %s, commit %s", version, commit)
+		logrus.Debugf("abra version '%s', commit '%s'", version, commit)
 
 		return nil
 	}
-
 	return app
 }
 

cli/domain/create.go

@@ -0,0 +1,150 @@
+package domain
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"github.com/libdns/gandi"
+	"github.com/libdns/libdns"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// DomainCreateCommand lists domains.
+var DomainCreateCommand = &cli.Command{
+	Name:      "create",
+	Usage:     "Create a new domain record",
+	Aliases:   []string{"c"},
+	ArgsUsage: "<zone> <type> <name> <value> [<ttl>] [<priority>]",
+	Flags: []cli.Flag{
+		internal.DNSProviderFlag,
+	},
+	Description: `
+This command creates a new domain name record for a specific zone.
+
+You must specify a zone (e.g. example.com) under which your domain name records
+are listed. This zone must already be created on your provider account.
+
+Example:
+
+    abra domain create -p gandi foo.com A myapp 192.168.178.44
+
+Which means you can then deploy an app against "myapp.foo.com" successfully.
+`,
+	Action: func(c *cli.Context) error {
+		zone := c.Args().First()
+		if zone == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no zone provided"))
+		}
+
+		recordType := c.Args().Get(1)
+		recordName := c.Args().Get(2)
+		recordValue := c.Args().Get(3)
+		recordTTL := c.Args().Get(4)
+		recordPriority := c.Args().Get(5)
+
+		if recordType == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no type provided"))
+		}
+		if recordName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no name provided"))
+		}
+		if recordValue == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no value provided"))
+		}
+
+		if recordTTL == "" {
+			recordTTL = "86400"
+		}
+
+		if recordType == "MX" || recordType == "SRV" || recordType == "URI" {
+			if recordPriority == "" {
+				logrus.Fatal("record priority must be set when using MX/SRV/URI records")
+			}
+		}
+
+		var err error
+		var provider gandi.Provider
+		switch internal.DNSProvider {
+		case "gandi":
+			provider, err = gandiPkg.New()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		default:
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+		}
+
+		ttl, err := strconv.Atoi(recordTTL)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		record := libdns.Record{
+			Type:  recordType,
+			Name:  recordName,
+			Value: recordValue,
+			TTL:   time.Duration(ttl),
+		}
+
+		if recordType == "MX" || recordType == "SRV" || recordType == "URI" {
+			priority, err := strconv.Atoi(recordPriority)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			record.Priority = priority
+		}
+
+		records, err := provider.GetRecords(c.Context, zone)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, existingRecord := range records {
+			if existingRecord.Type == record.Type &&
+				existingRecord.Name == record.Name &&
+				existingRecord.Value == record.Value {
+				logrus.Fatal("provider library reports that this record already exists?")
+			}
+		}
+
+		createdRecords, err := provider.SetRecords(
+			c.Context,
+			zone,
+			[]libdns.Record{record},
+		)
+
+		if len(createdRecords) == 0 {
+			logrus.Fatal("provider library reports no record created?")
+		}
+
+		createdRecord := createdRecords[0]
+
+		tableCol := []string{"type", "name", "value", "TTL", "priority"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		value := createdRecord.Value
+		if len(createdRecord.Value) > 30 {
+			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
+		}
+
+		table.Append([]string{
+			createdRecord.Type,
+			createdRecord.Name,
+			value,
+			createdRecord.TTL.String(),
+			strconv.Itoa(createdRecord.Priority),
+		})
+
+		table.Render()
+
+		logrus.Info("record created")
+
+		return nil
+	},
+}

cli/domain/domain.go

@@ -0,0 +1,38 @@
+package domain
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// DomainCommand supports managing DNS entries.
+var DomainCommand = &cli.Command{
+	Name:      "domain",
+	Usage:     "Manage domains via 3rd party providers",
+	Aliases:   []string{"d"},
+	ArgsUsage: "<domain>",
+	Description: `
+This command supports managing domain name records via 3rd party providers such
+as Gandi DNS. It supports listing, creating and removing all types of records
+that you might need for managing Co-op Cloud apps.
+
+The following providers are supported:
+
+    Gandi DNS https://www.gandi.net
+
+You need an account with such a provider already. Typically, you need to
+provide an API token on the Abra command-line when using these commands so that
+you can authenticate with your provider account.
+
+Any new provider can be integrated, we welcome change sets. See the underlying
+DNS library documentation for more. It supports many existing providers and
+allows to implement new provider support easily.
+
+    https://pkg.go.dev/github.com/libdns/libdns
+
+`,
+	Subcommands: []*cli.Command{
+		DomainListCommand,
+		DomainCreateCommand,
+		DomainRemoveCommand,
+	},
+}

cli/domain/list.go

@@ -1,46 +1,40 @@
-package record
+package domain
 
 import (
-	"context"
+	"errors"
 	"fmt"
 	"strconv"
 
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/libdns/gandi"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// RecordListCommand lists domains.
+// DomainListCommand lists domains.
-var RecordListCommand = cli.Command{
+var DomainListCommand = &cli.Command{
 	Name:      "list",
-	Usage:     "List domain name records",
+	Usage:     "List domain name records for a zone",
 	Aliases:   []string{"ls"},
 	ArgsUsage: "<zone>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
 		internal.DNSProviderFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Description: `
-List all domain name records managed by a 3rd party provider for a specific
+This command lists all domain name records managed by a 3rd party provider.
-zone.
 
 You must specify a zone (e.g. example.com) under which your domain name records
 are listed. This zone must already be created on your provider account.
 `,
 	Action: func(c *cli.Context) error {
-		if err := internal.EnsureDNSProvider(); err != nil {
+		zone := c.Args().First()
-			logrus.Fatal(err)
+		if zone == "" {
-		}
+			internal.ShowSubcommandHelpAndError(c, errors.New("no zone provided"))
-
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
 		}
 
+		var err error
 		var provider gandi.Provider
 		switch internal.DNSProvider {
 		case "gandi":
@@ -49,16 +43,16 @@ are listed. This zone must already be created on your provider account.
 				logrus.Fatal(err)
 			}
 		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
 		}
 
-		records, err := provider.GetRecords(context.Background(), zone)
+		records, err := provider.GetRecords(c.Context, zone)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
+		table := abraFormatter.CreateTable(tableCol)
 
 		for _, record := range records {
 			value := record.Value

cli/domain/remove.go

@@ -0,0 +1,127 @@
+package domain
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/libdns/gandi"
+	"github.com/libdns/libdns"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// DomainRemoveCommand lists domains.
+var DomainRemoveCommand = &cli.Command{
+	Name:      "remove",
+	Usage:     "Remove domain name records for a zone",
+	Aliases:   []string{"rm"},
+	ArgsUsage: "<zone> <type> <name>",
+	Flags: []cli.Flag{
+		internal.DNSProviderFlag,
+	},
+	Description: `
+This command removes a new domain name record for a specific zone.
+
+You must specify a zone (e.g. example.com) under which your domain name records
+are listed. This zone must already be created on your provider account.
+
+The "<id>" can be retrieved by running "abra domain list <zone>". This can be
+used then in this command for deletion of the record.
+
+Example:
+
+    abra domain remove -p gandi foo.com A myapp
+
+Which means that the domain name record of type A for myapp.foo.com will be deleted.
+`,
+	Action: func(c *cli.Context) error {
+		zone := c.Args().First()
+		if zone == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no zone provided"))
+		}
+
+		recordType := c.Args().Get(1)
+		recordName := c.Args().Get(2)
+
+		if recordType == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no type provided"))
+		}
+		if recordName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no name provided"))
+		}
+
+		var err error
+		var provider gandi.Provider
+		switch internal.DNSProvider {
+		case "gandi":
+			provider, err = gandiPkg.New()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		default:
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+		}
+
+		records, err := provider.GetRecords(c.Context, zone)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var toDelete libdns.Record
+		for _, record := range records {
+			if record.Type == recordType && record.Name == recordName {
+				toDelete = record
+				break
+			}
+		}
+
+		if (libdns.Record{}) == toDelete {
+			logrus.Fatal("provider library reports no matching record?")
+		}
+
+		tableCol := []string{"type", "name", "value", "TTL", "priority"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		value := toDelete.Value
+		if len(toDelete.Value) > 30 {
+			value = fmt.Sprintf("%s...", toDelete.Value[:30])
+		}
+
+		table.Append([]string{
+			toDelete.Type,
+			toDelete.Name,
+			value,
+			toDelete.TTL.String(),
+			strconv.Itoa(toDelete.Priority),
+		})
+
+		table.Render()
+
+		response := false
+		prompt := &survey.Confirm{
+			Message: "continue with record deletion?",
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return err
+		}
+
+		if !response {
+			logrus.Fatal("exiting as requested")
+		}
+
+		_, err = provider.DeleteRecords(c.Context, zone, []libdns.Record{toDelete})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Info("record successfully deleted")
+
+		return nil
+	},
+}

cli/formatter/formatter.go

@@ -1,23 +1,23 @@
 package formatter
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/docker/cli/cli/command/formatter"
 	"github.com/docker/go-units"
-	// "github.com/olekukonko/tablewriter"
+	"github.com/olekukonko/tablewriter"
-	"coopcloud.tech/abra/pkg/jsontable"
 	"github.com/schollz/progressbar/v3"
-	"github.com/sirupsen/logrus"
 )
 
 func ShortenID(str string) string {
 	return str[:12]
 }
 
-func SmallSHA(hash string) string {
+func Truncate(str string) string {
-	return hash[:8]
+	return fmt.Sprintf(`"%s"`, formatter.Ellipsis(str, 19))
 }
 
 // RemoveSha remove image sha from a string that are added in some docker outputs
@@ -33,9 +33,8 @@ func HumanDuration(timestamp int64) string {
 }
 
 // CreateTable prepares a table layout for output.
-func CreateTable(columns []string) *jsontable.JSONTable {
+func CreateTable(columns []string) *tablewriter.Table {
-	table := jsontable.NewJSONTable(os.Stdout)
+	table := tablewriter.NewWriter(os.Stdout)
-	table.SetAutoWrapText(false)
 	table.SetHeader(columns)
 	return table
 }
@@ -51,22 +50,3 @@ func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
 		progressbar.OptionSetDescription(title),
 	)
 }
-
-// StripTagMeta strips front-matter image tag data that we don't need for parsing.
-func StripTagMeta(image string) string {
-	originalImage := image
-
-	if strings.Contains(image, "docker.io") {
-		image = strings.Split(image, "/")[1]
-	}
-
-	if strings.Contains(image, "library") {
-		image = strings.Split(image, "/")[1]
-	}
-
-	if originalImage != image {
-		logrus.Debugf("stripped %s to %s for parsing", originalImage, image)
-	}
-
-	return image
-}

cli/internal/backup.go

@@ -1,35 +0,0 @@
-package internal
-
-import (
-	"strings"
-)
-
-// SafeSplit splits up a string into a list of commands safely.
-func SafeSplit(s string) []string {
-	split := strings.Split(s, " ")
-
-	var result []string
-	var inquote string
-	var block string
-	for _, i := range split {
-		if inquote == "" {
-			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
-				inquote = string(i[0])
-				block = strings.TrimPrefix(i, inquote) + " "
-			} else {
-				result = append(result, i)
-			}
-		} else {
-			if !strings.HasSuffix(i, inquote) {
-				block += i + " "
-			} else {
-				block += strings.TrimSuffix(i, inquote)
-				inquote = ""
-				result = append(result, block)
-				block = ""
-			}
-		}
-	}
-
-	return result
-}

cli/internal/cli.go

@@ -1,399 +0,0 @@
-package internal
-
-import (
-	"os"
-
-	logrusStack "github.com/Gurpartap/logrus-stack"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// Secrets stores the variable from SecretsFlag
-var Secrets bool
-
-// SecretsFlag turns on/off automatically generating secrets
-var SecretsFlag = &cli.BoolFlag{
-	Name:        "secrets, S",
-	Usage:       "Automatically generate secrets",
-	Destination: &Secrets,
-}
-
-// Pass stores the variable from PassFlag
-var Pass bool
-
-// PassFlag turns on/off storing generated secrets in pass
-var PassFlag = &cli.BoolFlag{
-	Name:        "pass, p",
-	Usage:       "Store the generated secrets in a local pass store",
-	Destination: &Pass,
-}
-
-// PassRemove stores the variable for PassRemoveFlag
-var PassRemove bool
-
-// PassRemoveFlag turns on/off removing generated secrets from pass
-var PassRemoveFlag = &cli.BoolFlag{
-	Name:        "pass, p",
-	Usage:       "Remove generated secrets from a local pass store",
-	Destination: &PassRemove,
-}
-
-// Force force functionality without asking.
-var Force bool
-
-// ForceFlag turns on/off force functionality.
-var ForceFlag = &cli.BoolFlag{
-	Name:        "force, f",
-	Usage:       "Perform action without further prompt. Use with care!",
-	Destination: &Force,
-}
-
-// Chaos engages chaos mode.
-var Chaos bool
-
-// ChaosFlag turns on/off chaos functionality.
-var ChaosFlag = &cli.BoolFlag{
-	Name:        "chaos, C",
-	Usage:       "Deploy uncommitted recipes changes. Use with care!",
-	Destination: &Chaos,
-}
-
-// DNSProvider specifies a DNS provider.
-var DNSProvider string
-
-// DNSProviderFlag selects a DNS provider.
-var DNSProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Value:       "",
-	Usage:       "DNS provider",
-	Destination: &DNSProvider,
-}
-
-var NoInput bool
-var NoInputFlag = &cli.BoolFlag{
-	Name:        "no-input, n",
-	Usage:       "Toggle non-interactive mode",
-	Destination: &NoInput,
-}
-
-var DNSType string
-
-var DNSTypeFlag = &cli.StringFlag{
-	Name:        "record-type, rt",
-	Value:       "",
-	Usage:       "Domain name record type (e.g. A)",
-	Destination: &DNSType,
-}
-
-var DNSName string
-
-var DNSNameFlag = &cli.StringFlag{
-	Name:        "record-name, rn",
-	Value:       "",
-	Usage:       "Domain name record name (e.g. mysubdomain)",
-	Destination: &DNSName,
-}
-
-var DNSValue string
-
-var DNSValueFlag = &cli.StringFlag{
-	Name:        "record-value, rv",
-	Value:       "",
-	Usage:       "Domain name record value (e.g. 192.168.1.1)",
-	Destination: &DNSValue,
-}
-
-var DNSTTL string
-var DNSTTLFlag = &cli.StringFlag{
-	Name:        "record-ttl, rl",
-	Value:       "600s",
-	Usage:       "Domain name TTL value (seconds)",
-	Destination: &DNSTTL,
-}
-
-var DNSPriority int
-
-var DNSPriorityFlag = &cli.IntFlag{
-	Name:        "record-priority, rp",
-	Value:       10,
-	Usage:       "Domain name priority value",
-	Destination: &DNSPriority,
-}
-
-var ServerProvider string
-
-var ServerProviderFlag = &cli.StringFlag{
-	Name:        "provider, p",
-	Usage:       "3rd party server provider",
-	Destination: &ServerProvider,
-}
-
-var CapsulInstanceURL string
-
-var CapsulInstanceURLFlag = &cli.StringFlag{
-	Name:        "capsul-url, cu",
-	Value:       "yolo.servers.coop",
-	Usage:       "capsul instance URL",
-	Destination: &CapsulInstanceURL,
-}
-
-var CapsulName string
-
-var CapsulNameFlag = &cli.StringFlag{
-	Name:        "capsul-name, cn",
-	Value:       "",
-	Usage:       "capsul name",
-	Destination: &CapsulName,
-}
-
-var CapsulType string
-
-var CapsulTypeFlag = &cli.StringFlag{
-	Name:        "capsul-type, ct",
-	Value:       "f1-xs",
-	Usage:       "capsul type",
-	Destination: &CapsulType,
-}
-
-var CapsulImage string
-
-var CapsulImageFlag = &cli.StringFlag{
-	Name:        "capsul-image, ci",
-	Value:       "debian10",
-	Usage:       "capsul image",
-	Destination: &CapsulImage,
-}
-
-var CapsulSSHKeys cli.StringSlice
-var CapsulSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "capsul-ssh-keys, cs",
-	Usage: "capsul SSH key",
-	Value: &CapsulSSHKeys,
-}
-
-var CapsulAPIToken string
-
-var CapsulAPITokenFlag = &cli.StringFlag{
-	Name:        "capsul-token, ca",
-	Usage:       "capsul API token",
-	EnvVar:      "CAPSUL_TOKEN",
-	Destination: &CapsulAPIToken,
-}
-
-var HetznerCloudName string
-
-var HetznerCloudNameFlag = &cli.StringFlag{
-	Name:        "hetzner-name, hn",
-	Value:       "",
-	Usage:       "hetzner cloud name",
-	Destination: &HetznerCloudName,
-}
-
-var HetznerCloudType string
-
-var HetznerCloudTypeFlag = &cli.StringFlag{
-	Name:        "hetzner-type, ht",
-	Usage:       "hetzner cloud type",
-	Destination: &HetznerCloudType,
-	Value:       "cx11",
-}
-
-var HetznerCloudImage string
-
-var HetznerCloudImageFlag = &cli.StringFlag{
-	Name:        "hetzner-image, hi",
-	Usage:       "hetzner cloud image",
-	Value:       "debian-10",
-	Destination: &HetznerCloudImage,
-}
-
-var HetznerCloudSSHKeys cli.StringSlice
-
-var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
-	Name:  "hetzner-ssh-keys, hs",
-	Usage: "hetzner cloud SSH keys (e.g. me@foo.com)",
-	Value: &HetznerCloudSSHKeys,
-}
-
-var HetznerCloudLocation string
-
-var HetznerCloudLocationFlag = &cli.StringFlag{
-	Name:        "hetzner-location, hl",
-	Usage:       "hetzner cloud server location",
-	Value:       "hel1",
-	Destination: &HetznerCloudLocation,
-}
-
-var HetznerCloudAPIToken string
-
-var HetznerCloudAPITokenFlag = &cli.StringFlag{
-	Name:        "hetzner-token, ha",
-	Usage:       "hetzner cloud API token",
-	EnvVar:      "HCLOUD_TOKEN",
-	Destination: &HetznerCloudAPIToken,
-}
-
-// Debug stores the variable from DebugFlag.
-var Debug bool
-
-// DebugFlag turns on/off verbose logging down to the DEBUG level.
-var DebugFlag = &cli.BoolFlag{
-	Name:        "debug, d",
-	Destination: &Debug,
-	Usage:       "Show DEBUG messages",
-}
-
-// MachineReadable stores the variable from MachineReadableFlag
-var MachineReadable bool
-
-// MachineReadableFlag turns on/off machine readable output where supported
-var MachineReadableFlag = &cli.BoolFlag{
-	Name:        "machine, m",
-	Destination: &MachineReadable,
-	Usage:       "Output in a machine-readable format (where supported)",
-}
-
-// RC signifies the latest release candidate
-var RC bool
-
-// RCFlag chooses the latest release candidate for install
-var RCFlag = &cli.BoolFlag{
-	Name:        "rc, r",
-	Destination: &RC,
-	Usage:       "Insatll the latest release candidate",
-}
-
-var Major bool
-var MajorFlag = &cli.BoolFlag{
-	Name:        "major, x",
-	Usage:       "Increase the major part of the version",
-	Destination: &Major,
-}
-
-var Minor bool
-var MinorFlag = &cli.BoolFlag{
-	Name:        "minor, y",
-	Usage:       "Increase the minor part of the version",
-	Destination: &Minor,
-}
-
-var Patch bool
-var PatchFlag = &cli.BoolFlag{
-	Name:        "patch, z",
-	Usage:       "Increase the patch part of the version",
-	Destination: &Patch,
-}
-
-var Dry bool
-var DryFlag = &cli.BoolFlag{
-	Name:        "dry-run, r",
-	Usage:       "Only reports changes that would be made",
-	Destination: &Dry,
-}
-
-var Publish bool
-var PublishFlag = &cli.BoolFlag{
-	Name:        "publish, p",
-	Usage:       "Publish changes to git.coopcloud.tech",
-	Destination: &Publish,
-}
-
-var Domain string
-var DomainFlag = &cli.StringFlag{
-	Name:        "domain, D",
-	Value:       "",
-	Usage:       "Choose a domain name",
-	Destination: &Domain,
-}
-
-var NewAppServer string
-var NewAppServerFlag = &cli.StringFlag{
-	Name:        "server, s",
-	Value:       "",
-	Usage:       "Show apps of a specific server",
-	Destination: &NewAppServer,
-}
-
-var NoDomainChecks bool
-var NoDomainChecksFlag = &cli.BoolFlag{
-	Name:        "no-domain-checks, D",
-	Usage:       "Disable app domain sanity checks",
-	Destination: &NoDomainChecks,
-}
-
-var StdErrOnly bool
-var StdErrOnlyFlag = &cli.BoolFlag{
-	Name:        "stderr, s",
-	Usage:       "Only tail stderr",
-	Destination: &StdErrOnly,
-}
-
-var SinceLogs string
-var SinceLogsFlag = &cli.StringFlag{
-	Name:        "since, S",
-	Value:       "",
-	Usage:       "tail logs since YYYY-MM-DDTHH:MM:SSZ",
-	Destination: &SinceLogs,
-}
-
-var DontWaitConverge bool
-var DontWaitConvergeFlag = &cli.BoolFlag{
-	Name:        "no-converge-checks, c",
-	Usage:       "Don't wait for converge logic checks",
-	Destination: &DontWaitConverge,
-}
-
-var Watch bool
-var WatchFlag = &cli.BoolFlag{
-	Name:        "watch, w",
-	Usage:       "Watch status by polling repeatedly",
-	Destination: &Watch,
-}
-
-var OnlyErrors bool
-var OnlyErrorFlag = &cli.BoolFlag{
-	Name:        "errors, e",
-	Usage:       "Only show errors",
-	Destination: &OnlyErrors,
-}
-
-var SkipUpdates bool
-var SkipUpdatesFlag = &cli.BoolFlag{
-	Name:        "skip-updates, s",
-	Usage:       "Skip updating recipe repositories",
-	Destination: &SkipUpdates,
-}
-
-var AllTags bool
-var AllTagsFlag = &cli.BoolFlag{
-	Name:        "all-tags, a",
-	Usage:       "List all tags, not just upgrades",
-	Destination: &AllTags,
-}
-
-var LocalCmd bool
-var LocalCmdFlag = &cli.BoolFlag{
-	Name:        "local, l",
-	Usage:       "Run command locally",
-	Destination: &LocalCmd,
-}
-
-var RemoteUser string
-var RemoteUserFlag = &cli.StringFlag{
-	Name:        "user, u",
-	Value:       "",
-	Usage:       "User to run command within a service context",
-	Destination: &RemoteUser,
-}
-
-// SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
-func SubCommandBefore(c *cli.Context) error {
-	if Debug {
-		logrus.SetLevel(logrus.DebugLevel)
-		logrus.SetFormatter(&logrus.TextFormatter{})
-		logrus.SetOutput(os.Stderr)
-		logrus.AddHook(logrusStack.StandardHook())
-	}
-
-	return nil
-}

cli/internal/common.go

@@ -0,0 +1,76 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Secrets stores the variable from SecretsFlag
+var Secrets bool
+
+// SecretsFlag turns on/off automatically generating secrets
+var SecretsFlag = &cli.BoolFlag{
+	Name:        "secrets",
+	Aliases:     []string{"S"},
+	Value:       false,
+	Usage:       "Automatically generate secrets",
+	Destination: &Secrets,
+}
+
+// Pass stores the variable from PassFlag
+var Pass bool
+
+// PassFlag turns on/off storing generated secrets in pass
+var PassFlag = &cli.BoolFlag{
+	Name:        "pass",
+	Aliases:     []string{"P"},
+	Value:       false,
+	Usage:       "Store the generated secrets in a local pass store",
+	Destination: &Pass,
+}
+
+// Context is temp
+var Context string
+
+// ContextFlag is temp
+var ContextFlag = &cli.StringFlag{
+	Name:        "context",
+	Value:       "",
+	Aliases:     []string{"c"},
+	Destination: &Context,
+}
+
+// Force force functionality without asking.
+var Force bool
+
+// ForceFlag turns on/off force functionality.
+var ForceFlag = &cli.BoolFlag{
+	Name:        "force",
+	Value:       false,
+	Aliases:     []string{"f"},
+	Destination: &Force,
+}
+
+// Chaos engages chaos mode.
+var Chaos bool
+
+// ChaosFlag turns on/off chaos functionality.
+var ChaosFlag = &cli.BoolFlag{
+	Name:        "chaos",
+	Value:       false,
+	Aliases:     []string{"ch"},
+	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Destination: &Chaos,
+}
+
+// DNSProvider specifies a DNS provider.
+var DNSProvider string
+
+// DNSProviderFlag selects a DNS provider.
+var DNSProviderFlag = &cli.StringFlag{
+	Name:        "provider",
+	Value:       "",
+	Aliases:     []string{"p"},
+	Usage:       "DNS provider",
+	Destination: &DNSProvider,
+	Required:    true,
+}

cli/internal/deploy.go

@@ -1,276 +0,0 @@
-package internal
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
-
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/dns"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/git"
-	"coopcloud.tech/abra/pkg/lint"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// DeployAction is the main command-line action for this package
-func DeployAction(c *cli.Context) error {
-	app := ValidateApp(c)
-	conf := runtime.New()
-
-	cl, err := client.New(app.Server)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if !Chaos {
-		if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	r, err := recipe.Get(app.Recipe, conf)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := lint.LintForErrors(r); err != nil {
-		logrus.Fatal(err)
-	}
-
-	logrus.Debugf("checking whether %s is already deployed", app.StackName())
-
-	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if isDeployed {
-		if Force || Chaos {
-			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
-		} else {
-			logrus.Fatalf("%s is already deployed", app.Name)
-		}
-	}
-
-	version := deployedVersion
-	if version == "unknown" && !Chaos {
-		catl, err := recipe.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if len(versions) > 0 {
-			version = versions[len(versions)-1]
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			head, err := git.GetRecipeHead(app.Recipe)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			version = formatter.SmallSHA(head.String())
-			logrus.Warn("no versions detected, using latest commit")
-			if err := recipe.EnsureLatest(app.Recipe, conf); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-	}
-
-	if version == "unknown" && !Chaos {
-		logrus.Debugf("choosing %s as version to deploy", version)
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	if version != "unknown" && !Chaos {
-		if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	if Chaos {
-		logrus.Warnf("chaos mode engaged")
-		var err error
-		version, err = recipe.ChaosVersion(app.Recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	for k, v := range abraShEnv {
-		app.Env[k] = v
-	}
-
-	composeFiles, err := config.GetAppComposeFiles(app.Recipe, app.Env)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	deployOpts := stack.Deploy{
-		Composefiles: composeFiles,
-		Namespace:    app.StackName(),
-		Prune:        false,
-		ResolveImage: stack.ResolveImageAlways,
-	}
-	compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	config.ExposeAllEnv(app.StackName(), compose, app.Env)
-	config.SetRecipeLabel(compose, app.StackName(), app.Recipe)
-	config.SetChaosLabel(compose, app.StackName(), Chaos)
-	config.SetUpdateLabel(compose, app.StackName(), app.Env)
-
-	if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if !NoDomainChecks {
-		domainName, ok := app.Env["DOMAIN"]
-		if ok {
-			if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
-		}
-	} else {
-		logrus.Warn("skipping domain checks as requested")
-	}
-
-	if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, DontWaitConverge); err != nil {
-		logrus.Fatal(err)
-	}
-
-	return nil
-}
-
-// DeployOverview shows a deployment overview
-func DeployOverview(app config.App, version, message string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "version"}
-	table := formatter.CreateTable(tableCol)
-
-	deployConfig := "compose.yml"
-	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
-		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
-	}
-
-	server := app.Server
-	if app.Server == "default" {
-		server = "local"
-	}
-
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, version})
-	table.Render()
-
-	if NoInput {
-		return nil
-	}
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: message,
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	return nil
-}
-
-// NewVersionOverview shows an upgrade or downgrade overview
-func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
-	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
-	table := formatter.CreateTable(tableCol)
-
-	deployConfig := "compose.yml"
-	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
-		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
-	}
-
-	server := app.Server
-	if app.Server == "default" {
-		server = "local"
-	}
-
-	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
-	table.Render()
-
-	if releaseNotes == "" {
-		var err error
-		releaseNotes, err = GetReleaseNotes(app.Recipe, newVersion)
-		if err != nil {
-			return err
-		}
-	}
-
-	if releaseNotes != "" && newVersion != "" {
-		fmt.Println()
-		fmt.Println(fmt.Sprintf("%s release notes:\n\n%s", newVersion, releaseNotes))
-	} else {
-		logrus.Warnf("no release notes available for %s", newVersion)
-	}
-
-	if NoInput {
-		return nil
-	}
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with deployment?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	return nil
-}
-
-// GetReleaseNotes prints release notes for a recipe version
-func GetReleaseNotes(recipeName, version string) (string, error) {
-	if version == "" {
-		return "", nil
-	}
-
-	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
-
-	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
-		releaseNotes, err := ioutil.ReadFile(fpath)
-		if err != nil {
-			return "", err
-		}
-		return string(releaseNotes), nil
-	}
-
-	return "", nil
-}

cli/internal/errors.go

@@ -4,7 +4,7 @@ import (
 	"os"
 
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // ShowSubcommandHelpAndError exits the program on error, logs the error to the

cli/internal/errors_test.go

@@ -0,0 +1,38 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Testing functions that call os.Exit
+// https://stackoverflow.com/questions/26225513/how-to-test-os-exit-scenarios-in-go
+// https://talks.golang.org/2014/testing.slide#23
+
+var testapp = &cli.App{
+	Name:  "abra",
+	Usage: `The Co-op Cloud command-line utility belt 🎩🐇`,
+}
+
+// not testing output as that changes. just if it exits with code 1
+// does not work because of some weird errors on cli's part. Its a hard lib to test effectively.
+// func TestShowSubcommandHelpAndError(t *testing.T) {
+// 	if os.Getenv("HelpAndError") == "1" {
+// 		ShowSubcommandHelpAndError(cli.NewContext(testapp, nil, nil), errors.New("Test error"))
+// 		return
+// 	}
+// 	cmd := exec.Command(os.Args[0], "-test.run=TestShowSubcommandHelpAndError")
+// 	cmd.Env = append(os.Environ(), "HelpAndError=1")
+// 	var out bytes.Buffer
+// 	cmd.Stderr = &out
+// 	err := cmd.Run()
+// 	println(out.String())
+// 	if !strings.Contains(out.String(), "Test error") {
+
+// 		t.Fatalf("expected command to show the error causing the exit, did not get correct stdout output")
+// 	}
+
+// 	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
+// 		return
+// 	}
+// 	t.Fatalf("process ran with err %v, want exit status 1", err)
+// }

cli/internal/list.go

@@ -1,10 +0,0 @@
-package internal
-
-// ReverseStringList reverses a list of a strings. Roll on Go generics.
-func ReverseStringList(strings []string) []string {
-	for i, j := 0, len(strings)-1; i < j; i, j = i+1, j-1 {
-		strings[i], strings[j] = strings[j], strings[i]
-	}
-
-	return strings
-}

cli/internal/new.go

@@ -1,198 +0,0 @@
-package internal
-
-import (
-	"fmt"
-	"path"
-
-	"coopcloud.tech/abra/pkg/app"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/jsontable"
-	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/secret"
-	"github.com/AlecAivazis/survey/v2"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// AppSecrets represents all app secrest
-type AppSecrets map[string]string
-
-// RecipeName is used for configuring recipe name programmatically
-var RecipeName string
-
-// createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets, error) {
-	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", Domain))
-	appEnv, err := config.ReadEnv(appEnvPath)
-	if err != nil {
-		return nil, err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
-	secrets, err := secret.GenerateSecrets(cl, secretEnvVars, sanitisedAppName, NewAppServer)
-	if err != nil {
-		return nil, err
-	}
-
-	if Pass {
-		for secretName := range secrets {
-			secretValue := secrets[secretName]
-			if err := secret.PassInsertSecret(secretValue, secretName, Domain, NewAppServer); err != nil {
-				return nil, err
-			}
-		}
-	}
-	return secrets, nil
-}
-
-// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
-func ensureDomainFlag(recipe recipe.Recipe, server string) error {
-	if Domain == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify app domain",
-			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
-		}
-		if err := survey.AskOne(prompt, &Domain); err != nil {
-			return err
-		}
-	}
-
-	if Domain == "" {
-		return fmt.Errorf("no domain provided")
-	}
-
-	return nil
-}
-
-// promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(appName string) error {
-	app, err := app.Get(appName)
-	if err != nil {
-		return err
-	}
-
-	secretEnvVars := secret.ReadSecretEnvVars(app.Env)
-	if len(secretEnvVars) == 0 {
-		logrus.Debugf("%s has no secrets to generate, skipping...", app.Recipe)
-		return nil
-	}
-
-	if !Secrets && !NoInput {
-		prompt := &survey.Confirm{
-			Message: "Generate app secrets?",
-		}
-		if err := survey.AskOne(prompt, &Secrets); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
-func ensureServerFlag() error {
-	servers, err := config.GetServers()
-	if err != nil {
-		return err
-	}
-
-	if NewAppServer == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select app server:",
-			Options: servers,
-		}
-		if err := survey.AskOne(prompt, &NewAppServer); err != nil {
-			return err
-		}
-	}
-
-	if NewAppServer == "" {
-		return fmt.Errorf("no server provided")
-	}
-
-	return nil
-}
-
-// NewAction is the new app creation logic
-func NewAction(c *cli.Context) error {
-	recipe := ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
-
-	if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ensureServerFlag(); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := ensureDomainFlag(recipe, NewAppServer); err != nil {
-		logrus.Fatal(err)
-	}
-
-	sanitisedAppName := config.SanitiseAppName(Domain)
-	logrus.Debugf("%s sanitised as %s for new app", Domain, sanitisedAppName)
-
-	if err := config.TemplateAppEnvSample(recipe.Name, Domain, NewAppServer, Domain); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := promptForSecrets(Domain); err != nil {
-		logrus.Fatal(err)
-	}
-
-	cl, err := client.New(NewAppServer)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	var secrets AppSecrets
-	var secretTable *jsontable.JSONTable
-	if Secrets {
-		secrets, err := createSecrets(cl, sanitisedAppName)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretCols := []string{"Name", "Value"}
-		secretTable = formatter.CreateTable(secretCols)
-		for secret := range secrets {
-			secretTable.Append([]string{secret, secrets[secret]})
-		}
-
-	}
-
-	if NewAppServer == "default" {
-		NewAppServer = "local"
-	}
-
-	tableCol := []string{"server", "recipe", "domain"}
-	table := formatter.CreateTable(tableCol)
-	table.Append([]string{NewAppServer, recipe.Name, Domain})
-
-	fmt.Println("")
-	fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
-	fmt.Println("")
-	table.Render()
-	fmt.Println("")
-	fmt.Println("You can configure this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app config %s", Domain))
-	fmt.Println("")
-	fmt.Println("You can deploy this app by running the following:")
-	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", Domain))
-	fmt.Println("")
-
-	if len(secrets) > 0 {
-		fmt.Println("Here are your generated secrets:")
-		fmt.Println("")
-		secretTable.Render()
-		fmt.Println("")
-		logrus.Warn("generated secrets are not shown again, please take note of them *now*")
-	}
-
-	return nil
-}

cli/internal/recipe.go

@@ -1,110 +0,0 @@
-package internal
-
-import (
-	"fmt"
-
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/distribution/reference"
-	"github.com/sirupsen/logrus"
-)
-
-// PromptBumpType prompts for version bump type
-func PromptBumpType(tagString, latestRelease string) error {
-	if (!Major && !Minor && !Patch) && tagString == "" {
-		fmt.Printf(`
-You need to make a decision about what kind of an update this new recipe
-version is. If someone else performs this upgrade, do they have to do some
-migration work or take care of some breaking changes? This can be signaled in
-the version you specify on the recipe deploy label and is called a semantic
-version.
-
-The latest published version is %s.
-
-Here is a semver cheat sheet (more on https://semver.org):
-
-    major: new features/bug fixes, backwards incompatible (e.g 1.0.0 -> 2.0.0).
-           the upgrade won't work without some preparation work and others need
-           to take care when performing it. "it could go wrong".
-
-    minor: new features/bug fixes, backwards compatible (e.g. 0.1.0 -> 0.2.0).
-           the upgrade should Just Work and there are no breaking changes in
-           the app and the recipe config. "it should go fine".
-
-    patch: bug fixes, backwards compatible (e.g. 0.0.1 -> 0.0.2). this upgrade
-           should also Just Work and is mostly to do with minor bug fixes
-           and/or security patches. "nothing to worry about".
-
-`, latestRelease)
-
-		var chosenBumpType string
-		prompt := &survey.Select{
-			Message: fmt.Sprintf("select recipe version increment type"),
-			Options: []string{"major", "minor", "patch"},
-		}
-
-		if err := survey.AskOne(prompt, &chosenBumpType); err != nil {
-			return err
-		}
-
-		SetBumpType(chosenBumpType)
-	}
-
-	return nil
-}
-
-// GetBumpType figures out which bump type is specified
-func GetBumpType() string {
-	var bumpType string
-
-	if Major {
-		bumpType = "major"
-	} else if Minor {
-		bumpType = "minor"
-	} else if Patch {
-		bumpType = "patch"
-	} else {
-		logrus.Fatal("no version bump type specififed?")
-	}
-
-	return bumpType
-}
-
-// SetBumpType figures out which bump type is specified
-func SetBumpType(bumpType string) {
-	if bumpType == "major" {
-		Major = true
-	} else if bumpType == "minor" {
-		Minor = true
-	} else if bumpType == "patch" {
-		Patch = true
-	} else {
-		logrus.Fatal("no version bump type specififed?")
-	}
-}
-
-// GetMainAppImage retrieves the main 'app' image name
-func GetMainAppImage(recipe recipe.Recipe) (string, error) {
-	var path string
-
-	for _, service := range recipe.Config.Services {
-		if service.Name == "app" {
-			img, err := reference.ParseNormalizedNamed(service.Image)
-			if err != nil {
-				return "", err
-			}
-
-			path = reference.Path(img)
-			path = formatter.StripTagMeta(path)
-
-			return path, nil
-		}
-	}
-
-	if path == "" {
-		return path, fmt.Errorf("%s has no main 'app' service?", recipe.Name)
-	}
-
-	return path, nil
-}

cli/internal/validate.go

@@ -2,134 +2,35 @@ package internal
 
 import (
 	"errors"
-	"fmt"
-	"os"
-	"strings"
 
 	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// AppName is used for configuring app name programmatically
-var AppName string
-
 // ValidateRecipe ensures the recipe arg is valid.
-func ValidateRecipe(c *cli.Context, opts ...runtime.Option) recipe.Recipe {
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
 	recipeName := c.Args().First()
-	conf := runtime.New(opts...)
 
 	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
+		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
 	}
 
-	chosenRecipe, err := recipe.Get(recipeName, conf)
+	recipe, err := recipe.Get(recipeName)
-	if err != nil {
-		if c.Command.Name == "generate" {
-			if strings.Contains(err.Error(), "missing a compose") {
-				logrus.Fatal(err)
-			}
-			logrus.Warn(err)
-		} else {
-			if strings.Contains(err.Error(), "template_driver is not allowed") {
-				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
-			}
-			logrus.Fatalf("unable to validate recipe: %s", err)
-		}
-	}
-
-	if conf.EnsureRecipeLatest {
-		if err := recipe.EnsureLatest(recipeName, conf); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	logrus.Debugf("validated %s as recipe argument", recipeName)
-
-	return chosenRecipe
-}
-
-// ValidateRecipeWithPrompt ensures a recipe argument is present before
-// validating, asking for input if required.
-func ValidateRecipeWithPrompt(c *cli.Context, opts ...runtime.Option) recipe.Recipe {
-	recipeName := c.Args().First()
-	conf := runtime.New(opts...)
-
-	if recipeName == "" && !NoInput {
-		var recipes []string
-
-		catl, err := recipe.ReadRecipeCatalogue()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		knownRecipes := make(map[string]bool)
-		for name := range catl {
-			knownRecipes[name] = true
-		}
-
-		localRecipes, err := recipe.GetRecipesLocal()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, recipeLocal := range localRecipes {
-			if _, ok := knownRecipes[recipeLocal]; !ok {
-				knownRecipes[recipeLocal] = true
-			}
-		}
-
-		for recipeName := range knownRecipes {
-			recipes = append(recipes, recipeName)
-		}
-
-		prompt := &survey.Select{
-			Message: "Select recipe",
-			Options: recipes,
-		}
-		if err := survey.AskOne(prompt, &recipeName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	if RecipeName != "" {
-		recipeName = RecipeName
-		logrus.Debugf("programmatically setting recipe name to %s", recipeName)
-	}
-
-	if recipeName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
-	}
-
-	chosenRecipe, err := recipe.Get(recipeName, conf)
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
-	if conf.EnsureRecipeLatest {
+	logrus.Debugf("validated '%s' as recipe argument", recipeName)
-		if err := recipe.EnsureLatest(recipeName, conf); err != nil {
-			logrus.Fatal(err)
-		}
-	}
 
-	logrus.Debugf("validated %s as recipe argument", recipeName)
+	return recipe
-
-	return chosenRecipe
 }
 
 // ValidateApp ensures the app name arg is valid.
-func ValidateApp(c *cli.Context, opts ...runtime.Option) config.App {
+func ValidateApp(c *cli.Context) config.App {
 	appName := c.Args().First()
-	conf := runtime.New(opts...)
-
-	if AppName != "" {
-		appName = AppName
-		logrus.Debugf("programmatically setting app name to %s", appName)
-	}
 
 	if appName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
@@ -140,11 +41,11 @@ func ValidateApp(c *cli.Context, opts ...runtime.Option) config.App {
 		logrus.Fatal(err)
 	}
 
-	if err := recipe.EnsureExists(app.Recipe, conf); err != nil {
+	if err := recipe.EnsureExists(app.Type); err != nil {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("validated %s as app argument", appName)
+	logrus.Debugf("validated '%s' as app argument", appName)
 
 	return app
 }
@@ -153,368 +54,11 @@ func ValidateApp(c *cli.Context, opts ...runtime.Option) config.App {
 func ValidateDomain(c *cli.Context) string {
 	domainName := c.Args().First()
 
-	if domainName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name",
-			Default: "example.com",
-		}
-		if err := survey.AskOne(prompt, &domainName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
 	if domainName == "" {
 		ShowSubcommandHelpAndError(c, errors.New("no domain provided"))
 	}
 
-	logrus.Debugf("validated %s as domain argument", domainName)
+	logrus.Debugf("validated '%s' as domain argument", domainName)
 
 	return domainName
 }
-
-// ValidateSubCmdFlags ensures flag order conforms to correct order
-func ValidateSubCmdFlags(c *cli.Context) bool {
-	for argIdx, arg := range c.Args() {
-		if !strings.HasPrefix(arg, "--") {
-			for _, flag := range c.Args()[argIdx:] {
-				if strings.HasPrefix(flag, "--") {
-					return false
-				}
-			}
-		}
-	}
-	return true
-}
-
-// ValidateServer ensures the server name arg is valid.
-func ValidateServer(c *cli.Context) string {
-	serverName := c.Args().First()
-
-	serverNames, err := config.ReadServerNames()
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	if serverName == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Specify a server name",
-			Options: serverNames,
-		}
-		if err := survey.AskOne(prompt, &serverName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	matched := false
-	for _, name := range serverNames {
-		if name == serverName {
-			matched = true
-		}
-	}
-
-	if !matched {
-		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
-	}
-
-	if serverName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
-	}
-
-	logrus.Debugf("validated %s as server argument", serverName)
-
-	return serverName
-}
-
-// EnsureDNSProvider ensures a DNS provider is chosen.
-func EnsureDNSProvider() error {
-	if DNSProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select DNS provider",
-			Options: []string{"gandi"},
-		}
-
-		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
-			return err
-		}
-	}
-
-	if DNSProvider == "" {
-		return fmt.Errorf("missing DNS provider?")
-	}
-
-	return nil
-}
-
-// EnsureDNSTypeFlag ensures a DNS type flag is present.
-func EnsureDNSTypeFlag(c *cli.Context) error {
-	if DNSType == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record type",
-			Default: "A",
-		}
-		if err := survey.AskOne(prompt, &DNSType); err != nil {
-			return err
-		}
-	}
-
-	if DNSType == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSNameFlag ensures a DNS name flag is present.
-func EnsureDNSNameFlag(c *cli.Context) error {
-	if DNSName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record name",
-			Default: "mysubdomain",
-		}
-		if err := survey.AskOne(prompt, &DNSName); err != nil {
-			return err
-		}
-	}
-
-	if DNSName == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
-	}
-
-	return nil
-}
-
-// EnsureDNSValueFlag ensures a DNS value flag is present.
-func EnsureDNSValueFlag(c *cli.Context) error {
-	if DNSValue == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify DNS record value",
-			Default: "192.168.1.2",
-		}
-		if err := survey.AskOne(prompt, &DNSValue); err != nil {
-			return err
-		}
-	}
-
-	if DNSValue == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
-	}
-
-	return nil
-}
-
-// EnsureZoneArgument ensures a zone argument is present.
-func EnsureZoneArgument(c *cli.Context) (string, error) {
-	zone := c.Args().First()
-
-	if zone == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "Specify a domain name zone",
-			Default: "example.com",
-		}
-
-		if err := survey.AskOne(prompt, &zone); err != nil {
-			return zone, err
-		}
-	}
-
-	if zone == "" {
-		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
-	}
-
-	return zone, nil
-}
-
-// EnsureServerProvider ensures a 3rd party server provider is chosen.
-func EnsureServerProvider() error {
-	if ServerProvider == "" && !NoInput {
-		prompt := &survey.Select{
-			Message: "Select server provider",
-			Options: []string{"capsul", "hetzner-cloud"},
-		}
-
-		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
-			return err
-		}
-	}
-
-	if ServerProvider == "" {
-		return fmt.Errorf("missing server provider?")
-	}
-
-	return nil
-}
-
-// EnsureNewCapsulVPSFlags ensure all flags are present.
-func EnsureNewCapsulVPSFlags(c *cli.Context) error {
-	if CapsulName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul name",
-		}
-		if err := survey.AskOne(prompt, &CapsulName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul instance URL",
-			Default: CapsulInstanceURL,
-		}
-		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul type",
-			Default: CapsulType,
-		}
-		if err := survey.AskOne(prompt, &CapsulType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify capsul image",
-			Default: CapsulImage,
-		}
-		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
-			return err
-		}
-	}
-
-	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify capsul SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
-			return err
-		}
-		CapsulSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if CapsulAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("CAPSUL_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify capsul API token",
-			}
-			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
-				return err
-			}
-		} else {
-			CapsulAPIToken = token
-		}
-	}
-
-	if CapsulName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
-	}
-	if CapsulInstanceURL == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
-	}
-	if CapsulType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
-	}
-	if CapsulImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
-	}
-	if len(CapsulSSHKeys.Value()) == 0 {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
-	}
-	if CapsulAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
-	}
-
-	return nil
-}
-
-// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
-func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
-	if HetznerCloudName == "" && !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS type",
-			Default: HetznerCloudType,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
-			return err
-		}
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS image",
-			Default: HetznerCloudImage,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
-			return err
-		}
-	}
-
-	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
-		var sshKeys string
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
-			Default: "",
-		}
-		if err := survey.AskOne(prompt, &sshKeys); err != nil {
-			return err
-		}
-		HetznerCloudSSHKeys = cli.StringSlice(strings.Split(sshKeys, ","))
-	}
-
-	if !NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS location",
-			Default: HetznerCloudLocation,
-		}
-		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
-			return err
-		}
-	}
-
-	if HetznerCloudAPIToken == "" && !NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			HetznerCloudAPIToken = token
-		}
-	}
-
-	if HetznerCloudName == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
-	}
-	if HetznerCloudType == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
-	}
-	if HetznerCloudImage == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
-	}
-	if HetznerCloudLocation == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
-	}
-	if HetznerCloudAPIToken == "" {
-		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
-	}
-
-	return nil
-}

cli/recipe/fetch.go

@@ -1,40 +0,0 @@
-package recipe
-
-import (
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var recipeFetchCommand = cli.Command{
-	Name:        "fetch",
-	Usage:       "Fetch recipe local copies",
-	Aliases:     []string{"f"},
-	ArgsUsage:   "[<recipe>]",
-	Description: "Fetchs all recipes without arguments.",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-		if recipeName != "" {
-			internal.ValidateRecipe(c)
-			return nil // ValidateRecipe ensures latest checkout
-		}
-
-		repos, err := recipe.ReadReposMetadata()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}

cli/recipe/lint.go

@@ -2,102 +2,105 @@ package recipe
 
 import (
 	"fmt"
+	"os"
+	"strconv"
 
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/tagcmp"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeLintCommand = cli.Command{
+var recipeLintCommand = &cli.Command{
 	Name:      "lint",
 	Usage:     "Lint a recipe",
 	Aliases:   []string{"l"},
 	ArgsUsage: "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OnlyErrorFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+		expectedVersion := false
+		if recipe.Config.Version == "3.8" {
+			expectedVersion = true
+		}
+
+		envSampleProvided := false
+		envSample := fmt.Sprintf("%s/%s/.env.sample", config.APPS_DIR, recipe.Name)
+		if _, err := os.Stat(envSample); !os.IsNotExist(err) {
+			envSampleProvided = true
+		} else if err != nil {
 			logrus.Fatal(err)
 		}
 
-		tableCol := []string{"ref", "rule", "severity", "satisfied", "skipped", "resolve"}
+		serviceNamedApp := false
-		table := formatter.CreateTable(tableCol)
+		traefikEnabled := false
+		healthChecksForAllServices := true
+		allImagesTagged := true
+		noUnstableTags := true
+		semverLikeTags := true
+		for _, service := range recipe.Config.Services {
+			if service.Name == "app" {
+				serviceNamedApp = true
+			}
 
-		hasError := false
+			for label := range service.Deploy.Labels {
-		bar := formatter.CreateProgressbar(-1, "running recipe lint rules...")
+				if label == "traefik.enable" {
-		for level := range lint.LintRules {
+					if service.Deploy.Labels[label] == "true" {
-			for _, rule := range lint.LintRules[level] {
+						traefikEnabled = true
-				if internal.OnlyErrors && rule.Level != "error" {
-					logrus.Debugf("skipping %s, does not have level \"error\"", rule.Ref)
-					continue
-				}
-
-				skipped := false
-				if rule.Skip(recipe) {
-					skipped = true
-				}
-
-				skippedOutput := "-"
-				if skipped {
-					skippedOutput = "yes"
-				}
-
-				satisfied := false
-				if !skipped {
-					ok, err := rule.Function(recipe)
-					if err != nil {
-						logrus.Warn(err)
-					}
-
-					if !ok && rule.Level == "error" {
-						hasError = true
-					}
-
-					if ok {
-						satisfied = true
 					}
 				}
+			}
 
-				satisfiedOutput := "yes"
+			img, err := reference.ParseNormalizedNamed(service.Image)
-				if !satisfied {
+			if err != nil {
-					satisfiedOutput = "NO"
+				logrus.Fatal(err)
-					if skipped {
+			}
-						satisfiedOutput = "-"
+			if reference.IsNameOnly(img) {
-					}
+				allImagesTagged = false
-				}
+			}
 
-				table.Append([]string{
+			tag := img.(reference.NamedTagged).Tag()
-					rule.Ref,
+			if tag == "latest" {
-					rule.Description,
+				noUnstableTags = false
-					rule.Level,
+			}
-					satisfiedOutput,
-					skippedOutput,
-					rule.HowToResolve,
-				})
 
-				bar.Add(1)
+			if !tagcmp.IsParsable(tag) {
+				semverLikeTags = false
+			}
+
+			if service.HealthCheck == nil {
+				healthChecksForAllServices = false
 			}
 		}
 
-		if table.NumLines() > 0 {
+		tableCol := []string{"rule", "satisfied"}
-			fmt.Println()
+		table := formatter.CreateTable(tableCol)
-			table.Render()
+		table.Append([]string{"compose files have the expected version", strconv.FormatBool(expectedVersion)})
-		}
+		table.Append([]string{"environment configuration is provided", strconv.FormatBool(envSampleProvided)})
-
+		table.Append([]string{"recipe contains a service named 'app'", strconv.FormatBool(serviceNamedApp)})
-		if hasError {
+		table.Append([]string{"traefik routing enabled on at least one service", strconv.FormatBool(traefikEnabled)})
-			logrus.Warn("watch out, some critical errors are present in your recipe config")
+		table.Append([]string{"all services have a healthcheck enabled", strconv.FormatBool(healthChecksForAllServices)})
-		}
+		table.Append([]string{"all images are using a tag", strconv.FormatBool(allImagesTagged)})
+		table.Append([]string{"no usage of unstable 'latest' tags", strconv.FormatBool(noUnstableTags)})
+		table.Append([]string{"all tags are using a semver-like format", strconv.FormatBool(semverLikeTags)})
+		table.Render()
 
 		return nil
 	},
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
 }

cli/recipe/list.go

@@ -3,79 +3,36 @@ package recipe
 import (
 	"fmt"
 	"sort"
-	"strconv"
-	"strings"
 
-	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/formatter"
-	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var pattern string
+var recipeListCommand = &cli.Command{
-var patternFlag = &cli.StringFlag{
-	Name:        "pattern, p",
-	Value:       "",
-	Usage:       "Simple string to filter recipes",
-	Destination: &pattern,
-}
-
-var recipeListCommand = cli.Command{
 	Name:    "list",
 	Usage:   "List available recipes",
 	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.MachineReadableFlag,
-		patternFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		catl, err := recipe.ReadRecipeCatalogue()
+		catl, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err.Error())
 		}
 
 		recipes := catl.Flatten()
-		sort.Sort(recipe.ByRecipeName(recipes))
+		sort.Sort(catalogue.ByRecipeName(recipes))
 
-		tableCol := []string{"name", "category", "status", "healthcheck", "backups", "email", "tests", "SSO"}
+		tableCol := []string{"name", "category", "status"}
 		table := formatter.CreateTable(tableCol)
 
-		len := 0
 		for _, recipe := range recipes {
-			tableRow := []string{
+			status := fmt.Sprintf("%v", recipe.Features.Status)
-				recipe.Name,
+			tableRow := []string{recipe.Name, recipe.Category, status}
-				recipe.Category,
+			table.Append(tableRow)
-				strconv.Itoa(recipe.Features.Status),
-				recipe.Features.Healthcheck,
-				recipe.Features.Backups,
-				recipe.Features.Email,
-				recipe.Features.Tests,
-				recipe.Features.SSO,
-			}
-
-			if pattern != "" {
-				if strings.Contains(recipe.Name, pattern) {
-					table.Append(tableRow)
-					len++
-				}
-			} else {
-				table.Append(tableRow)
-				len++
-			}
 		}
 
-		if table.NumLines() > 0 {
+		table.Render()
-			if internal.MachineReadable {
-				table.SetCaption(false, "")
-				table.JSONRender()
-			} else {
-				table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
-				table.Render()
-			}
-		}
 
 		return nil
 	},

cli/recipe/new.go

@@ -1,10 +1,7 @@
 package recipe
 
 import (
-	"bytes"
-	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"text/template"
@@ -13,128 +10,70 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/git"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// recipeMetadata is the recipe metadata for the README.md
+var recipeNewCommand = &cli.Command{
-type recipeMetadata struct {
+	Name:      "new",
-	Name        string
-	Description string
-	Category    string
-	Status      string
-	Image       string
-	Healthcheck string
-	Backups     string
-	Email       string
-	Tests       string
-	SSO         string
-}
-
-var recipeNewCommand = cli.Command{
-	Name:    "new",
-	Aliases: []string{"n"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:    internal.SubCommandBefore,
 	Usage:     "Create a new recipe",
+	Aliases:   []string{"n"},
 	ArgsUsage: "<recipe>",
-	Description: `
-Create a new recipe.
-
-Abra uses the built-in example repository which is available here:
-
-    https://git.coopcloud.tech/coop-cloud/example
-
-Files within the example repository make use of the Golang templating system
-which Abra uses to inject values into the generated recipe folder (e.g. name of
-recipe and domain in the sample environment config).
-`,
 	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
+		recipe := internal.ValidateRecipe(c)
 
-		if recipeName == "" {
+		directory := path.Join(config.APPS_DIR, recipe.Name)
-			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
-		}
-
-		directory := path.Join(config.RECIPES_DIR, recipeName)
 		if _, err := os.Stat(directory); !os.IsNotExist(err) {
-			logrus.Fatalf("%s recipe directory already exists?", directory)
+			logrus.Fatalf("'%s' recipe directory already exists?", directory)
+			return nil
 		}
 
 		url := fmt.Sprintf("%s/example.git", config.REPOS_BASE_URL)
 		if err := git.Clone(directory, url); err != nil {
-			logrus.Fatal(err)
+			return err
 		}
 
-		gitRepo := path.Join(config.RECIPES_DIR, recipeName, ".git")
+		gitRepo := path.Join(config.APPS_DIR, recipe.Name, ".git")
 		if err := os.RemoveAll(gitRepo); err != nil {
 			logrus.Fatal(err)
+			return nil
 		}
-		logrus.Debugf("removed example git repo in %s", gitRepo)
+		logrus.Debugf("removed git repo in '%s'", gitRepo)
-
-		meta := newRecipeMeta(recipeName)
 
 		toParse := []string{
-			path.Join(config.RECIPES_DIR, recipeName, "README.md"),
+			path.Join(config.APPS_DIR, recipe.Name, "README.md"),
-			path.Join(config.RECIPES_DIR, recipeName, ".env.sample"),
+			path.Join(config.APPS_DIR, recipe.Name, ".env.sample"),
+			path.Join(config.APPS_DIR, recipe.Name, ".drone.yml"),
 		}
 		for _, path := range toParse {
+			file, err := os.OpenFile(path, os.O_RDWR, 0755)
+			if err != nil {
+				logrus.Fatal(err)
+				return nil
+			}
+
 			tpl, err := template.ParseFiles(path)
 			if err != nil {
 				logrus.Fatal(err)
+				return nil
 			}
 
-			var templated bytes.Buffer
+			// TODO: ask for description and probably other things so that the
-			if err := tpl.Execute(&templated, meta); err != nil {
+			// template repository is more "ready" to go than the current best-guess
+			// mode of templating
+			if err := tpl.Execute(file, struct {
+				Name        string
+				Description string
+			}{recipe.Name, "TODO"}); err != nil {
 				logrus.Fatal(err)
+				return nil
 			}
-
-			if err := ioutil.WriteFile(path, templated.Bytes(), 0644); err != nil {
-				logrus.Fatal(err)
-			}
-
 		}
 
-		newGitRepo := path.Join(config.RECIPES_DIR, recipeName)
+		logrus.Infof(
-		if err := git.Init(newGitRepo, true); err != nil {
+			"new recipe '%s' created in %s, happy hacking!\n",
-			logrus.Fatal(err)
+			recipe.Name, path.Join(config.APPS_DIR, recipe.Name),
-		}
+		)
-
-		fmt.Print(fmt.Sprintf(`
-Your new %s recipe has been created in %s.
-
-In order to share your recipe, you can upload it the git repository to:
-
-    https://git.coopcloud.tech/coop-cloud/%s
-
-If you're not sure how to do that, come chat with us:
-
-    https://docs.coopcloud.tech/intro/contact
-
-See "abra recipe -h" for additional recipe maintainer commands.
-
-Happy Hacking!
-
-`, recipeName, path.Join(config.RECIPES_DIR, recipeName), recipeName))
 
 		return nil
 	},
 }
-
-// newRecipeMeta creates a new recipeMetadata instance with defaults
-func newRecipeMeta(recipeName string) recipeMetadata {
-	return recipeMetadata{
-		Name:        recipeName,
-		Description: "> One line description of the recipe",
-		Category:    "Apps",
-		Status:      "0",
-		Image:       fmt.Sprintf("[`%s`](https://hub.docker.com/r/%s), 4, upstream", recipeName, recipeName),
-		Healthcheck: "No",
-		Backups:     "No",
-		Email:       "No",
-		Tests:       "No",
-		SSO:         "No",
-	}
-}

cli/recipe/recipe.go

@@ -1,34 +1,51 @@
 package recipe
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-// RecipeCommand defines all recipe related sub-commands.
+var Major bool
-var RecipeCommand = cli.Command{
+var MajorFlag = &cli.BoolFlag{
-	Name:      "recipe",
+	Name:        "major",
-	Aliases:   []string{"r"},
+	Value:       false,
-	Usage:     "Manage recipes",
+	Aliases:     []string{"ma", "x"},
-	ArgsUsage: "<recipe>",
+	Destination: &Major,
-	Description: `
+}
-A recipe is a blueprint for an app. It is a bunch of config files which
-describe how to deploy and maintain an app. Recipes are maintained by the Co-op
-Cloud community and you can use Abra to read them, deploy them and create apps
-for you.
 
-Anyone who uses a recipe can become a maintainer. Maintainers typically make
+var Minor bool
-sure the recipe is in good working order and the config upgraded in a timely
+var MinorFlag = &cli.BoolFlag{
-manner. Abra supports convenient automation for recipe maintainenace, see the
+	Name:        "minor",
-"abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands.
+	Value:       false,
+	Aliases:     []string{"mi", "y"},
+	Destination: &Minor,
+}
+
+var Patch bool
+var PatchFlag = &cli.BoolFlag{
+	Name:        "patch",
+	Value:       false,
+	Aliases:     []string{"p", "z"},
+	Destination: &Patch,
+}
+
+// RecipeCommand defines all recipe related sub-commands.
+var RecipeCommand = &cli.Command{
+	Name:      "recipe",
+	Usage:     "Manage recipes (for maintainers)",
+	ArgsUsage: "<recipe>",
+	Aliases:   []string{"r"},
+	Description: `
+A recipe is a blueprint for an app. It is a bunch of configuration files which
+describe how to deploy and maintain an app. Recipes are maintained by the Co-op
+Cloud community and you can use Abra to read them and create apps for you.
 `,
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
-		recipeFetchCommand,
-		recipeLintCommand,
 		recipeListCommand,
-		recipeNewCommand,
-		recipeReleaseCommand,
-		recipeSyncCommand,
-		recipeUpgradeCommand,
 		recipeVersionCommand,
+		recipeReleaseCommand,
+		recipeNewCommand,
+		recipeUpgradeCommand,
+		recipeSyncCommand,
+		recipeLintCommand,
 	},
 }

cli/recipe/release.go

@@ -7,448 +7,313 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeReleaseCommand = cli.Command{
+var Push bool
+var PushFlag = &cli.BoolFlag{
+	Name:        "push",
+	Value:       false,
+	Destination: &Push,
+}
+
+var Dry bool
+var DryFlag = &cli.BoolFlag{
+	Name:        "dry-run",
+	Value:       false,
+	Aliases:     []string{"d"},
+	Destination: &Dry,
+}
+
+var CommitMessage string
+var CommitMessageFlag = &cli.StringFlag{
+	Name:        "commit-message",
+	Usage:       "commit message. Implies --commit",
+	Aliases:     []string{"cm"},
+	Destination: &CommitMessage,
+}
+
+var Commit bool
+var CommitFlag = &cli.BoolFlag{
+	Name:        "commit",
+	Usage:       "add compose.yml to staging area and commit changes",
+	Value:       false,
+	Aliases:     []string{"c"},
+	Destination: &Commit,
+}
+
+var TagMessage string
+var TagMessageFlag = &cli.StringFlag{
+	Name:        "tag-comment",
+	Usage:       "tag comment. If not given, user will be asked for it",
+	Aliases:     []string{"t", "tm"},
+	Destination: &TagMessage,
+}
+
+var recipeReleaseCommand = &cli.Command{
 	Name:      "release",
+	Usage:     "tag a recipe",
 	Aliases:   []string{"rl"},
-	Usage:     "Release a new recipe version",
+	ArgsUsage: "<recipe> [<tag>]",
-	ArgsUsage: "<recipe> [<version>]",
 	Description: `
-Create a new version of a recipe. These versions are then published on the
+This command is used to specify a new tag for a recipe. These tags are used to
-Co-op Cloud recipe catalogue. These versions take the following form:
+identify different versions of the recipe and are published on the Co-op Cloud
+recipe catalogue.
+
+These tags take the following form:
 
     a.b.c+x.y.z
 
-Where the "a.b.c" part is a semantic version determined by the maintainer. The
+Where the "a.b.c" part is maintained as a semantic version of the recipe by the
-"x.y.z" part is the image tag of the recipe "app" service (the main container
+recipe maintainer. And the "x.y.z" part is the image tag of the recipe "app"
-which contains the software to be used, by naming convention).
+service (the main container which contains the software to be used).
 
-We maintain a semantic versioning scheme ("a.b.c") alongside the recipe
+We maintain a semantic versioning scheme ("a.b.c") alongside the libre app
-versioning scheme ("x.y.z") in order to maximise the chances that the nature of
+versioning scheme in order to maximise the chances that the nature of recipe
-recipe updates are properly communicated. I.e. developers of an app might
+updates are properly communicated.
-publish a minor version but that might lead to changes in the recipe which are
-major and therefore require intervention while doing the upgrade work.
 
-Publish your new release to git.coopcloud.tech with "-p/--publish". This
+Abra does its best to read the "a.b.c" version scheme and communicate what
-requires that you have permission to git push to these repositories and have
+action needs to be taken when performing different operations such as an update
-your SSH keys configured on your account.
+or a rollback of an app.
 `,
 	Flags: []cli.Flag{
-		internal.DebugFlag,
+		DryFlag,
-		internal.NoInputFlag,
+		PatchFlag,
-		internal.DryFlag,
+		MinorFlag,
-		internal.MajorFlag,
+		MajorFlag,
-		internal.MinorFlag,
+		PushFlag,
-		internal.PatchFlag,
+		CommitFlag,
-		internal.PublishFlag,
+		CommitMessageFlag,
+		TagMessageFlag,
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
+		recipe := internal.ValidateRecipe(c)
-
+		directory := path.Join(config.APPS_DIR, recipe.Name)
-		imagesTmp, err := getImageVersions(recipe)
+		tagstring := c.Args().Get(1)
-		if err != nil {
+		imagesTmp := getImageVersions(recipe)
-			logrus.Fatal(err)
+		mainApp := getMainApp(recipe)
-		}
-
-		mainApp, err := internal.GetMainAppImage(recipe)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		mainAppVersion := imagesTmp[mainApp]
-		if mainAppVersion == "" {
+
-			logrus.Fatalf("main app service version for %s is empty?", recipe.Name)
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
 		}
 
-		tagString := c.Args().Get(1)
+		if mainAppVersion == "" {
-		if tagString != "" {
+			logrus.Fatal("main app version is empty?")
-			if _, err := tagcmp.Parse(tagString); err != nil {
+		}
-				logrus.Fatalf("cannot parse %s, invalid tag specified?", tagString)
+
+		if tagstring != "" {
+			if _, err := tagcmp.Parse(tagstring); err != nil {
+				logrus.Fatal("invalid tag specified")
 			}
 		}
 
-		if (internal.Major || internal.Minor || internal.Patch) && tagString != "" {
+		if TagMessage == "" {
-			logrus.Fatal("cannot specify tag and bump type at the same time")
+			prompt := &survey.Input{
-		}
+				Message: "tag message",
-
+			}
-		if tagString != "" {
+			if err := survey.AskOne(prompt, &TagMessage); err != nil {
-			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		tags, err := recipe.Tags()
+		var createTagOptions git.CreateTagOptions
-		if err != nil {
+		createTagOptions.Message = TagMessage
-			logrus.Fatal(err)
-		}
 
-		if tagString == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+		if Commit || (CommitMessage != "") {
-			var err error
+			commitRepo, err := git.PlainOpen(directory)
-			tagString, err = getLabelVersion(recipe, false)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+			commitWorktree, err := commitRepo.Worktree()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if CommitMessage == "" {
+				prompt := &survey.Input{
+					Message: "commit message",
+				}
+				if err := survey.AskOne(prompt, &CommitMessage); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			err = commitWorktree.AddGlob("compose.**yml")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debug("staged compose.**yml for commit")
+
+			_, err = commitWorktree.Commit(CommitMessage, &git.CommitOptions{})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info("changes commited")
 		}
 
-		if len(tags) > 0 {
+		repo, err := git.PlainOpen(directory)
-			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
+		if err != nil {
-			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
+			logrus.Fatal(err)
+		}
+		head, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		// bumpType is used to decide what part of the tag should be incremented
+		bumpType := btoi(Major)*4 + btoi(Minor)*2 + btoi(Patch)
+		if bumpType != 0 {
+			// a bitwise check if the number is a power of 2
+			if (bumpType & (bumpType - 1)) != 0 {
+				logrus.Fatal("you can only use one of: --major, --minor, --patch.")
+			}
+		}
+
+		if tagstring != "" {
+			if bumpType > 0 {
+				logrus.Warn("user specified a version number and --major/--minor/--patch at the same time! using version number...")
+			}
+			tag, err := tagcmp.Parse(tagstring)
+			if err != nil {
 				logrus.Fatal(err)
 			}
+			if tag.MissingMinor {
+				tag.Minor = "0"
+				tag.MissingMinor = false
+			}
+			if tag.MissingPatch {
+				tag.Patch = "0"
+				tag.MissingPatch = false
+			}
+			tagstring = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
+			if Dry {
+				logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", tagstring, head.Hash()))
+				return nil
+			}
+
+			repo.CreateTag(tagstring, head.Hash(), &createTagOptions)
+			logrus.Info(fmt.Sprintf("created tag %s at %s", tagstring, head.Hash()))
+			if Push {
+				if err := repo.Push(&git.PushOptions{}); err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("pushed tag %s to remote", tagstring))
+			}
+
+			return nil
+		}
+
+		// get the latest tag with its hash, name etc
+		var lastGitTag tagcmp.Tag
+		iter, err := repo.Tags()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if err := iter.ForEach(func(ref *plumbing.Reference) error {
+			obj, err := repo.TagObject(ref.Hash())
+			if err != nil {
+				return err
+			}
+			tagcmpTag, err := tagcmp.Parse(obj.Name)
+			if err != nil {
+				return err
+			}
+			if (lastGitTag == tagcmp.Tag{}) {
+				lastGitTag = tagcmpTag
+			} else if tagcmpTag.IsGreaterThan(lastGitTag) {
+				lastGitTag = tagcmpTag
+			}
+			return nil
+		}); err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Println(lastGitTag)
+
+		newTag := lastGitTag
+		var newTagString string
+		if bumpType > 0 {
+			if Patch {
+				now, err := strconv.Atoi(newTag.Patch)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = strconv.Itoa(now + 1)
+			} else if Minor {
+				now, err := strconv.Atoi(newTag.Minor)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = "0"
+				newTag.Minor = strconv.Itoa(now + 1)
+			} else if Major {
+				now, err := strconv.Atoi(newTag.Major)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = "0"
+				newTag.Minor = "0"
+				newTag.Major = strconv.Itoa(now + 1)
+			}
 		} else {
-			logrus.Warnf("no tag specified and no previous tag available for %s, assuming this is the initial release", recipe.Name)
+			logrus.Fatal("we don't support automatic tag generation yet - specify a version or use one of: --major --minor --patch")
+		}
+		newTag.Metadata = mainAppVersion
+		newTagString = newTag.String()
+		if Dry {
+			logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", newTagString, head.Hash()))
+			return nil
+		}
 
-			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+		repo.CreateTag(newTagString, head.Hash(), &createTagOptions)
-				if cleanUpErr := cleanUpTag(tagString, recipe.Name); err != nil {
+		logrus.Info(fmt.Sprintf("created tag %s at %s", newTagString, head.Hash()))
-					logrus.Fatal(cleanUpErr)
+		if Push {
-				}
+			if err := repo.Push(&git.PushOptions{}); err != nil {
 				logrus.Fatal(err)
 			}
+			logrus.Info(fmt.Sprintf("pushed tag %s to remote", newTagString))
 		}
 
 		return nil
+
 	},
 }
 
-// getImageVersions retrieves image versions for a recipe
+func getImageVersions(recipe recipe.Recipe) map[string]string {
-func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
-	var services = make(map[string]string)
 
-	missingTag := false
+	var services = make(map[string]string)
 	for _, service := range recipe.Config.Services {
 		if service.Image == "" {
 			continue
 		}
-
+		srv := strings.Split(service.Image, ":")
-		img, err := reference.ParseNormalizedNamed(service.Image)
+		services[srv[0]] = srv[1]
-		if err != nil {
-			return services, err
-		}
-
-		path := reference.Path(img)
-
-		path = formatter.StripTagMeta(path)
-
-		var tag string
-		switch img.(type) {
-		case reference.NamedTagged:
-			tag = img.(reference.NamedTagged).Tag()
-		case reference.Named:
-			if service.Name == "app" {
-				missingTag = true
-			}
-			continue
-		}
-
-		services[path] = tag
 	}
 
-	if missingTag {
+	return services
-		return services, fmt.Errorf("app service is missing image tag?")
-	}
-
-	return services, nil
 }
 
-// createReleaseFromTag creates a new release based on a supplied recipe version string
+func getMainApp(recipe recipe.Recipe) string {
-func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string) error {
+	for _, service := range recipe.Config.Services {
-	var err error
+		name := service.Name
-
+		if name == "app" {
-	directory := path.Join(config.RECIPES_DIR, recipe.Name)
+			return strings.Split(service.Image, ":")[0]
-	repo, err := git.PlainOpen(directory)
+		}
-	if err != nil {
-		return err
 	}
-
+	return ""
-	tag, err := tagcmp.Parse(tagString)
-	if err != nil {
-		return err
-	}
-
-	if tag.MissingMinor {
-		tag.Minor = "0"
-		tag.MissingMinor = false
-	}
-
-	if tag.MissingPatch {
-		tag.Patch = "0"
-		tag.MissingPatch = false
-	}
-
-	if tagString == "" {
-		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
-	}
-
-	if err := commitRelease(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := tagRelease(tagString, repo); err != nil {
-		logrus.Fatal(err)
-	}
-
-	if err := pushRelease(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
-	return nil
 }
 
-// btoi converts a boolean value into an integer
 func btoi(b bool) int {
 	if b {
 		return 1
 	}
-
 	return 0
 }
-
-// getTagCreateOptions constructs git tag create options
-func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
-	msg := fmt.Sprintf("chore: publish %s release", tag)
-	return git.CreateTagOptions{Message: msg}, nil
-}
-
-func commitRelease(recipe recipe.Recipe, tag string) error {
-	if internal.Dry {
-		logrus.Debugf("dry run: no changes committed")
-		return nil
-	}
-
-	isClean, err := gitPkg.IsClean(recipe.Dir())
-	if err != nil {
-		return err
-	}
-
-	if isClean {
-		if !internal.Dry {
-			return fmt.Errorf("no changes discovered in %s, nothing to publish?", recipe.Dir())
-		}
-	}
-
-	msg := fmt.Sprintf("chore: publish %s release", tag)
-	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func tagRelease(tagString string, repo *git.Repository) error {
-	if internal.Dry {
-		logrus.Debugf("dry run: no git tag created (%s)", tagString)
-		return nil
-	}
-
-	head, err := repo.Head()
-	if err != nil {
-		return err
-	}
-
-	createTagOptions, err := getTagCreateOptions(tagString)
-	if err != nil {
-		return err
-	}
-
-	_, err = repo.CreateTag(tagString, head.Hash(), &createTagOptions)
-	if err != nil {
-		return err
-	}
-
-	hash := formatter.SmallSHA(head.Hash().String())
-	logrus.Debugf(fmt.Sprintf("created tag %s at %s", tagString, hash))
-
-	return nil
-}
-
-func pushRelease(recipe recipe.Recipe, tagString string) error {
-	if internal.Dry {
-		logrus.Info("dry run: no changes published")
-		return nil
-	}
-
-	if !internal.Publish && !internal.NoInput {
-		prompt := &survey.Confirm{
-			Message: "publish new release?",
-		}
-
-		if err := survey.AskOne(prompt, &internal.Publish); err != nil {
-			return err
-		}
-	}
-
-	if internal.Publish {
-		if err := recipe.Push(internal.Dry); err != nil {
-			return err
-		}
-		url := fmt.Sprintf("%s/%s/src/tag/%s", config.REPOS_BASE_URL, recipe.Name, tagString)
-		logrus.Infof("new release published: %s", url)
-	} else {
-		logrus.Info("no -p/--publish passed, not publishing")
-	}
-
-	return nil
-}
-
-func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recipe.Recipe, tags []string) error {
-	directory := path.Join(config.RECIPES_DIR, recipe.Name)
-	repo, err := git.PlainOpen(directory)
-	if err != nil {
-		return err
-	}
-
-	bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
-	if bumpType != 0 {
-		if (bumpType & (bumpType - 1)) != 0 {
-			return fmt.Errorf("you can only use one of: --major, --minor, --patch")
-		}
-	}
-
-	var lastGitTag tagcmp.Tag
-	for _, tag := range tags {
-		parsed, err := tagcmp.Parse(tag)
-		if err != nil {
-			return err
-		}
-
-		if (lastGitTag == tagcmp.Tag{}) {
-			lastGitTag = parsed
-		} else if parsed.IsGreaterThan(lastGitTag) {
-			lastGitTag = parsed
-		}
-	}
-
-	newTag := lastGitTag
-	if internal.Patch {
-		now, err := strconv.Atoi(newTag.Patch)
-		if err != nil {
-			return err
-		}
-
-		newTag.Patch = strconv.Itoa(now + 1)
-	} else if internal.Minor {
-		now, err := strconv.Atoi(newTag.Minor)
-		if err != nil {
-			return err
-		}
-
-		newTag.Patch = "0"
-		newTag.Minor = strconv.Itoa(now + 1)
-	} else if internal.Major {
-		now, err := strconv.Atoi(newTag.Major)
-		if err != nil {
-			return err
-		}
-
-		newTag.Patch = "0"
-		newTag.Minor = "0"
-		newTag.Major = strconv.Itoa(now + 1)
-	}
-
-	if tagString == "" {
-		if err := internal.PromptBumpType(tagString, lastGitTag.String()); err != nil {
-			return err
-		}
-	}
-
-	if internal.Major || internal.Minor || internal.Patch {
-		newTag.Metadata = mainAppVersion
-		tagString = newTag.String()
-	}
-
-	if lastGitTag.String() == tagString {
-		logrus.Fatalf("latest git tag (%s) and synced label (%s) are the same?", lastGitTag, tagString)
-	}
-
-	if !internal.NoInput {
-		prompt := &survey.Confirm{
-			Message: fmt.Sprintf("current: %s, new: %s, correct?", lastGitTag, tagString),
-		}
-
-		var ok bool
-		if err := survey.AskOne(prompt, &ok); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !ok {
-			logrus.Fatal("exiting as requested")
-		}
-	}
-
-	if err := commitRelease(recipe, tagString); err != nil {
-		logrus.Fatalf("failed to commit changes: %s", err.Error())
-	}
-
-	if err := tagRelease(tagString, repo); err != nil {
-		logrus.Fatalf("failed to tag release: %s", err.Error())
-	}
-
-	if err := pushRelease(recipe, tagString); err != nil {
-		logrus.Fatalf("failed to publish new release: %s", err.Error())
-	}
-
-	return nil
-}
-
-// cleanUpTag removes a freshly created tag
-func cleanUpTag(tag, recipeName string) error {
-	directory := path.Join(config.RECIPES_DIR, recipeName)
-	repo, err := git.PlainOpen(directory)
-	if err != nil {
-		return err
-	}
-
-	if err := repo.DeleteTag(tag); err != nil {
-		if !strings.Contains(err.Error(), "not found") {
-			return err
-		}
-	}
-
-	logrus.Debugf("removed freshly created tag %s", tag)
-
-	return nil
-}
-
-func getLabelVersion(recipe recipe.Recipe, prompt bool) (string, error) {
-	initTag, err := recipePkg.GetVersionLabelLocal(recipe)
-	if err != nil {
-		return "", err
-	}
-
-	if initTag == "" {
-		logrus.Fatalf("unable to read version for %s from synced label. Did you try running \"abra recipe sync %s\" already?", recipe.Name, recipe.Name)
-	}
-
-	logrus.Warnf("discovered %s as currently synced recipe label", initTag)
-
-	if prompt && !internal.NoInput {
-		var response bool
-		prompt := &survey.Confirm{Message: fmt.Sprintf("use %s as the new version?", initTag)}
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return "", err
-		}
-
-		if !response {
-			return "", fmt.Errorf("please fix your synced label for %s and re-run this command", recipe.Name)
-		}
-	}
-
-	return initTag, nil
-}

cli/recipe/sync.go

@@ -1,200 +1,88 @@
 package recipe
 
 import (
+	"errors"
 	"fmt"
-	"path"
-	"strconv"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeSyncCommand = cli.Command{
+var recipeSyncCommand = &cli.Command{
 	Name:      "sync",
+	Usage:     "Ensure recipe version labels are up-to-date",
 	Aliases:   []string{"s"},
-	Usage:     "Sync recipe version label",
+	ArgsUsage: "<recipe> <version>",
-	ArgsUsage: "<recipe> [<version>]",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DryFlag,
-		internal.MajorFlag,
-		internal.MinorFlag,
-		internal.PatchFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Description: `
-Generate labels for the main recipe service (i.e. by convention, the service
+This command will generate labels for the main recipe service (i.e. by
-named "app") which corresponds to the following format:
+convention, typically the service named "app") which corresponds to the
+following format:
 
     coop-cloud.${STACK_NAME}.version=<version>
 
-Where <version> can be specifed on the command-line or Abra can attempt to
+The <version> is determined by the recipe maintainer and is specified on the
-auto-generate it for you. The <recipe> configuration will be updated on the
+command-line. The <recipe> configuration will be updated on the local file
-local file system.
+system.
 `,
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c, runtime.WithEnsureRecipeLatest(false))
+		if c.Args().Len() != 2 {
-
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <recipe>/<version> arguments?"))
-		mainApp, err := internal.GetMainAppImage(recipe)
-		if err != nil {
-			logrus.Fatal(err)
 		}
 
-		imagesTmp, err := getImageVersions(recipe)
+		recipe := internal.ValidateRecipe(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		mainAppVersion := imagesTmp[mainApp]
-
-		tags, err := recipe.Tags()
-		if err != nil {
-			logrus.Fatal(err)
-		}
 
+		// TODO: validate with tagcmp when new commits come in
+		// See https://git.coopcloud.tech/coop-cloud/abra/pulls/109
 		nextTag := c.Args().Get(1)
-		if len(tags) == 0 && nextTag == "" {
-			logrus.Warnf("no git tags found for %s", recipe.Name)
-			fmt.Println(fmt.Sprintf(`
-The following options are two types of initial semantic version that you can
-pick for %s that will be published in the recipe catalogue. This follows the
-semver convention (more on https://semver.org), here is a short cheatsheet
-
-    0.1.0: development release, still hacking. when you make a major upgrade
-           you increment the "y" part (i.e. 0.1.0 -> 0.2.0) and only move to
-           using the "x" part when things are stable.
-
-    1.0.0: public release, assumed to be working. you already have a stable
-           and reliable deployment of this app and feel relatively confident
-           about it.
-
-If you want people to be able alpha test your current config for %s but don't
-think it is quite reliable, go with 0.1.0 and people will know that things are
-likely to change.
-
-`, recipe.Name, recipe.Name))
-			var chosenVersion string
-			edPrompt := &survey.Select{
-				Message: "which version do you want to begin with?",
-				Options: []string{"0.1.0", "1.0.0"},
-			}
-
-			if err := survey.AskOne(edPrompt, &chosenVersion); err != nil {
-				logrus.Fatal(err)
-			}
-
-			nextTag = fmt.Sprintf("%s+%s", chosenVersion, mainAppVersion)
-		}
-
-		if nextTag == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
-			latestRelease := tags[len(tags)-1]
-			if err := internal.PromptBumpType("", latestRelease); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		if nextTag == "" {
-			recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
-			repo, err := git.PlainOpen(recipeDir)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			var lastGitTag tagcmp.Tag
-			iter, err := repo.Tags()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if err := iter.ForEach(func(ref *plumbing.Reference) error {
-				obj, err := repo.TagObject(ref.Hash())
-				if err != nil {
-					return err
-				}
-
-				tagcmpTag, err := tagcmp.Parse(obj.Name)
-				if err != nil {
-					return err
-				}
-
-				if (lastGitTag == tagcmp.Tag{}) {
-					lastGitTag = tagcmpTag
-				} else if tagcmpTag.IsGreaterThan(lastGitTag) {
-					lastGitTag = tagcmpTag
-				}
-
-				return nil
-			}); err != nil {
-				logrus.Fatal(err)
-			}
-
-			// bumpType is used to decide what part of the tag should be incremented
-			bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
-			if bumpType != 0 {
-				// a bitwise check if the number is a power of 2
-				if (bumpType & (bumpType - 1)) != 0 {
-					logrus.Fatal("you can only use one version flag: --major, --minor or --patch")
-				}
-			}
-
-			newTag := lastGitTag
-			if bumpType > 0 {
-				if internal.Patch {
-					now, err := strconv.Atoi(newTag.Patch)
-					if err != nil {
-						logrus.Fatal(err)
-					}
-
-					newTag.Patch = strconv.Itoa(now + 1)
-				} else if internal.Minor {
-					now, err := strconv.Atoi(newTag.Minor)
-					if err != nil {
-						logrus.Fatal(err)
-					}
-
-					newTag.Patch = "0"
-					newTag.Minor = strconv.Itoa(now + 1)
-				} else if internal.Major {
-					now, err := strconv.Atoi(newTag.Major)
-					if err != nil {
-						logrus.Fatal(err)
-					}
-
-					newTag.Patch = "0"
-					newTag.Minor = "0"
-					newTag.Major = strconv.Itoa(now + 1)
-				}
-			}
-
-			newTag.Metadata = mainAppVersion
-			logrus.Debugf("choosing %s as new version for %s", newTag.String(), recipe.Name)
-			nextTag = newTag.String()
-		}
-
-		if _, err := tagcmp.Parse(nextTag); err != nil {
-			logrus.Fatalf("invalid version %s specified", nextTag)
-		}
 
 		mainService := "app"
-		label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", nextTag)
+		var services []string
-		if !internal.Dry {
+		hasAppService := false
-			if err := recipe.UpdateLabel("compose.y*ml", mainService, label); err != nil {
+		for _, service := range recipe.Config.Services {
+			services = append(services, service.Name)
+			if service.Name == "app" {
+				hasAppService = true
+				logrus.Debugf("detected app service in '%s'", recipe.Name)
+			}
+		}
+
+		if !hasAppService {
+			logrus.Warnf("no 'app' service defined in '%s'", recipe.Name)
+			var chosenService string
+			prompt := &survey.Select{
+				Message: fmt.Sprintf("what is the main service name for '%s'?", recipe.Name),
+				Options: services,
+			}
+			if err := survey.AskOne(prompt, &chosenService); err != nil {
 				logrus.Fatal(err)
 			}
-		} else {
+			mainService = chosenService
-			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
+		logrus.Debugf("selecting '%s' as the service to sync version labels", mainService)
+
+		label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", nextTag)
+		if err := recipe.UpdateLabel(mainService, label); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("synced label '%s' to service '%s'", label, mainService)
+
 		return nil
 	},
-	BashComplete: autocomplete.RecipeNameComplete,
 }

cli/recipe/upgrade.go

@@ -1,71 +1,43 @@
 package recipe
 
 import (
-	"bufio"
 	"fmt"
-	"os"
-	"path"
 	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-type imgPin struct {
+var recipeUpgradeCommand = &cli.Command{
-	image   string
-	version tagcmp.Tag
-}
-
-var recipeUpgradeCommand = cli.Command{
 	Name:    "upgrade",
-	Aliases: []string{"u"},
 	Usage:   "Upgrade recipe image tags",
+	Aliases: []string{"u"},
 	Description: `
-Parse all image tags within the given <recipe> configuration and prompt with
+This command reads and attempts to parse all image tags within the given
-more recent tags to upgrade to. It will update the relevant compose file tags
+<recipe> configuration and prompt with more recent tags to upgrade to. It will
-on the local file system.
+update the relevant compose file tags on the local file system.
 
 Some image tags cannot be parsed because they do not follow some sort of
 semver-like convention. In this case, all possible tags will be listed and it
 is up to the end-user to decide.
-
-The command is interactive and will show a select input which allows you to
-make a seclection. Use the "?" key to see more help on navigating this
-interface.
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra recipe upgrade
 `,
-	BashComplete: autocomplete.RecipeNameComplete,
+	ArgsUsage: "<recipe>",
-	ArgsUsage:    "<recipe>",
 	Flags: []cli.Flag{
-		internal.DebugFlag,
+		PatchFlag,
-		internal.NoInputFlag,
+		MinorFlag,
-		internal.PatchFlag,
+		MajorFlag,
-		internal.MinorFlag,
-		internal.MajorFlag,
-		internal.AllTagsFlag,
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipeWithPrompt(c)
+		recipe := internal.ValidateRecipe(c)
 
-		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+		bumpType := btoi(Major)*4 + btoi(Minor)*2 + btoi(Patch)
-			logrus.Fatal(err)
-		}
-
-		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
 		if bumpType != 0 {
 			// a bitwise check if the number is a power of 2
 			if (bumpType & (bumpType - 1)) != 0 {
@@ -73,79 +45,46 @@ You may invoke this command in "wizard" mode and be prompted for input:
 			}
 		}
 
-		// check for versions file and load pinned versions
+		for _, service := range recipe.Config.Services {
-		versionsPresent := false
+			catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name)
-		recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
-		versionsPath := path.Join(recipeDir, "versions")
-		var servicePins = make(map[string]imgPin)
-		if _, err := os.Stat(versionsPath); err == nil {
-			logrus.Debugf("found versions file for %s", recipe.Name)
-			file, err := os.Open(versionsPath)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			scanner := bufio.NewScanner(file)
-			for scanner.Scan() {
-				line := scanner.Text()
-				splitLine := strings.Split(line, " ")
-				if splitLine[0] != "pin" || len(splitLine) != 3 {
-					logrus.Fatalf("malformed version pin specification: %s", line)
-				}
-				pinSlice := strings.Split(splitLine[2], ":")
-				pinTag, err := tagcmp.Parse(pinSlice[1])
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				pin := imgPin{
-					image:   pinSlice[0],
-					version: pinTag,
-				}
-				servicePins[splitLine[1]] = pin
-			}
-			if err := scanner.Err(); err != nil {
-				logrus.Error(err)
-			}
-			versionsPresent = true
-		} else {
-			logrus.Debugf("did not find versions file for %s", recipe.Name)
-		}
 
-		for _, service := range recipe.Config.Services {
 			img, err := reference.ParseNormalizedNamed(service.Image)
 			if err != nil {
 				logrus.Fatal(err)
 			}
 
-			regVersions, err := client.GetRegistryTags(img)
+			image := reference.Path(img)
+			regVersions, err := client.GetRegistryTags(image)
 			if err != nil {
 				logrus.Fatal(err)
 			}
+			logrus.Debugf("retrieved '%s' from remote registry for '%s'", regVersions, image)
 
-			image := reference.Path(img)
+			if strings.Contains(image, "library") {
-			logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
-			image = formatter.StripTagMeta(image)
+				// postgres:<tag>, i.e. images which do not have a username in the
+				// first position of the string
+				image = strings.Split(image, "/")[1]
+			}
 
-			switch img.(type) {
+			semverLikeTag := true
-			case reference.NamedTagged:
+			if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
-				if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+				logrus.Debugf("'%s' not considered semver-like", img.(reference.NamedTagged).Tag())
-					logrus.Debugf("%s not considered semver-like", img.(reference.NamedTagged).Tag())
+				semverLikeTag = false
-				}
-			default:
-				logrus.Warnf("unable to read tag for image %s, is it missing? skipping upgrade for %s", image, service.Name)
-				continue
 			}
 
 			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
-			if err != nil {
+			if err != nil && semverLikeTag {
-				logrus.Warnf("unable to parse %s, error was: %s, skipping upgrade for %s", image, err.Error(), service.Name)
+				logrus.Fatal(err)
-				continue
 			}
-
+			logrus.Debugf("parsed '%s' for '%s'", tag, service.Name)
-			logrus.Debugf("parsed %s for %s", tag, service.Name)
 
 			var compatible []tagcmp.Tag
 			for _, regVersion := range regVersions {
-				other, err := tagcmp.Parse(regVersion)
+				other, err := tagcmp.Parse(regVersion.Name)
 				if err != nil {
 					continue // skip tags that cannot be parsed
 				}
@@ -155,21 +94,16 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 
-			logrus.Debugf("detected potential upgradable tags %s for %s", compatible, service.Name)
+			logrus.Debugf("detected potential upgradable tags '%s' for '%s'", compatible, service.Name)
 
 			sort.Sort(tagcmp.ByTagDesc(compatible))
 
-			if len(compatible) == 0 && !internal.AllTags {
+			if len(compatible) == 0 && semverLikeTag {
-				logrus.Info(fmt.Sprintf("no new versions available for %s, assuming %s is the latest (use -a/--all-tags to see all anyway)", image, tag))
+				logrus.Info(fmt.Sprintf("no new versions available for '%s', '%s' is the latest", image, tag))
 				continue // skip on to the next tag and don't update any compose files
 			}
 
-			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name)
+			var compatibleStrings []string
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			compatibleStrings := []string{"skip"}
 			for _, compat := range compatible {
 				skip := false
 				for _, catlVersion := range catlVersions {
@@ -182,85 +116,49 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 
-			logrus.Debugf("detected compatible upgradable tags %s for %s", compatibleStrings, service.Name)
+			logrus.Debugf("detected compatible upgradable tags '%s' for '%s'", compatibleStrings, service.Name)
-
 			var upgradeTag string
-			_, ok := servicePins[service.Name]
+			if bumpType != 0 {
-			if versionsPresent && ok {
+				for _, upTag := range compatible {
-				pinnedTag := servicePins[service.Name].version
+					upElement, err := tag.UpgradeDelta(upTag)
-				if tag.IsLessThan(pinnedTag) {
+					if err != nil {
-					pinnedTagString := pinnedTag.String()
+						return err
-					contains := false
-					for _, v := range compatible {
-						if pinnedTag.IsUpgradeCompatible(v) {
-							contains = true
-							upgradeTag = v.String()
-							break
-						}
 					}
-					if contains {
+					delta := upElement.UpgradeType()
-						logrus.Infof("upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
+					if delta <= bumpType {
-					} else {
+						upgradeTag = upTag.String()
-						logrus.Infof("service %s, image %s pinned to %s, no compatible upgrade found", service.Name, servicePins[service.Name].image, pinnedTagString)
+						break
-						continue
 					}
-				} else {
+				}
-					logrus.Fatalf("service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
+				if upgradeTag == "" {
+					logrus.Warnf("not upgrading from '%s' to '%s' for '%s', because the upgrade type is more serious than what user wants.", tag.String(), compatible[0].String(), image)
 					continue
 				}
 			} else {
-				if bumpType != 0 {
+				msg := fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
-					for _, upTag := range compatible {
+				if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
-						upElement, err := tag.UpgradeDelta(upTag)
+					tag := img.(reference.NamedTagged).Tag()
-						if err != nil {
+					logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of '%s', listing all tags", tag))
-							return err
+					msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
-						}
+					compatibleStrings = []string{}
-						delta := upElement.UpgradeType()
+					for _, regVersion := range regVersions {
-						if delta <= bumpType {
+						compatibleStrings = append(compatibleStrings, regVersion.Name)
-							upgradeTag = upTag.String()
-							break
-						}
-					}
-					if upgradeTag == "" {
-						logrus.Warnf("not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants", tag.String(), compatible[0].String(), image)
-						continue
-					}
-				} else {
-					msg := fmt.Sprintf("upgrade to which tag? (service: %s, image: %s, tag: %s)", service.Name, image, tag)
-					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) || internal.AllTags {
-						tag := img.(reference.NamedTagged).Tag()
-						if !internal.AllTags {
-							logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of %s, listing all tags", tag))
-						}
-						msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
-						compatibleStrings = []string{"skip"}
-						for _, regVersion := range regVersions {
-							compatibleStrings = append(compatibleStrings, regVersion)
-						}
-					}
-
-					prompt := &survey.Select{
-						Message: msg,
-						Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
-						VimMode: true,
-						Options: compatibleStrings,
-					}
-					if err := survey.AskOne(prompt, &upgradeTag); err != nil {
-						logrus.Fatal(err)
 					}
 				}
-			}
+
-			if upgradeTag != "skip" {
+				prompt := &survey.Select{
-				ok, err := recipe.UpdateTag(image, upgradeTag)
+					Message: msg,
-				if err != nil {
+					Options: compatibleStrings,
+				}
+				if err := survey.AskOne(prompt, &upgradeTag); err != nil {
 					logrus.Fatal(err)
 				}
-				if ok {
-					logrus.Infof("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)
-				}
-			} else {
-				logrus.Warnf("not upgrading %s, skipping as requested", image)
 			}
+
+			if err := recipe.UpdateTag(image, upgradeTag); err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Infof("tag upgraded from '%s' to '%s' for '%s'", tag.String(), upgradeTag, image)
 		}
 
 		return nil

cli/recipe/version.go

@@ -1,56 +1,44 @@
 package recipe
 
 import (
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
-	"coopcloud.tech/abra/pkg/formatter"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var recipeVersionCommand = cli.Command{
+var recipeVersionCommand = &cli.Command{
 	Name:      "versions",
-	Aliases:   []string{"v"},
 	Usage:     "List recipe versions",
+	Aliases:   []string{"v"},
 	ArgsUsage: "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c, runtime.WithEnsureRecipeLatest(false))
+		recipe := internal.ValidateRecipe(c)
 
-		catalogue, err := recipePkg.ReadRecipeCatalogue()
+		catalogue, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		recipeMeta, ok := catalogue[recipe.Name]
 		if !ok {
-			logrus.Fatalf("%s recipe doesn't exist?", recipe.Name)
+			logrus.Fatalf("'%s' recipe doesn't exist?", recipe.Name)
 		}
 
-		tableCol := []string{"Version", "Service", "Image", "Tag"}
+		tableCol := []string{"Version", "Service", "Image", "Tag", "Digest"}
 		table := formatter.CreateTable(tableCol)
 
-		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
+		for _, serviceVersion := range recipeMeta.Versions {
-			for tag, meta := range recipeMeta.Versions[i] {
+			for tag, meta := range serviceVersion {
 				for service, serviceMeta := range meta {
-					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag})
+					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag, serviceMeta.Digest})
 				}
 			}
 		}
 
 		table.SetAutoMergeCells(true)
-
+		table.Render()
-		if table.NumLines() > 0 {
-			table.Render()
-		} else {
-			logrus.Fatalf("%s has no published versions?", recipe.Name)
-		}
 
 		return nil
 	},

cli/record/new.go

@@ -1,148 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/dns"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordNewCommand creates a new domain name record.
-var RecordNewCommand = cli.Command{
-	Name:      "new",
-	Usage:     "Create a new domain record",
-	Aliases:   []string{"n"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-		internal.DNSValueFlag,
-		internal.DNSTTLFlag,
-		internal.DNSPriorityFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Create a new domain name record for a specific zone.
-
-You must specify a zone (e.g. example.com) under which your domain name records
-are listed. This zone must already be created on your provider account.
-
-Example:
-
-    abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSValueFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		ttl, err := dns.GetTTL(internal.DNSTTL)
-		if err != nil {
-			return err
-		}
-
-		record := libdns.Record{
-			Type:  internal.DNSType,
-			Name:  internal.DNSName,
-			Value: internal.DNSValue,
-			TTL:   ttl,
-		}
-
-		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
-			record.Priority = internal.DNSPriority
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, existingRecord := range records {
-			if existingRecord.Type == record.Type &&
-				existingRecord.Name == record.Name &&
-				existingRecord.Value == record.Value {
-				logrus.Fatalf("%s record for %s already exists?", record.Type, zone)
-			}
-		}
-
-		createdRecords, err := provider.SetRecords(
-			context.Background(),
-			zone,
-			[]libdns.Record{record},
-		)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(createdRecords) == 0 {
-			logrus.Fatal("provider library reports that no record was created?")
-		}
-
-		createdRecord := createdRecords[0]
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := createdRecord.Value
-		if len(createdRecord.Value) > 30 {
-			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
-		}
-
-		table.Append([]string{
-			createdRecord.Type,
-			createdRecord.Name,
-			value,
-			createdRecord.TTL.String(),
-			strconv.Itoa(createdRecord.Priority),
-		})
-
-		table.Render()
-
-		logrus.Info("record created")
-
-		return nil
-	},
-}

cli/record/record.go

@@ -1,37 +0,0 @@
-package record
-
-import (
-	"github.com/urfave/cli"
-)
-
-// RecordCommand supports managing DNS entries.
-var RecordCommand = cli.Command{
-	Name:      "record",
-	Usage:     "Manage domain name records",
-	Aliases:   []string{"rc"},
-	ArgsUsage: "<record>",
-	Description: `
-Manage domain name records via 3rd party providers such as Gandi DNS. It
-supports listing, creating and removing all types of records that you might
-need for managing Co-op Cloud apps.
-
-The following providers are supported:
-
-    Gandi DNS https://www.gandi.net
-
-You need an account with such a provider already. Typically, you need to
-provide an API token on the Abra command-line when using these commands so that
-you can authenticate with your provider account.
-
-New providers can be integrated, we welcome change sets. See the underlying DNS
-library documentation for more. It supports many existing providers and allows
-to implement new provider support easily.
-
-    https://pkg.go.dev/github.com/libdns/libdns
-`,
-	Subcommands: []cli.Command{
-		RecordListCommand,
-		RecordNewCommand,
-		RecordRemoveCommand,
-	},
-}

cli/record/remove.go

@@ -1,136 +0,0 @@
-package record
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"coopcloud.tech/abra/cli/internal"
-	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/libdns/gandi"
-	"github.com/libdns/libdns"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// RecordRemoveCommand lists domains.
-var RecordRemoveCommand = cli.Command{
-	Name:      "remove",
-	Usage:     "Remove a domain name record",
-	Aliases:   []string{"rm"},
-	ArgsUsage: "<zone>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.DNSProviderFlag,
-		internal.DNSTypeFlag,
-		internal.DNSNameFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Remove a domain name record for a specific zone.
-
-It uses the type of record and name to match existing records and choose one
-for deletion. You must specify a zone (e.g. example.com) under which your
-domain name records are listed. This zone must already be created on your
-provider account.
-
-Example:
-
-    abra record remove foo.com -p gandi -t A -n myapp
-
-You may also invoke this command in "wizard" mode and be prompted for input:
-
-    abra record rm
-`,
-	Action: func(c *cli.Context) error {
-		zone, err := internal.EnsureZoneArgument(c)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSProvider(); err != nil {
-			logrus.Fatal(err)
-		}
-
-		var provider gandi.Provider
-		switch internal.DNSProvider {
-		case "gandi":
-			provider, err = gandiPkg.New()
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		default:
-			logrus.Fatalf("%s is not a supported DNS provider", internal.DNSProvider)
-		}
-
-		if err := internal.EnsureDNSTypeFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := internal.EnsureDNSNameFlag(c); err != nil {
-			logrus.Fatal(err)
-		}
-
-		records, err := provider.GetRecords(context.Background(), zone)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		var toDelete libdns.Record
-		for _, record := range records {
-			if record.Type == internal.DNSType && record.Name == internal.DNSName {
-				toDelete = record
-				break
-			}
-		}
-
-		if (libdns.Record{}) == toDelete {
-			logrus.Fatal("provider library reports no matching record?")
-		}
-
-		tableCol := []string{"type", "name", "value", "TTL", "priority"}
-		table := formatter.CreateTable(tableCol)
-
-		value := toDelete.Value
-		if len(toDelete.Value) > 30 {
-			value = fmt.Sprintf("%s...", toDelete.Value[:30])
-		}
-
-		table.Append([]string{
-			toDelete.Type,
-			toDelete.Name,
-			value,
-			toDelete.TTL.String(),
-			strconv.Itoa(toDelete.Priority),
-		})
-
-		table.Render()
-
-		if !internal.NoInput {
-			response := false
-			prompt := &survey.Confirm{
-				Message: "continue with record deletion?",
-			}
-
-			if err := survey.AskOne(prompt, &response); err != nil {
-				return err
-			}
-
-			if !response {
-				logrus.Fatal("exiting as requested")
-			}
-		}
-
-		_, err = provider.DeleteRecords(context.Background(), zone, []libdns.Record{toDelete})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Info("record successfully deleted")
-
-		return nil
-	},
-}

cli/server/add.go

@@ -1,180 +1,150 @@
 package server
 
 import (
+	"context"
 	"errors"
-	"os"
+	"os/exec"
-	"path/filepath"
+	"os/user"
+	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	contextPkg "coopcloud.tech/abra/pkg/context"
-	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/server"
-	sshPkg "coopcloud.tech/abra/pkg/ssh"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 var local bool
 var localFlag = &cli.BoolFlag{
-	Name:        "local, l",
+	Name:        "local",
-	Usage:       "Use local server",
+	Aliases:     []string{"L"},
+	Value:       false,
+	Usage:       "Set up the local server",
 	Destination: &local,
 }
 
-func cleanUp(domainName string) {
+var serverAddCommand = &cli.Command{
-	if domainName != "default" {
+	Name:  "add",
-		logrus.Infof("cleaning up context for %s", domainName)
+	Usage: "Add a new server",
-		if err := client.DeleteContext(domainName); err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	logrus.Infof("attempting to clean up server directory for %s", domainName)
-
-	serverDir := filepath.Join(config.SERVERS_DIR, domainName)
-	files, err := config.GetAllFilesInDirectory(serverDir)
-	if err != nil {
-		logrus.Fatalf("unable to list files in %s: %s", serverDir, err)
-	}
-
-	if len(files) > 0 {
-		logrus.Warnf("aborting clean up of %s because it is not empty", serverDir)
-		return
-	}
-
-	if err := os.RemoveAll(serverDir); err != nil {
-		logrus.Fatal(err)
-	}
-}
-
-// newContext creates a new internal Docker context for a server. This is how
-// Docker manages SSH connection details. These are stored to disk in
-// ~/.docker. Abra can manage this completely for the user, so it's an
-// implementation detail.
-func newContext(c *cli.Context, domainName, username, port string) error {
-	store := contextPkg.NewDefaultDockerContextStore()
-	contexts, err := store.Store.List()
-	if err != nil {
-		return err
-	}
-
-	for _, context := range contexts {
-		if context.Name == domainName {
-			logrus.Debugf("context for %s already exists", domainName)
-			return nil
-		}
-	}
-
-	logrus.Debugf("creating context with domain %s, username %s and port %s", domainName, username, port)
-
-	if err := client.CreateContext(domainName, username, port); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// createServerDir creates the ~/.abra/servers/... directory for a new server.
-func createServerDir(domainName string) error {
-	if err := server.CreateServerDir(domainName); err != nil {
-		if !os.IsExist(err) {
-			return err
-		}
-		logrus.Debugf("server dir for %s already created", domainName)
-	}
-
-	return nil
-}
-
-var serverAddCommand = cli.Command{
-	Name:    "add",
-	Aliases: []string{"a"},
-	Usage:   "Add a server to your configuration",
 	Description: `
-Add a new server to your configuration so that it can be managed by Abra.
+This command adds a new server that abra will communicate with, to deploy apps.
-
-Abra uses the SSH command-line to discover connection details for your server.
-It is advised to configure an entry per-host in your ~/.ssh/config for each
-server. For example:
-
-Host example.com
-  Hostname example.com
-  User exampleUser
-  Port 12345
-  IdentityFile ~/.ssh/example@somewhere
-
-Abra can then load SSH connection details from this configuratiion with:
-
-    abra server add example.com
 
 If "--local" is passed, then Abra assumes that the current local server is
-intended as the target server. This is useful when you want to have your entire
+intended as the target server.
-Co-op Cloud config located on the server itself, and not on your local
+
-developer machine.
+Otherwise, you may specify a remote server. The <domain> argument must be a
+publicy accessible domain name which points to your server. You should have SSH
+access to this server, Abra will assume port 22 and will use your current
+system username to make an initial connection. You can use the <user> and
+<port> arguments to adjust this.
+
+For example:
+
+    abra server add varia.zone glodemodem 12345
+
+Abra will construct the following SSH connection string then:
+
+    ssh://globemodem@varia.zone:12345
+
+All communication between Abra and the server will use this SSH connection.
 `,
+	Aliases: []string{"a"},
 	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
 		localFlag,
 	},
-	Before:    internal.SubCommandBefore,
+	ArgsUsage: "<domain> [<user>] [<port>]",
-	ArgsUsage: "<domain>",
 	Action: func(c *cli.Context) error {
-		if len(c.Args()) > 0 && local || !internal.ValidateSubCmdFlags(c) {
+
-			err := errors.New("cannot use <domain> and --local together")
+		if c.Args().Len() == 2 && !local {
+			err := errors.New("missing arguments <domain> or '--local'")
 			internal.ShowSubcommandHelpAndError(c, err)
 		}
 
-		var domainName string
+		if c.Args().Get(2) != "" && local {
-		if local {
+			err := errors.New("cannot use '<domain>' and '--local' together")
-			domainName = "default"
+			internal.ShowSubcommandHelpAndError(c, err)
-		} else {
-			domainName = internal.ValidateDomain(c)
 		}
 
+		domainName := "default"
+
 		if local {
-			if err := createServerDir(domainName); err != nil {
+			if err := server.CreateServerDir(domainName); err != nil {
 				logrus.Fatal(err)
 			}
-
+			if _, err := exec.LookPath("docker"); err != nil {
-			logrus.Infof("attempting to create client for %s", domainName)
+				return errors.New("docker command not found on $PATH, is it installed?")
-			if _, err := client.New(domainName); err != nil {
-				cleanUp(domainName)
-				logrus.Fatal(err)
 			}
-
+			logrus.Info("local server has been added")
-			logrus.Info("local server added")
-
 			return nil
 		}
 
-		if _, err := dns.EnsureIPv4(domainName); err != nil {
+		domainName = internal.ValidateDomain(c)
-			logrus.Fatal(err)
+
+		var username string
+		var port string
+
+		username = c.Args().Get(1)
+		if username == "" {
+			systemUser, err := user.Current()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			username = systemUser.Username
 		}
 
-		if err := createServerDir(domainName); err != nil {
+		port = c.Args().Get(2)
-			logrus.Fatal(err)
+		if port == "" {
+			port = "22"
 		}
 
-		hostConfig, err := sshPkg.GetHostConfig(domainName)
+		store := client.NewDefaultDockerContextStore()
+		contexts, err := store.Store.List()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
+		for _, context := range contexts {
+			if context.Name == domainName {
+				logrus.Fatalf("server at '%s' already exists?", domainName)
+			}
+		}
+
+		logrus.Debugf("creating context with domain '%s', username '%s' and port '%s'", domainName, username, port)
+
+		if err := client.CreateContext(domainName, username, port); err != nil {
 			logrus.Fatal(err)
 		}
 
-		logrus.Infof("attempting to create client for %s", domainName)
+		ctx := context.Background()
-		if _, err := client.New(domainName); err != nil {
+		cl, err := client.New(domainName)
-			cleanUp(domainName)
+		if err != nil {
-			logrus.Debugf("failed to construct client for %s, saw %s", domainName, err.Error())
+			logrus.Warn("cleaning up context due to connection failure")
-			logrus.Fatal(sshPkg.Fatal(domainName, err))
+			if err := client.DeleteContext(domainName); err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Fatal(err)
 		}
 
-		logrus.Infof("%s added", domainName)
+		if _, err := cl.Info(ctx); err != nil {
+			if strings.Contains(err.Error(), "command not found") {
+				logrus.Fatalf("docker is not installed on '%s'?", domainName)
+			} else {
+				logrus.Warn("cleaning up context due to connection failure")
+				if err := client.DeleteContext(domainName); err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Fatalf("unable to make a connection to '%s'?", domainName)
+			}
+			logrus.Debug(err)
+		}
+
+		logrus.Debugf("remote connection to '%s' is definitely up", domainName)
+
+		if err := server.CreateServerDir(domainName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("server at '%s' has been added", domainName)
 
 		return nil
 	},

cli/server/init.go

@@ -0,0 +1,81 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var serverInitCommand = &cli.Command{
+	Name:      "init",
+	Usage:     "Initialise server for deploying apps",
+	Aliases:   []string{"i"},
+	HideHelp:  true,
+	ArgsUsage: "<domain>",
+	Description: `
+Initialise swarm mode on the target <domain>.
+
+This initialisation explicitly chooses the "single host swarm" mode which uses
+the default IPv4 address as the advertising address. This can be re-configured
+later for more advanced use cases.
+`,
+	Action: func(c *cli.Context) error {
+		domainName := internal.ValidateDomain(c)
+
+		cl, err := client.New(domainName)
+		if err != nil {
+			return err
+		}
+
+		// https://www.privacy-handbuch.de/handbuch_93d.htm
+		freifunkDNS := "5.1.66.255:53"
+
+		resolver := &net.Resolver{
+			PreferGo: false,
+			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+				d := net.Dialer{
+					Timeout: time.Millisecond * time.Duration(10000),
+				}
+				// comrade librehosters DNS resolver https://snopyta.org/service/dns/
+				return d.DialContext(ctx, "udp", freifunkDNS)
+			},
+		}
+		logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
+
+		ctx := context.Background()
+		ips, err := resolver.LookupIPAddr(ctx, domainName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(ips) == 0 {
+			return fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
+		}
+		ipv4 := ips[0].IP.To4().String()
+
+		initReq := swarm.InitRequest{
+			ListenAddr:    "0.0.0.0:2377",
+			AdvertiseAddr: ipv4,
+		}
+		if _, err := cl.SwarmInit(ctx, initReq); err != nil {
+			return err
+		}
+		logrus.Debugf("initialised swarm on '%s'", domainName)
+
+		netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
+		if _, err := cl.NetworkCreate(ctx, "proxy", netOpts); err != nil {
+			return err
+		}
+		logrus.Debug("swarm overlay network 'proxy' created")
+
+		return nil
+	},
+}

cli/server/list.go

@@ -3,47 +3,30 @@ package server
 import (
 	"strings"
 
-	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/context"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/docker/cli/cli/connhelper/ssh"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var problemsFilter bool
+var serverListCommand = &cli.Command{
-
+	Name:      "list",
-var problemsFilterFlag = &cli.BoolFlag{
+	Aliases:   []string{"ls"},
-	Name:        "problems, p",
+	Usage:     "List managed servers",
-	Usage:       "Show only servers with potential connection problems",
+	ArgsUsage: " ",
-	Destination: &problemsFilter,
+	HideHelp:  true,
-}
-
-var serverListCommand = cli.Command{
-	Name:    "list",
-	Aliases: []string{"ls"},
-	Usage:   "List managed servers",
-	Flags: []cli.Flag{
-		problemsFilterFlag,
-		internal.DebugFlag,
-		internal.MachineReadableFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		dockerContextStore := context.NewDefaultDockerContextStore()
+		dockerContextStore := client.NewDefaultDockerContextStore()
 		contexts, err := dockerContextStore.Store.List()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		tableColumns := []string{"name", "host", "user", "port"}
+		tableColumns := []string{"Name", "Connection"}
 		table := formatter.CreateTable(tableColumns)
-		if internal.MachineReadable {
+		defer table.Render()
-			defer table.JSONRender()
+
-		} else {
-			defer table.Render()
-		}
 		serverNames, err := config.ReadServerNames()
 		if err != nil {
 			logrus.Fatal(err)
@@ -52,36 +35,23 @@ var serverListCommand = cli.Command{
 		for _, serverName := range serverNames {
 			var row []string
 			for _, ctx := range contexts {
-				endpoint, err := context.GetContextEndpoint(ctx)
+				endpoint, err := client.GetContextEndpoint(ctx)
 				if err != nil && strings.Contains(err.Error(), "does not exist") {
 					// No local context found, we can continue safely
 					continue
 				}
-
 				if ctx.Name == serverName {
-					sp, err := ssh.ParseURL(endpoint)
+					row = []string{serverName, endpoint}
-					if err != nil {
-						logrus.Fatal(err)
-					}
-					row = []string{serverName, sp.Host, sp.User, sp.Port}
 				}
 			}
-
 			if len(row) == 0 {
 				if serverName == "default" {
-					row = []string{serverName, "local", "n/a", "n/a"}
+					row = []string{serverName, "local"}
 				} else {
-					row = []string{serverName, "unknown", "unknown", "unknown"}
+					row = []string{serverName, "unknown"}
 				}
 			}
-
+			table.Append(row)
-			if problemsFilter {
-				if row[1] == "unknown" {
-					table.Append(row)
-				}
-			} else {
-				table.Append(row)
-			}
 		}
 
 		return nil

cli/server/new.go

@@ -2,259 +2,219 @@ package server
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"strings"
 
+	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/libcapsul"
-	"github.com/AlecAivazis/survey/v2"
 	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-func newHetznerCloudVPS(c *cli.Context) error {
+var hetznerCloudType string
-	if err := internal.EnsureNewHetznerCloudVPSFlags(c); err != nil {
+var hetznerCloudImage string
-		return err
+var hetznerCloudSSHKeys cli.StringSlice
-	}
+var hetznerCloudLocation string
-
+var hetznerCloudAPIToken string
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
+var serverNewHetznerCloudCommand = &cli.Command{
-
+	Name:      "hetzner",
-	var sshKeysRaw []string
+	Usage:     "Create a new Hetzner virtual server",
-	var sshKeys []*hcloud.SSHKey
+	ArgsUsage: "<name>",
-	for _, sshKey := range c.StringSlice("hetzner-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-
-		sshKey, _, err := client.SSHKey.GetByName(context.Background(), sshKey)
-		if err != nil {
-			return err
-		}
-		sshKeys = append(sshKeys, sshKey)
-		sshKeysRaw = append(sshKeysRaw, sshKey.Name)
-	}
-
-	serverOpts := hcloud.ServerCreateOpts{
-		Name:       internal.HetznerCloudName,
-		ServerType: &hcloud.ServerType{Name: internal.HetznerCloudType},
-		Image:      &hcloud.Image{Name: internal.HetznerCloudImage},
-		SSHKeys:    sshKeys,
-		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
-	}
-
-	sshKeyIDs := strings.Join(sshKeysRaw, "\n")
-	if sshKeyIDs == "" {
-		sshKeyIDs = "N/A (password auth)"
-	}
-
-	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.HetznerCloudName,
-		internal.HetznerCloudType,
-		internal.HetznerCloudImage,
-		sshKeyIDs,
-		internal.HetznerCloudLocation,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	res, _, err := client.Server.Create(context.Background(), serverOpts)
-	if err != nil {
-		return err
-	}
-
-	var rootPassword string
-	if len(sshKeys) > 0 {
-		rootPassword = "N/A (using SSH keys)"
-	} else {
-		rootPassword = res.RootPassword
-	}
-
-	ip := res.Server.PublicNet.IPv4.IP.String()
-
-	fmt.Println(fmt.Sprintf(`
-Your new Hetzner Cloud VPS has successfully been created! Here are the details:
-
-    name: %s
-    IP address: %s
-    root password: %s
-
-You can access this new VPS via SSH using the following command:
-
-    ssh root@%s
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record (manually
-or by using "abra record new") and add the server to your Abra configuration
-("abra server add") to have a working server that you can deploy Co-op Cloud
-apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A %s
-    *  1800 IN A %s
-	`,
-		internal.HetznerCloudName, ip, rootPassword,
-		ip, ip, ip,
-	))
-
-	return nil
-}
-
-func newCapsulVPS(c *cli.Context) error {
-	if err := internal.EnsureNewCapsulVPSFlags(c); err != nil {
-		return err
-	}
-
-	capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", internal.CapsulInstanceURL)
-
-	var sshKeys []string
-	for _, sshKey := range c.StringSlice("capsul-ssh-keys") {
-		if sshKey == "" {
-			continue
-		}
-		sshKeys = append(sshKeys, sshKey)
-	}
-
-	tableColumns := []string{"instance", "name", "type", "image", "ssh-keys"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		internal.CapsulInstanceURL,
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		strings.Join(sshKeys, "\n"),
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with capsul creation?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	capsulClient := libcapsul.New(capsulCreateURL, internal.CapsulAPIToken)
-	resp, err := capsulClient.Create(
-		internal.CapsulName,
-		internal.CapsulType,
-		internal.CapsulImage,
-		sshKeys,
-	)
-	if err != nil {
-		return err
-	}
-
-	fmt.Println(fmt.Sprintf(`
-Your new Capsul has successfully been created! Here are the details:
-
-    Capsul name: %s
-    Capsul ID:   %v
-
-You will need to log into your Capsul instance web interface to retrieve the IP
-address. You can learn all about how to get SSH access to your new Capsul on:
-
-    %s/about-ssh
-
-Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
-not list this server)! You will need to assign a domain name record (manually
-or by using "abra record new") and add the server to your Abra configuration
-("abra server add") to have a working server that you can deploy Co-op Cloud
-apps to.
-
-When setting up domain name records, you probably want to set up the following
-2 A records. This supports deploying apps to your root domain (e.g.
-example.com) and other apps on sub-domains (e.g. foo.example.com,
-bar.example.com).
-
-    @  1800 IN A <your-capsul-ip>
-    *  1800 IN A <your-capsul-ip>
-	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
-
-	return nil
-}
-
-var serverNewCommand = cli.Command{
-	Name:    "new",
-	Aliases: []string{"n"},
-	Usage:   "Create a new server using a 3rd party provider",
 	Description: `
-Create a new server via a 3rd party provider.
+Create a new Hetzner virtual server.
 
-The following providers are supported:
+This command uses the uses the Hetzner Cloud API bindings to send a server
+creation request. You must already have a Hetzner Cloud account and an account
+API token before using this command.
 
-		Capsul https://git.cyberia.club/Cyberia/capsul-flask
+Your token can be loaded from the environment using the HCLOUD_TOKEN
-    Hetzner Cloud https://docs.hetzner.com/cloud
+environment variable or otherwise passing the "--env/-e" flag.
-
-You may invoke this command in "wizard" mode and be prompted for input:
-
-    abra record new
-
-API tokens are read from the environment if specified, e.g.
-
-    export HCLOUD_TOKEN=...
 `,
 	Flags: []cli.Flag{
-		internal.DebugFlag,
+		&cli.StringFlag{
-		internal.NoInputFlag,
+			Name:        "type",
-		internal.ServerProviderFlag,
+			Aliases:     []string{"t"},
-
+			Usage:       "Server type",
-		// Capsul
+			Destination: &hetznerCloudType,
-		internal.CapsulInstanceURLFlag,
+			Value:       "cx11",
-		internal.CapsulTypeFlag,
+		},
-		internal.CapsulImageFlag,
+		&cli.StringFlag{
-		internal.CapsulSSHKeysFlag,
+			Name:        "image",
-		internal.CapsulAPITokenFlag,
+			Aliases:     []string{"i"},
-
+			Usage:       "Image type",
-		// Hetzner
+			Value:       "debian-10",
-		internal.HetznerCloudNameFlag,
+			Destination: &hetznerCloudImage,
-		internal.HetznerCloudTypeFlag,
+		},
-		internal.HetznerCloudImageFlag,
+		&cli.StringSliceFlag{
-		internal.HetznerCloudSSHKeysFlag,
+			Name:        "ssh-keys",
-		internal.HetznerCloudLocationFlag,
+			Aliases:     []string{"s"},
-		internal.HetznerCloudAPITokenFlag,
+			Usage:       "SSH keys",
+			Destination: &hetznerCloudSSHKeys,
+		},
+		&cli.StringFlag{
+			Name:        "location",
+			Aliases:     []string{"l"},
+			Usage:       "Server location",
+			Value:       "hel1",
+			Destination: &hetznerCloudLocation,
+		},
+		&cli.StringFlag{
+			Name:        "token",
+			Aliases:     []string{"T"},
+			Usage:       "Hetzner Cloud API token",
+			EnvVars:     []string{"HCLOUD_TOKEN"},
+			Destination: &hetznerCloudAPIToken,
+		},
 	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		if err := internal.EnsureServerProvider(); err != nil {
+		name := c.Args().First()
+		if name == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no name provided"))
+		}
+
+		if hetznerCloudAPIToken == "" {
+			logrus.Fatal("Hetzner Cloud API token is missing")
+		}
+
+		ctx := context.Background()
+		client := hcloud.NewClient(hcloud.WithToken(hetznerCloudAPIToken))
+
+		logrus.Debugf("successfully created hetzner cloud API client")
+
+		var sshKeys []*hcloud.SSHKey
+		for _, sshKey := range c.StringSlice("ssh-keys") {
+			sshKey, _, err := client.SSHKey.GetByName(ctx, sshKey)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			sshKeys = append(sshKeys, sshKey)
+		}
+
+		serverOpts := hcloud.ServerCreateOpts{
+			Name:       name,
+			ServerType: &hcloud.ServerType{Name: hetznerCloudType},
+			Image:      &hcloud.Image{Name: hetznerCloudImage},
+			SSHKeys:    sshKeys,
+			Location:   &hcloud.Location{Name: hetznerCloudLocation},
+		}
+		res, _, err := client.Server.Create(ctx, serverOpts)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		switch internal.ServerProvider {
+		logrus.Debugf("new server '%s' created", name)
-		case "capsul":
+
-			if err := newCapsulVPS(c); err != nil {
+		tableColumns := []string{"Name", "IPv4", "Root Password"}
-				logrus.Fatal(err)
+		table := formatter.CreateTable(tableColumns)
-			}
+
-		case "hetzner-cloud":
+		if len(sshKeys) > 0 {
-			if err := newHetznerCloudVPS(c); err != nil {
+			table.Append([]string{name, res.Server.PublicNet.IPv4.IP.String(), "N/A (using SSH keys)"})
-				logrus.Fatal(err)
+		} else {
-			}
+			table.Append([]string{name, res.Server.PublicNet.IPv4.IP.String(), res.RootPassword})
 		}
 
+		table.Render()
+
 		return nil
 	},
 }
+
+var capsulInstance string
+var capsulType string
+var capsulImage string
+var capsulSSHKey string
+var capsulAPIToken string
+var serverNewCapsulCommand = &cli.Command{
+	Name:      "capsul",
+	Usage:     "Create a new Capsul virtual server",
+	ArgsUsage: "<name>",
+	Description: `
+Create a new Capsul virtual server.
+
+This command uses the uses the Capsul API bindings of your chosen instance to
+send a server creation request. You must already have an account on your chosen
+Capsul instance before using this command.
+
+Your token can be loaded from the environment using the CAPSUL_TOKEN
+environment variable or otherwise passing the "--env/-e" flag.
+`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:        "instance",
+			Aliases:     []string{"I"},
+			Usage:       "Capsul instance",
+			Destination: &capsulInstance,
+			Value:       "yolo.servers.coop",
+		},
+		&cli.StringFlag{
+			Name:        "type",
+			Aliases:     []string{"t"},
+			Usage:       "Server type",
+			Value:       "f1-xs",
+			Destination: &capsulType,
+		},
+		&cli.StringFlag{
+			Name:        "image",
+			Aliases:     []string{"i"},
+			Usage:       "Image type",
+			Value:       "debian10",
+			Destination: &capsulImage,
+		},
+		&cli.StringFlag{
+			Name:        "ssh-key",
+			Aliases:     []string{"s"},
+			Usage:       "SSH key",
+			Value:       "",
+			Destination: &capsulSSHKey,
+		},
+		&cli.StringFlag{
+			Name:        "token",
+			Aliases:     []string{"T"},
+			Usage:       "Capsul instance API token",
+			EnvVars:     []string{"CAPSUL_TOKEN"},
+			Destination: &capsulAPIToken,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		capsulName := c.Args().First()
+		if capsulName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no name provided"))
+		}
+
+		if capsulAPIToken == "" {
+			logrus.Fatal("Capsul API token is missing")
+		}
+
+		capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", capsulInstance)
+
+		capsulClient := libcapsul.New(capsulCreateURL, capsulAPIToken)
+		resp, err := capsulClient.Create(capsulName, capsulType, capsulImage, capsulSSHKey)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableColumns := []string{"Name", "ID"}
+		table := formatter.CreateTable(tableColumns)
+		table.Append([]string{capsulName, resp.ID})
+		table.Render()
+
+		return nil
+	},
+}
+
+var serverNewCommand = &cli.Command{
+	Name:    "new",
+	Aliases: []string{"n"},
+	Usage:   "Create a new server using a 3rd party provider",
+	Description: `
+Use a provider plugin to create a new server which can then be used to house a
+new Co-op Cloud installation.
+`,
+	ArgsUsage: "<provider>",
+	Subcommands: []*cli.Command{
+		serverNewHetznerCloudCommand,
+		serverNewCapsulCommand,
+	},
+}

cli/server/remove.go

@@ -1,183 +1,29 @@
 package server
 
 import (
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/hetznercloud/hcloud-go/hcloud"
 	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var rmServer bool
+var serverRemoveCommand = &cli.Command{
-var rmServerFlag = &cli.BoolFlag{
+	Name:    "remove",
-	Name:        "server, s",
+	Aliases: []string{"rm"},
-	Usage:       "remove the actual server also",
+	Usage:   "Remove a server",
-	Destination: &rmServer,
-}
-
-func rmHetznerCloudVPS(c *cli.Context) error {
-	if internal.HetznerCloudName == "" && !internal.NoInput {
-		prompt := &survey.Input{
-			Message: "specify hetzner cloud VPS name",
-		}
-		if err := survey.AskOne(prompt, &internal.HetznerCloudName); err != nil {
-			return err
-		}
-	}
-
-	if internal.HetznerCloudAPIToken == "" && !internal.NoInput {
-		token, ok := os.LookupEnv("HCLOUD_TOKEN")
-		if !ok {
-			prompt := &survey.Input{
-				Message: "specify hetzner cloud API token",
-			}
-			if err := survey.AskOne(prompt, &internal.HetznerCloudAPIToken); err != nil {
-				return err
-			}
-		} else {
-			internal.HetznerCloudAPIToken = token
-		}
-	}
-
-	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
-
-	server, _, err := client.Server.Get(context.Background(), internal.HetznerCloudName)
-	if err != nil {
-		return err
-	}
-
-	if server == nil {
-		logrus.Fatalf("library provider reports that %s doesn't exist?", internal.HetznerCloudName)
-	}
-
-	fmt.Println(fmt.Sprintf(`
-You have requested that Abra delete the following server (%s). Please be
-absolutely sure that this is indeed the server that you would like to have
-removed. There will be no going back once you confirm, the server will be
-destroyed.
-`, server.Name))
-
-	tableColumns := []string{"name", "type", "image", "location"}
-	table := formatter.CreateTable(tableColumns)
-	table.Append([]string{
-		server.Name,
-		server.ServerType.Name,
-		server.Image.Name,
-		server.Datacenter.Name,
-	})
-	table.Render()
-
-	response := false
-	prompt := &survey.Confirm{
-		Message: "continue with hetzner cloud VPS removal?",
-	}
-
-	if err := survey.AskOne(prompt, &response); err != nil {
-		return err
-	}
-
-	if !response {
-		logrus.Fatal("exiting as requested")
-	}
-
-	_, err = client.Server.Delete(context.Background(), server)
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof("%s has been deleted from your hetzner cloud account", internal.HetznerCloudName)
-
-	return nil
-}
-
-var serverRemoveCommand = cli.Command{
-	Name:      "remove",
-	Aliases:   []string{"rm"},
-	ArgsUsage: "[<server>]",
-	Usage:     "Remove a managed server",
 	Description: `
-Remova a server from Abra management.
+This does not destroy the actual server. It simply removes it from Abra
-
+internal bookkeeping so that it is not managed any more.
-Depending on whether you used a 3rd party provider to create this server ("abra
-server new"), you can also destroy the virtual server as well. Pass
-"--server/-s" to tell Abra to try to delete this VPS.
-
-Otherwise, Abra will remove the internal bookkeeping (~/.abra/servers/...) and
-underlying client connection context. This server will then be lost in time,
-like tears in rain.
 `,
-	Flags: []cli.Flag{
+	HideHelp: true,
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		rmServerFlag,
-		internal.ServerProviderFlag,
-
-		// Hetzner
-		internal.HetznerCloudNameFlag,
-		internal.HetznerCloudAPITokenFlag,
-	},
-	Before: internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
-		serverName := internal.ValidateServer(c)
+		domainName := internal.ValidateDomain(c)
 
-		warnMsg := `Did not pass -s/--server for actual server deletion, prompting!
+		if err := client.DeleteContext(domainName); err != nil {
-
-Abra doesn't currently know if it helped you create this server with one of the
-3rd party integrations (e.g. Capsul). You have a choice here to actually,
-really and finally destroy this server using those integrations. If you want to
-do this, choose Yes.
-
-If you just want to remove the server config files & context, choose No.
-`
-
-		if !rmServer {
-			logrus.Warn(fmt.Sprintf(warnMsg))
-
-			response := false
-			prompt := &survey.Confirm{
-				Message: "delete actual live server?",
-			}
-			if err := survey.AskOne(prompt, &response); err != nil {
-				logrus.Fatal(err)
-			}
-			if response {
-				logrus.Info("setting -s/--server and attempting to remove actual server")
-				rmServer = true
-			}
-		}
-
-		if rmServer {
-			if err := internal.EnsureServerProvider(); err != nil {
-				logrus.Fatal(err)
-			}
-
-			switch internal.ServerProvider {
-			case "capsul":
-				logrus.Warn("capsul provider does not support automatic removal yet, sorry!")
-			case "hetzner-cloud":
-				if err := rmHetznerCloudVPS(c); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
-		if err := client.DeleteContext(serverName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, serverName)); err != nil {
+		logrus.Infof("server at '%s' has been forgotten", domainName)
-			logrus.Fatal(err)
-		}
-
-		logrus.Infof("server at %s has been lost in time, like tears in rain", serverName)
 
 		return nil
 	},

cli/server/server.go

@@ -1,24 +1,24 @@
 package server
 
 import (
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // ServerCommand defines the `abra server` command and its subcommands
-var ServerCommand = cli.Command{
+var ServerCommand = &cli.Command{
 	Name:    "server",
 	Aliases: []string{"s"},
-	Usage:   "Manage servers",
+	Usage:   "Manage servers via 3rd party providers",
 	Description: `
-Create, manage and remove servers using 3rd party integrations. 
+Manage the lifecycle of a server.
 
-Servers can be created from scratch using the "abra server new" command. If you
+These commands support creating new servers using 3rd party integrations,
-already have a server, you can add it to your configuration using "abra server
+initialising existing servers to support Co-op Cloud deployments and managing
-add". Abra can provision servers so that they are ready to deploy Co-op Cloud
+the connections to those servers.
-recipes, see available flags on "abra server add" for more.
 `,
-	Subcommands: []cli.Command{
+	Subcommands: []*cli.Command{
 		serverNewCommand,
+		serverInitCommand,
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,

cli/updater/updater.go

@@ -1,497 +0,0 @@
-package updater
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/lint"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/runtime"
-	"coopcloud.tech/abra/pkg/upstream/convert"
-	"coopcloud.tech/abra/pkg/upstream/stack"
-	"coopcloud.tech/tagcmp"
-	composetypes "github.com/docker/cli/cli/compose/types"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerclient "github.com/docker/docker/client"
-
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-const SERVER = "localhost"
-
-var majorUpdate bool
-var majorFlag = &cli.BoolFlag{
-	Name:        "major, m",
-	Usage:       "Also check for major updates",
-	Destination: &majorUpdate,
-}
-
-var updateAll bool
-var allFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Usage:       "Update all deployed apps",
-	Destination: &updateAll,
-}
-
-// Notify checks for available upgrades
-var Notify = cli.Command{
-	Name:    "notify",
-	Aliases: []string{"n"},
-	Usage:   "Check for available upgrades",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		majorFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-It reads the deployed app versions and looks for new versions in the recipe
-catalogue. If a new patch/minor version is available, a notification is
-printed. To include major versions use the --major flag.
-`,
-	Action: func(c *cli.Context) error {
-		cl, err := client.New("default")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		stacks, err := stack.GetStacks(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, stackInfo := range stacks {
-			stackName := stackInfo.Name
-			recipeName, err := getLabel(cl, stackName, "recipe")
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if recipeName != "" {
-				_, err = getLatestUpgrade(cl, stackName, recipeName)
-				if err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
-		return nil
-	},
-}
-
-// UpgradeApp upgrades apps.
-var UpgradeApp = cli.Command{
-	Name:      "upgrade",
-	Aliases:   []string{"u"},
-	Usage:     "Upgrade apps",
-	ArgsUsage: "<stack-name> <recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.ChaosFlag,
-		majorFlag,
-		allFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Upgrade an app by specifying its stack name and recipe. By passing "--all"
-instead, every deployed app is upgraded. For each apps with enabled auto
-updates the deployed version is compared with the current recipe catalogue
-version. If a new patch/minor version is available, the app is upgraded. To
-include major versions use the "--major" flag. Don't do that, it will probably
-break things. Only apps that are not deployed with "--chaos" are upgraded, to
-update chaos deployments use the "--chaos" flag. Use it with care.
-`,
-	Action: func(c *cli.Context) error {
-		cl, err := client.New("default")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		conf := runtime.New()
-
-		if !updateAll {
-			stackName := c.Args().Get(0)
-			recipeName := c.Args().Get(1)
-			err = tryUpgrade(cl, stackName, recipeName, conf)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			return nil
-		}
-
-		stacks, err := stack.GetStacks(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, stackInfo := range stacks {
-			stackName := stackInfo.Name
-			recipeName, err := getLabel(cl, stackName, "recipe")
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			err = tryUpgrade(cl, stackName, recipeName, conf)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		return nil
-	},
-}
-
-// getLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
-func getLabel(cl *dockerclient.Client, stackName string, label string) (string, error) {
-	filter := filters.NewArgs()
-	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
-
-	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
-	if err != nil {
-		return "", err
-	}
-
-	for _, service := range services {
-		labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
-		if labelValue, ok := service.Spec.Labels[labelKey]; ok {
-			return labelValue, nil
-		}
-	}
-
-	logrus.Debugf("no %s label found for %s", label, stackName)
-
-	return "", nil
-}
-
-// getBoolLabel reads a boolean docker label.
-func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool, error) {
-	lableValue, err := getLabel(cl, stackName, label)
-	if err != nil {
-		return false, err
-	}
-
-	if lableValue != "" {
-		value, err := strconv.ParseBool(lableValue)
-		if err != nil {
-			return false, err
-		}
-
-		return value, nil
-	}
-
-	logrus.Debugf("Boolean label %s could not be found for %s, set default to false.", label, stackName)
-
-	return false, nil
-}
-
-// getEnv reads env variables from docker services.
-func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) {
-	envMap := make(map[string]string)
-	filter := filters.NewArgs()
-	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
-
-	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
-	if err != nil {
-		return nil, err
-	}
-
-	for _, service := range services {
-		envList := service.Spec.TaskTemplate.ContainerSpec.Env
-		for _, envString := range envList {
-			splitString := strings.SplitN(envString, "=", 2)
-			if len(splitString) != 2 {
-				logrus.Debugf("can't separate key from value: %s (this variable is probably unset)", envString)
-				continue
-			}
-			k := splitString[0]
-			v := splitString[1]
-			logrus.Debugf("For %s read env %s with value: %s from docker service", stackName, k, v)
-			envMap[k] = v
-		}
-	}
-
-	return envMap, nil
-}
-
-// getLatestUpgrade returns the latest available version for an app respecting
-// the "--major" flag if it is newer than the currently deployed version.
-func getLatestUpgrade(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
-	deployedVersion, err := getDeployedVersion(cl, stackName, recipeName)
-	if err != nil {
-		return "", err
-	}
-
-	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion)
-	if err != nil {
-		return "", err
-	}
-
-	if len(availableUpgrades) == 0 {
-		logrus.Debugf("no available upgrades for %s", stackName)
-		return "", nil
-	}
-
-	var chosenUpgrade string
-	if len(availableUpgrades) > 0 {
-		chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
-		logrus.Infof("%s (%s) can be upgraded from version %s to %s", stackName, recipeName, deployedVersion, chosenUpgrade)
-	}
-
-	return chosenUpgrade, nil
-}
-
-// getDeployedVersion returns the currently deployed version of an app.
-func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
-	logrus.Debugf("Retrieve deployed version whether %s is already deployed", stackName)
-
-	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
-	if err != nil {
-		return "", err
-	}
-
-	if !isDeployed {
-		return "", fmt.Errorf("%s is not deployed?", stackName)
-	}
-
-	if deployedVersion == "unknown" {
-		return "", fmt.Errorf("failed to determine deployed version of %s", stackName)
-	}
-
-	return deployedVersion, nil
-}
-
-// getAvailableUpgrades returns all available versions of an app that are newer
-// than the deployed version. It only includes major upgrades if the "--major"
-// flag is set.
-func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
-	deployedVersion string) ([]string, error) {
-	catl, err := recipe.ReadRecipeCatalogue()
-	if err != nil {
-		return nil, err
-	}
-
-	versions, err := recipe.GetRecipeCatalogueVersions(recipeName, catl)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(versions) == 0 {
-		return nil, fmt.Errorf("no published releases for %s in the recipe catalogue?", recipeName)
-	}
-
-	var availableUpgrades []string
-	for _, version := range versions {
-		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
-		if err != nil {
-			return nil, err
-		}
-
-		parsedVersion, err := tagcmp.Parse(version)
-		if err != nil {
-			return nil, err
-		}
-
-		versionDelta, err := parsedDeployedVersion.UpgradeDelta(parsedVersion)
-		if err != nil {
-			return nil, err
-		}
-
-		if 0 < versionDelta.UpgradeType() && (versionDelta.UpgradeType() < 4 || majorUpdate) {
-			availableUpgrades = append(availableUpgrades, version)
-		}
-	}
-
-	logrus.Debugf("Available updates for %s: %s", stackName, availableUpgrades)
-
-	return availableUpgrades, nil
-}
-
-// processRecipeRepoVersion clones, pulls, checks out the version and lints the
-// recipe repository.
-func processRecipeRepoVersion(recipeName, version string, conf *runtime.Config) error {
-	if err := recipe.EnsureExists(recipeName, conf); err != nil {
-		return err
-	}
-
-	if err := recipe.EnsureUpToDate(recipeName); err != nil {
-		return err
-	}
-
-	if err := recipe.EnsureVersion(recipeName, version); err != nil {
-		return err
-	}
-
-	if r, err := recipe.Get(recipeName, conf); err != nil {
-		return err
-	} else if err := lint.LintForErrors(r); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// mergeAbraShEnv merges abra.sh env vars into the app env vars.
-func mergeAbraShEnv(recipeName string, env config.AppEnv) error {
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
-	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
-	if err != nil {
-		return err
-	}
-
-	for k, v := range abraShEnv {
-		logrus.Debugf("read v:%s k: %s", v, k)
-		env[k] = v
-	}
-
-	return nil
-}
-
-// createDeployConfig merges and enriches the compose config for the deployment.
-func createDeployConfig(recipeName string, stackName string, env config.AppEnv) (*composetypes.Config, stack.Deploy, error) {
-	env["STACK_NAME"] = stackName
-
-	deployOpts := stack.Deploy{
-		Namespace:    stackName,
-		Prune:        false,
-		ResolveImage: stack.ResolveImageAlways,
-	}
-
-	composeFiles, err := config.GetAppComposeFiles(recipeName, env)
-	if err != nil {
-		return nil, deployOpts, err
-	}
-
-	deployOpts.Composefiles = composeFiles
-	compose, err := config.GetAppComposeConfig(stackName, deployOpts, env)
-	if err != nil {
-		return nil, deployOpts, err
-	}
-
-	config.ExposeAllEnv(stackName, compose, env)
-
-	// after the upgrade the deployment won't be in chaos state anymore
-	config.SetChaosLabel(compose, stackName, false)
-	config.SetRecipeLabel(compose, stackName, recipeName)
-	config.SetUpdateLabel(compose, stackName, env)
-
-	return compose, deployOpts, nil
-}
-
-// tryUpgrade performs the upgrade if all the requirements are fulfilled.
-func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string, conf *runtime.Config) error {
-	if recipeName == "" {
-		logrus.Debugf("Don't update %s due to missing recipe name", stackName)
-		return nil
-	}
-
-	chaos, err := getBoolLabel(cl, stackName, "chaos")
-	if err != nil {
-		return err
-	}
-
-	if chaos && !internal.Chaos {
-		logrus.Debugf("Don't update %s due to chaos deployment.", stackName)
-		return nil
-	}
-
-	updatesEnabled, err := getBoolLabel(cl, stackName, "autoupdate")
-	if err != nil {
-		return err
-	}
-
-	if !updatesEnabled {
-		logrus.Debugf("Don't update %s due to disabling auto updates or missing ENABLE_AUTOUPDATE env.", stackName)
-		return nil
-	}
-
-	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName)
-	if err != nil {
-		return err
-	}
-
-	if upgradeVersion == "" {
-		logrus.Debugf("Don't update %s due to no new version.", stackName)
-		return nil
-	}
-
-	err = upgrade(cl, stackName, recipeName, upgradeVersion, conf)
-
-	return err
-}
-
-// upgrade performs all necessary steps to upgrade an app.
-func upgrade(cl *dockerclient.Client, stackName, recipeName, upgradeVersion string, conf *runtime.Config) error {
-	env, err := getEnv(cl, stackName)
-	if err != nil {
-		return err
-	}
-
-	app := config.App{
-		Name:   stackName,
-		Recipe: recipeName,
-		Server: SERVER,
-		Env:    env,
-	}
-
-	if err = processRecipeRepoVersion(recipeName, upgradeVersion, conf); err != nil {
-		return err
-	}
-
-	if err = mergeAbraShEnv(recipeName, app.Env); err != nil {
-		return err
-	}
-
-	compose, deployOpts, err := createDeployConfig(recipeName, stackName, app.Env)
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof("Upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
-
-	err = stack.RunDeploy(cl, deployOpts, compose, stackName, true)
-
-	return err
-}
-
-func newAbraApp(version, commit string) *cli.App {
-	app := &cli.App{
-		Name: "kadabra",
-		Usage: `The Co-op Cloud auto-updater
-    ____                           ____ _                 _
-   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
-  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
-  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
-   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
-                         |_|
-`,
-		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
-		Commands: []cli.Command{
-			Notify,
-			UpgradeApp,
-		},
-	}
-
-	app.Before = func(c *cli.Context) error {
-		logrus.Debugf("kadabra version %s, commit %s", version, commit)
-		return nil
-	}
-
-	return app
-}
-
-// RunApp runs CLI abra app.
-func RunApp(version, commit string) {
-	app := newAbraApp(version, commit)
-
-	if err := app.Run(os.Args); err != nil {
-		logrus.Fatal(err)
-	}
-}

cli/upgrade.go

@@ -0,0 +1,23 @@
+package cli
+
+import (
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = &cli.Command{
+	Name:  "upgrade",
+	Usage: "Upgrade abra",
+	Action: func(c *cli.Context) error {
+		cmd := exec.Command("bash", "-c", "curl -s https://install.abra.coopcloud.tech | bash")
+		logrus.Debugf("attempting to run '%s'", cmd)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+		return nil
+	},
+}

cmd/abra/main.go

@@ -5,13 +5,14 @@ import (
 	"coopcloud.tech/abra/cli"
 )
 
-// Version is the current version of Abra.
+// Version is the current version of abra.
 var Version string
 
-// Commit is the current git commit of Abra.
+// Commit is the current commit of abra.
 var Commit string
 
 func main() {
+	// If not set in the ld-flags
 	if Version == "" {
 		Version = "dev"
 	}

cmd/kadabra/main.go

@@ -1,23 +0,0 @@
-// Package main provides the command-line entrypoint.
-package main
-
-import (
-	"coopcloud.tech/abra/cli/updater"
-)
-
-// Version is the current version of Kadabra.
-var Version string
-
-// Commit is the current git commit of Kadabra.
-var Commit string
-
-func main() {
-	if Version == "" {
-		Version = "dev"
-	}
-	if Commit == "" {
-		Commit = "       "
-	}
-
-	updater.RunApp(Version, Commit)
-}

go.mod

@@ -3,48 +3,42 @@ module coopcloud.tech/abra
 go 1.16
 
 require (
-	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	coopcloud.tech/tagcmp v0.0.0-20211011140827-4f27c74467eb
-	github.com/AlecAivazis/survey/v2 v2.3.6
+	github.com/AlecAivazis/survey/v2 v2.3.1
-	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
+	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.23+incompatible
+	github.com/docker/cli v20.10.8+incompatible
-	github.com/docker/distribution v2.8.1+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.23+incompatible
+	github.com/docker/docker v20.10.8+incompatible
-	github.com/docker/go-units v0.5.0
+	github.com/docker/go-units v0.4.0
-	github.com/go-git/go-git/v5 v5.5.2
+	github.com/go-git/go-git/v5 v5.4.2
-	github.com/hetznercloud/hcloud-go v1.40.0
+	github.com/hetznercloud/hcloud-go v1.32.0
-	github.com/moby/sys/signal v0.7.0
+	github.com/moby/sys/signal v0.5.0
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.13.0
+	github.com/schollz/progressbar/v3 v3.8.3
-	github.com/sirupsen/logrus v1.9.0
+	github.com/schultz-is/passgen v1.0.1
-	gotest.tools/v3 v3.4.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/urfave/cli/v2 v2.3.0
+	gotest.tools/v3 v3.0.3
 )
 
 require (
-	coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
+	coopcloud.tech/libcapsul v0.0.0-20211020153234-f1386b5cf79d
-	github.com/buger/goterm v1.0.4
+	github.com/Microsoft/hcsshim v0.8.21 // indirect
-	github.com/containerd/containerd v1.5.9 // indirect
+	github.com/containerd/containerd v1.5.5 // indirect
-	github.com/containers/image v3.0.2+incompatible
-	github.com/containers/storage v1.38.2 // indirect
-	github.com/decentral1se/passgen v1.0.1
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.2
-	github.com/klauspost/pgzip v1.2.5
 	github.com/libdns/gandi v1.0.2
 	github.com/libdns/libdns v0.2.1
 	github.com/moby/sys/mount v0.2.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
-	github.com/spf13/cobra v1.3.0 // indirect
 	github.com/theupdateframework/notary v0.7.0 // indirect
-	github.com/urfave/cli v1.22.9
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0
-	golang.org/x/crypto v0.5.0 // indirect
-	golang.org/x/sys v0.5.0
 )

pkg/app/app.go

@@ -1,9 +1,13 @@
 package app
 
 import (
+	"context"
+	"fmt"
 	"strings"
 
+	"coopcloud.tech/abra/pkg/client/stack"
 	"coopcloud.tech/abra/pkg/config"
+	apiclient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
@@ -19,7 +23,7 @@ func Get(appName string) (config.App, error) {
 		return config.App{}, err
 	}
 
-	logrus.Debugf("retrieved %s for %s", app, appName)
+	logrus.Debugf("retrieved '%s' for '%s'", app, appName)
 
 	return app, nil
 }
@@ -33,10 +37,49 @@ type deployedServiceSpec struct {
 // VersionSpec represents a deployed app and associated metadata.
 type VersionSpec map[string]deployedServiceSpec
 
-// ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label.
+// DeployedVersions lists metadata (e.g. versions) for deployed
+func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App) (VersionSpec, bool, error) {
+	services, err := stack.GetStackServices(ctx, cl, app.StackName())
+	if err != nil {
+		return VersionSpec{}, false, err
+	}
+
+	appSpec := make(VersionSpec)
+	for _, service := range services {
+		serviceName := ParseServiceName(service.Spec.Name)
+		label := fmt.Sprintf("coop-cloud.%s.%s.version", app.StackName(), serviceName)
+		if deployLabel, ok := service.Spec.Labels[label]; ok {
+			version, _ := ParseVersionLabel(deployLabel)
+			appSpec[serviceName] = deployedServiceSpec{Name: serviceName, Version: version}
+		}
+	}
+
+	deployed := len(services) > 0
+
+	if deployed {
+		logrus.Debugf("detected '%s' as deployed versions of '%s'", appSpec, app.Name)
+	} else {
+		logrus.Debugf("detected '%s' as not deployed", app.Name)
+	}
+
+	return appSpec, len(services) > 0, nil
+}
+
+// ParseVersionLabel parses a $VERSION-$DIGEST app service label.
+func ParseVersionLabel(label string) (string, string) {
+	// versions may look like v4.2-abcd or v4.2-alpine-abcd
+	idx := strings.LastIndex(label, "-")
+	version := label[:idx]
+	digest := label[idx+1:]
+	logrus.Debugf("parsed '%s' as version from '%s'", version, label)
+	logrus.Debugf("parsed '%s' as digest from '%s'", digest, label)
+	return version, digest
+}
+
+// ParseVersionName parses a $STACK_NAME_$SERVICE_NAME service label.
 func ParseServiceName(label string) string {
 	idx := strings.LastIndex(label, "_")
 	serviceName := label[idx+1:]
-	logrus.Debugf("parsed %s as service name from %s", serviceName, label)
+	logrus.Debugf("parsed '%s' as service name from '%s'", serviceName, label)
 	return serviceName
 }

pkg/autocomplete/autocomplete.go

@@ -1,63 +0,0 @@
-package autocomplete
-
-import (
-	"fmt"
-
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-// AppNameComplete copletes app names
-func AppNameComplete(c *cli.Context) {
-	appNames, err := config.GetAppNames()
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	if c.NArg() > 0 {
-		return
-	}
-
-	for _, a := range appNames {
-		fmt.Println(a)
-	}
-}
-
-// RecipeNameComplete completes recipe names
-func RecipeNameComplete(c *cli.Context) {
-	catl, err := recipe.ReadRecipeCatalogue()
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	if c.NArg() > 0 {
-		return
-	}
-
-	for name := range catl {
-		fmt.Println(name)
-	}
-}
-
-// SubcommandComplete completes subcommands.
-func SubcommandComplete(c *cli.Context) {
-	if c.NArg() > 0 {
-		return
-	}
-
-	subcmds := []string{
-		"app",
-		"autocomplete",
-		"catalogue",
-		"recipe",
-		"record",
-		"server",
-		"upgrade",
-	}
-
-	for _, cmd := range subcmds {
-		fmt.Println(cmd)
-	}
-}

pkg/catalogue/catalogue.go

@@ -1,122 +1,500 @@
+// Package catalogue provides ways of interacting with recipe catalogues which
+// are JSON data structures which contain meta information about recipes (e.g.
+// what versions of the Nextcloud recipe are available?).
 package catalogue
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"strings"
+	"time"
 
+	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/web"
+	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
+// RecipeCatalogueURL is the only current recipe catalogue available.
-var CatalogueSkipList = map[string]bool{
+const RecipeCatalogueURL = "https://apps.coopcloud.tech"
-	"abra":                        true,
+
-	"abra-apps":                   true,
+// ReposMetadataURL is the recipe repository metadata
-	"abra-aur":                    true,
+const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
-	"abra-bash":                   true,
+
-	"abra-capsul":                 true,
+// image represents a recipe container image.
-	"abra-gandi":                  true,
+type image struct {
-	"abra-hetzner":                true,
+	Image  string `json:"image"`
-	"apps":                        true,
+	Rating string `json:"rating"`
-	"aur-abra-git":                true,
+	Source string `json:"source"`
-	"auto-recipes-catalogue-json": true,
+	URL    string `json:"url"`
-	"auto-mirror":                 true,
-	"backup-bot":                  true,
-	"backup-bot-two":              true,
-	"beta.coopcloud.tech":         true,
-	"comrade-renovate-bot":        true,
-	"coopcloud.tech":              true,
-	"coturn":                      true,
-	"docker-cp-deploy":            true,
-	"docker-dind-bats-kcov":       true,
-	"docs.coopcloud.tech":         true,
-	"drone-abra":                  true,
-	"example":                     true,
-	"gardening":                   true,
-	"go-abra":                     true,
-	"organising":                  true,
-	"pyabra":                      true,
-	"radicle-seed-node":           true,
-	"recipes-catalogue-json":      true,
-	"recipes-wishlist":            true,
-	"recipes.coopcloud.tech":      true,
-	"stack-ssh-deploy":            true,
-	"swarm-cronjob":               true,
-	"tagcmp":                      true,
-	"traefik-cert-dumper":         true,
-	"tyop":                        true,
 }
 
-// EnsureCatalogue ensures that the catalogue is cloned locally & present.
+// features represent what top-level features a recipe supports (e.g. does this
-func EnsureCatalogue() error {
+// recipe support backups?).
-	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+type features struct {
-	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+	Backups     string `json:"backups"`
-		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+	Email       string `json:"email"`
-		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+	Healthcheck string `json:"healthcheck"`
-			return err
+	Image       image  `json:"image"`
-		}
+	Status      int    `json:"status"`
+	Tests       string `json:"tests"`
+}
 
-		logrus.Debugf("cloned catalogue repository to %s", catalogueDir)
+// tag represents a git tag.
+type tag = string
+
+// service represents a service within a recipe.
+type service = string
+
+// ServiceMeta represents meta info associated with a service.
+type ServiceMeta struct {
+	Digest string `json:"digest"`
+	Image  string `json:"image"`
+	Tag    string `json:"tag"`
+}
+
+// RecipeVersions are the versions associated with a recipe.
+type RecipeVersions []map[tag]map[service]ServiceMeta
+
+// RecipeMeta represents metadata for a recipe in the abra catalogue.
+type RecipeMeta struct {
+	Category      string         `json:"category"`
+	DefaultBranch string         `json:"default_branch"`
+	Description   string         `json:"description"`
+	Features      features       `json:"features"`
+	Icon          string         `json:"icon"`
+	Name          string         `json:"name"`
+	Repository    string         `json:"repository"`
+	Versions      RecipeVersions `json:"versions"`
+	Website       string         `json:"website"`
+}
+
+// LatestVersion returns the latest version of a recipe.
+func (r RecipeMeta) LatestVersion() string {
+	var version string
+
+	// apps.json versions are sorted so the last key is latest
+	latest := r.Versions[len(r.Versions)-1]
+
+	for tag := range latest {
+		version = tag
 	}
 
+	logrus.Debugf("choosing '%s' as latest version of '%s'", version, r.Name)
+
+	return version
+}
+
+// Name represents a recipe name.
+type Name = string
+
+// RecipeCatalogue represents the entire recipe catalogue.
+type RecipeCatalogue map[Name]RecipeMeta
+
+// Flatten converts AppCatalogue to slice
+func (r RecipeCatalogue) Flatten() []RecipeMeta {
+	recipes := make([]RecipeMeta, 0, len(r))
+
+	for name := range r {
+		recipes = append(recipes, r[name])
+	}
+
+	return recipes
+}
+
+// ByRecipeName sorts recipes by name.
+type ByRecipeName []RecipeMeta
+
+func (r ByRecipeName) Len() int      { return len(r) }
+func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+func (r ByRecipeName) Less(i, j int) bool {
+	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
+}
+
+// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
+// is up to date.
+func recipeCatalogueFSIsLatest() (bool, error) {
+	httpClient := &http.Client{Timeout: web.Timeout}
+	res, err := httpClient.Head(RecipeCatalogueURL)
+	if err != nil {
+		return false, err
+	}
+
+	lastModified := res.Header["Last-Modified"][0]
+	parsed, err := time.Parse(time.RFC1123, lastModified)
+	if err != nil {
+		return false, err
+	}
+
+	info, err := os.Stat(config.APPS_JSON)
+	if err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no recipe catalogue found in file system cache")
+			return false, nil
+		}
+		return false, err
+	}
+
+	localModifiedTime := info.ModTime().Unix()
+	remoteModifiedTime := parsed.Unix()
+
+	if localModifiedTime < remoteModifiedTime {
+		logrus.Debug("file system cached recipe catalogue is out-of-date")
+		return false, nil
+	}
+
+	logrus.Debug("file system cached recipe catalogue is up-to-date")
+
+	return true, nil
+}
+
+// ReadRecipeCatalogue reads the recipe catalogue.
+func ReadRecipeCatalogue() (RecipeCatalogue, error) {
+	recipes := make(RecipeCatalogue)
+
+	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
+	if err != nil {
+		return nil, err
+	}
+
+	if !recipeFSIsLatest {
+		logrus.Debugf("reading recipe catalogue from web to get latest")
+		if err := readRecipeCatalogueWeb(&recipes); err != nil {
+			return nil, err
+		}
+		return recipes, nil
+	}
+
+	logrus.Debugf("reading recipe catalogue from file system cache to get latest")
+	if err := readRecipeCatalogueFS(&recipes); err != nil {
+		return nil, err
+	}
+
+	return recipes, nil
+}
+
+// readRecipeCatalogueFS reads the catalogue from the file system.
+func readRecipeCatalogueFS(target interface{}) error {
+	recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
+	if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
+		return err
+	}
+
+	logrus.Debugf("read recipe catalogue from file system cache in '%s'", config.APPS_JSON)
+
 	return nil
 }
 
-// EnsureUpToDate ensures that the local catalogue has no unstaged changes as
+// readRecipeCatalogueWeb reads the catalogue from the web.
-// is up to date. This is useful to run before doing catalogue generation.
+func readRecipeCatalogueWeb(target interface{}) error {
-func EnsureUpToDate() error {
+	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
-	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
+		return err
+	}
+
+	recipesJSON, err := json.MarshalIndent(target, "", "    ")
 	if err != nil {
 		return err
 	}
 
-	if !isClean {
+	if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
-		msg := "%s has locally unstaged changes? please commit/remove your changes before proceeding"
-		return fmt.Errorf(msg, config.CATALOGUE_DIR)
-	}
-
-	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
-	if err != nil {
 		return err
 	}
 
-	remotes, err := repo.Remotes()
+	logrus.Debugf("read recipe catalogue from web at '%s'", RecipeCatalogueURL)
+
+	return nil
+}
+
+// VersionsOfService lists the version of a service.
+func VersionsOfService(recipe, serviceName string) ([]string, error) {
+	catalogue, err := ReadRecipeCatalogue()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	if len(remotes) == 0 {
+	rec, ok := catalogue[recipe]
-		msg := "cannot ensure %s is up-to-date, no git remotes configured"
+	if !ok {
-		logrus.Debugf(msg, config.CATALOGUE_DIR)
+		return nil, fmt.Errorf("recipe '%s' does not exist?", recipe)
-		return nil
+	}
+
+	versions := []string{}
+	alreadySeen := make(map[string]bool)
+	for _, serviceVersion := range rec.Versions {
+		for tag := range serviceVersion {
+			if _, ok := alreadySeen[tag]; !ok {
+				alreadySeen[tag] = true
+				versions = append(versions, tag)
+			}
+		}
+	}
+
+	logrus.Debugf("detected versions '%s' for '%s'", strings.Join(versions, ", "), recipe)
+
+	return versions, nil
+}
+
+// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
+func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
+	catl, err := ReadRecipeCatalogue()
+	if err != nil {
+		return RecipeMeta{}, err
+	}
+
+	recipeMeta, ok := catl[recipeName]
+	if !ok {
+		err := fmt.Errorf("recipe '%s' does not exist?", recipeName)
+		return RecipeMeta{}, err
+	}
+
+	if err := recipe.EnsureExists(recipeName); err != nil {
+		return RecipeMeta{}, err
+	}
+
+	logrus.Debugf("recipe metadata retrieved for '%s'", recipeName)
+
+	return recipeMeta, nil
+}
+
+// RepoMeta is a single recipe repo metadata.
+type RepoMeta struct {
+	ID                        int `json:"id"`
+	Owner                     Owner
+	Name                      string      `json:"name"`
+	FullName                  string      `json:"full_name"`
+	Description               string      `json:"description"`
+	Empty                     bool        `json:"empty"`
+	Private                   bool        `json:"private"`
+	Fork                      bool        `json:"fork"`
+	Template                  bool        `json:"template"`
+	Parent                    interface{} `json:"parent"`
+	Mirror                    bool        `json:"mirror"`
+	Size                      int         `json:"size"`
+	HTMLURL                   string      `json:"html_url"`
+	SSHURL                    string      `json:"ssh_url"`
+	CloneURL                  string      `json:"clone_url"`
+	OriginalURL               string      `json:"original_url"`
+	Website                   string      `json:"website"`
+	StarsCount                int         `json:"stars_count"`
+	ForksCount                int         `json:"forks_count"`
+	WatchersCount             int         `json:"watchers_count"`
+	OpenIssuesCount           int         `json:"open_issues_count"`
+	OpenPRCount               int         `json:"open_pr_counter"`
+	ReleaseCounter            int         `json:"release_counter"`
+	DefaultBranch             string      `json:"default_branch"`
+	Archived                  bool        `json:"archived"`
+	CreatedAt                 string      `json:"created_at"`
+	UpdatedAt                 string      `json:"updated_at"`
+	Permissions               Permissions
+	HasIssues                 bool `json:"has_issues"`
+	InternalTracker           InternalTracker
+	HasWiki                   bool   `json:"has_wiki"`
+	HasPullRequests           bool   `json:"has_pull_requests"`
+	HasProjects               bool   `json:"has_projects"`
+	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
+	AllowMergeCommits         bool   `json:"allow_merge_commits"`
+	AllowRebase               bool   `json:"allow_rebase"`
+	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
+	AllowSquashMerge          bool   `json:"allow_squash_merge"`
+	AvatarURL                 string `json:"avatar_url"`
+	Internal                  bool   `json:"internal"`
+	MirrorInterval            string `json:"mirror_interval"`
+}
+
+// Owner is the repo organisation owner metadata.
+type Owner struct {
+	ID         int    `json:"id"`
+	Login      string `json:"login"`
+	FullName   string `json:"full_name"`
+	Email      string `json:"email"`
+	AvatarURL  string `json:"avatar_url"`
+	Language   string `json:"language"`
+	IsAdmin    bool   `json:"is_admin"`
+	LastLogin  string `json:"last_login"`
+	Created    string `json:"created"`
+	Restricted bool   `json:"restricted"`
+	Username   string `json:"username"`
+}
+
+// Permissions is perms metadata for a repo.
+type Permissions struct {
+	Admin bool `json:"admin"`
+	Push  bool `json:"push"`
+	Pull  bool `json:"pull"`
+}
+
+// InternalTracker is issue tracker metadata for a repo.
+type InternalTracker struct {
+	EnableTimeTracker                bool `json:"enable_time_tracker"`
+	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
+	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
+}
+
+// RepoCatalogue represents all the recipe repo metadata.
+type RepoCatalogue map[string]RepoMeta
+
+// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
+func ReadReposMetadata() (RepoCatalogue, error) {
+	reposMeta := make(RepoCatalogue)
+
+	pageIdx := 1
+	for {
+		var reposList []RepoMeta
+
+		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
+
+		logrus.Debugf("fetching repo metadata from '%s'", pagedURL)
+
+		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
+			return reposMeta, err
+		}
+
+		if len(reposList) == 0 {
+			break
+		}
+
+		for idx, repo := range reposList {
+			reposMeta[repo.Name] = reposList[idx]
+		}
+
+		pageIdx++
+	}
+
+	return reposMeta, nil
+}
+
+// GetRecipeVersions retrieves all recipe versions.
+func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
+	versions := RecipeVersions{}
+
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return versions, err
 	}
 
 	worktree, err := repo.Worktree()
 	if err != nil {
-		return err
+		logrus.Fatal(err)
 	}
 
-	branch, err := gitPkg.CheckoutDefaultBranch(repo, config.CATALOGUE_DIR)
+	gitTags, err := repo.Tags()
 	if err != nil {
-		return err
+		return versions, err
 	}
 
-	opts := &git.PullOptions{
+	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
-		Force:         true,
+		tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
-		ReferenceName: branch,
-	}
 
-	if err := worktree.Pull(opts); err != nil {
+		logrus.Debugf("processing '%s' for '%s'", tag, recipeName)
-		if !strings.Contains(err.Error(), "already up-to-date") {
+
+		checkOutOpts := &git.CheckoutOptions{
+			Create: false,
+			Force:  true,
+			Branch: plumbing.ReferenceName(ref.Name()),
+		}
+		if err := worktree.Checkout(checkOutOpts); err != nil {
+			logrus.Debugf("failed to check out '%s' in '%s'", tag, recipeDir)
 			return err
 		}
+
+		logrus.Debugf("successfully checked out '%s' in '%s'", ref.Name(), recipeDir)
+
+		recipe, err := recipe.Get(recipeName)
+		if err != nil {
+			return err
+		}
+
+		versionMeta := make(map[string]ServiceMeta)
+		for _, service := range recipe.Config.Services {
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return err
+			}
+
+			path := reference.Path(img)
+			if strings.Contains(path, "library") {
+				path = strings.Split(path, "/")[1]
+			}
+
+			digest, err := client.GetTagDigest(img)
+			if err != nil {
+				return err
+			}
+
+			versionMeta[service.Name] = ServiceMeta{
+				Digest: digest,
+				Image:  path,
+				Tag:    img.(reference.NamedTagged).Tag(),
+			}
+
+			logrus.Debugf("collecting digest: '%s', image: '%s', tag: '%s'", digest, path, tag)
+		}
+
+		versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
+
+		return nil
+	}); err != nil {
+		return versions, err
+	}
+
+	branch := "master"
+	if _, err := repo.Branch("master"); err != nil {
+		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", recipeDir)
+			logrus.Fatal(err)
+		}
+		branch = "main"
+	}
+
+	refName := fmt.Sprintf("refs/heads/%s", branch)
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: plumbing.ReferenceName(refName),
+	}
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", branch, recipeDir)
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("switched back to '%s' in '%s'", branch, recipeDir)
+	logrus.Debugf("collected '%s' for '%s'", versions, recipeName)
+
+	return versions, nil
+}
+
+// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
+func GetRecipeCatalogueVersions(recipeName string) ([]string, error) {
+	var versions []string
+
+	catl, err := ReadRecipeCatalogue()
+	if err != nil {
+		return versions, err
+	}
+
+	if recipeMeta, exists := catl[recipeName]; exists {
+		for _, versionMeta := range recipeMeta.Versions {
+			for tag := range versionMeta {
+				versions = append(versions, tag)
+			}
+		}
 	}
 
-	logrus.Debugf("fetched latest git changes for %s", config.CATALOGUE_DIR)
+	return versions, nil
-
-	return nil
 }

pkg/client/README.md

@@ -0,0 +1,4 @@
+IMPORTANT POINT ABOUT CONTEXTS
+
+Please use context names starting with `testContext` for testing purposes to ensure that no data is lost. such as `testContext`, `testContext2`, `testContextFail` etc
+

pkg/client/client.go

@@ -2,47 +2,37 @@
 package client
 
 import (
-	"context"
-	"errors"
-	"fmt"
 	"net/http"
 	"os"
 	"time"
 
-	contextPkg "coopcloud.tech/abra/pkg/context"
+	"github.com/docker/cli/cli/connhelper"
-	sshPkg "coopcloud.tech/abra/pkg/ssh"
-	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
 	"github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
 
-// New initiates a new Docker client. New client connections are validated so
+// New initiates a new Docker client.
-// that we ensure connections via SSH to the daemon can succeed. It takes into
+func New(contextName string) (*client.Client, error) {
-// account that you may only want the local client and not communicate via SSH.
-// For this use-case, please pass "default" as the contextName.
-func New(serverName string) (*client.Client, error) {
 	var clientOpts []client.Opt
 
-	if serverName != "default" {
+	clientOpts = append(clientOpts, client.WithTimeout(3*time.Second))
-		context, err := GetContext(serverName)
-		if err != nil {
-			return nil, fmt.Errorf("unknown server, run \"abra server add %s\"?", serverName)
-		}
 
-		ctxEndpoint, err := contextPkg.GetContextEndpoint(context)
+	if contextName != "default" {
+		context, err := GetContext(contextName)
 		if err != nil {
 			return nil, err
 		}
 
-		helper, err := commandconnPkg.NewConnectionHelper(ctxEndpoint)
+		ctxEndpoint, err := GetContextEndpoint(context)
 		if err != nil {
 			return nil, err
 		}
 
+		helper := newConnectionHelper(ctxEndpoint)
 		httpClient := &http.Client{
+			// No tls, no proxy
 			Transport: &http.Transport{
-				DialContext:     helper.Dialer,
+				DialContext: helper.Dialer,
-				IdleConnTimeout: 30 * time.Second,
 			},
 		}
 
@@ -65,20 +55,15 @@ func New(serverName string) (*client.Client, error) {
 		return nil, err
 	}
 
-	logrus.Debugf("created client for %s", serverName)
+	logrus.Debugf("created client for '%s'", contextName)
-
-	info, err := cl.Info(context.Background())
-	if err != nil {
-		return cl, sshPkg.Fatal(serverName, err)
-	}
-
-	if info.Swarm.LocalNodeState == "inactive" {
-		if serverName != "default" {
-			return cl, fmt.Errorf("swarm mode not enabled on %s?", serverName)
-		} else {
-			return cl, errors.New("swarm mode not enabled on local server?")
-		}
-	}
 
 	return cl, nil
 }
+
+func newConnectionHelper(daemonURL string) *connhelper.ConnectionHelper {
+	helper, err := connhelper.GetConnectionHelper(daemonURL)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	return helper
+}

pkg/client/client_test.go

@@ -0,0 +1,46 @@
+package client_test
+
+import (
+	"fmt"
+	"testing"
+
+	"coopcloud.tech/abra/pkg/client"
+)
+
+// use at the start to ensure testContext[0, 1, ..., amnt-1] exist and
+// testContextFail[0, 1, ..., failAmnt-1] don't exist
+func ensureTestState(amnt, failAmnt int) error {
+	for i := 0; i < amnt; i++ {
+		err := client.CreateContext(fmt.Sprintf("testContext%d", i), "", "")
+		if err != nil {
+			return err
+		}
+	}
+	for i := 0; i < failAmnt; i++ {
+		if _, er := client.GetContext(fmt.Sprintf("testContextFail%d", i)); er == nil {
+			err := client.DeleteContext(fmt.Sprintf("testContextFail%d", i))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func TestNew(t *testing.T) {
+	err := ensureTestState(1, 1)
+	if err != nil {
+		t.Errorf("Couldn't ensure existence/nonexistence of contexts: %s", err)
+	}
+	contextName := "testContext0"
+	_, err = client.New(contextName)
+	if err != nil {
+		t.Errorf("couldn't initialise a new client with context %s: %s", contextName, err)
+	}
+	contextName = "testContextFail0"
+	_, err = client.New(contextName)
+	if err == nil {
+		t.Errorf("client.New(\"testContextFail0\") should have failed but didn't return an error")
+	}
+
+}

pkg/client/container/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2013-2017 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/client/container/README.md

@@ -0,0 +1,7 @@
+# github.com/docker/cli/cli/command/container
+
+Due to this literally just being copy-pasted from the lib, the Apache license
+will be posted in this folder. Small edits to the source code have been to
+function names and parts we don't need deleted.
+
+Same vibe as [../convert](../convert).

pkg/client/container/exec.go

@@ -1,4 +1,4 @@
-package container // https://github.com/docker/cli/blob/master/cli/command/container/exec.go
+package container
 
 import (
 	"context"

pkg/client/container/hijack.go

@@ -1,4 +1,4 @@
-package container // https://github.com/docker/cli/blob/master/cli/command/container/hijack.go
+package container
 
 import (
 	"context"
@@ -16,6 +16,7 @@ import (
 )
 
 // The default escape key sequence: ctrl-p, ctrl-q
+// TODO: This could be moved to `pkg/term`.
 var defaultEscapeKeys = []byte{16, 17}
 
 // A hijackedIOStreamer handles copying input to and output from streams to the

pkg/client/container/tty.go

@@ -1,4 +1,4 @@
-package container // https://github.com/docker/cli/blob/master/cli/command/container/tty.go
+package container
 
 import (
 	"context"

pkg/client/context.go

@@ -4,11 +4,15 @@ import (
 	"errors"
 	"fmt"
 
-	"coopcloud.tech/abra/pkg/context"
+	command "github.com/docker/cli/cli/command"
-	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
 	dConfig "github.com/docker/cli/cli/config"
+	context "github.com/docker/cli/cli/context"
 	"github.com/docker/cli/cli/context/docker"
 	contextStore "github.com/docker/cli/cli/context/store"
+	dCliContextStore "github.com/docker/cli/cli/context/store"
+	cliflags "github.com/docker/cli/cli/flags"
+	dClient "github.com/docker/docker/client"
+	"github.com/moby/term"
 	"github.com/sirupsen/logrus"
 )
 
@@ -26,13 +30,13 @@ func CreateContext(contextName string, user string, port string) error {
 	if err := createContext(contextName, host); err != nil {
 		return err
 	}
-	logrus.Debugf("created the %s context", contextName)
+	logrus.Debugf("created the '%s' context", contextName)
 	return nil
 }
 
 // createContext interacts with Docker Context to create a Docker context config
 func createContext(name string, host string) error {
-	s := context.NewDefaultDockerContextStore()
+	s := NewDefaultDockerContextStore()
 	contextMetadata := contextStore.Metadata{
 		Endpoints: make(map[string]interface{}),
 		Name:      name,
@@ -42,7 +46,7 @@ func createContext(name string, host string) error {
 		Endpoints: make(map[string]contextStore.EndpointTLSData),
 	}
 
-	dockerEP, dockerTLS, err := commandconnPkg.GetDockerEndpointMetadataAndTLS(host)
+	dockerEP, dockerTLS, err := getDockerEndpointMetadataAndTLS(host)
 	if err != nil {
 		return err
 	}
@@ -72,20 +76,81 @@ func DeleteContext(name string) error {
 		return err
 	}
 
+	// remove any context that might be loaded
+	// TODO: Check if the context we are removing is the active one rather than doing it all the time
 	cfg := dConfig.LoadDefaultConfigFile(nil)
 	cfg.CurrentContext = ""
 	if err := cfg.Save(); err != nil {
 		return err
 	}
 
-	return context.NewDefaultDockerContextStore().Remove(name)
+	return NewDefaultDockerContextStore().Remove(name)
 }
 
 func GetContext(contextName string) (contextStore.Metadata, error) {
-	ctx, err := context.NewDefaultDockerContextStore().GetMetadata(contextName)
+	ctx, err := NewDefaultDockerContextStore().GetMetadata(contextName)
 	if err != nil {
 		return contextStore.Metadata{}, err
 	}
 
 	return ctx, nil
 }
+
+func GetContextEndpoint(ctx contextStore.Metadata) (string, error) {
+	endpointmeta, ok := ctx.Endpoints["docker"].(context.EndpointMetaBase)
+	if !ok {
+		err := errors.New("context lacks Docker endpoint")
+		return "", err
+	}
+	return endpointmeta.Host, nil
+}
+
+func newContextStore(dir string, config contextStore.Config) contextStore.Store {
+	return contextStore.New(dir, config)
+}
+
+func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
+	_, _, stderr := term.StdStreams()
+	dockerConfig := dConfig.LoadDefaultConfigFile(stderr)
+	contextDir := dConfig.ContextStoreDir()
+	storeConfig := command.DefaultContextStoreConfig()
+	store := newContextStore(contextDir, storeConfig)
+
+	opts := &cliflags.CommonOptions{Context: "default"}
+
+	dockerContextStore := &command.ContextStoreWithDefault{
+		Store: store,
+		Resolver: func() (*command.DefaultContext, error) {
+			return command.ResolveDefaultContext(opts, dockerConfig, storeConfig, stderr)
+		},
+	}
+
+	return dockerContextStore
+}
+
+func getDockerEndpointMetadataAndTLS(host string) (docker.EndpointMeta, *dCliContextStore.EndpointTLSData, error) {
+	ep, err := getDockerEndpoint(host)
+	if err != nil {
+		return docker.EndpointMeta{}, nil, err
+	}
+	return ep.EndpointMeta, ep.TLSData.ToStoreTLSData(), nil
+}
+
+func getDockerEndpoint(host string) (docker.Endpoint, error) {
+	skipTLSVerify := false
+	ep := docker.Endpoint{
+		EndpointMeta: docker.EndpointMeta{
+			Host:          host,
+			SkipTLSVerify: skipTLSVerify,
+		},
+	}
+	// try to resolve a docker client, validating the configuration
+	opts, err := ep.ClientOpts()
+	if err != nil {
+		return docker.Endpoint{}, err
+	}
+	if _, err := dClient.NewClientWithOpts(opts...); err != nil {
+		return docker.Endpoint{}, err
+	}
+	return ep, nil
+}

pkg/client/context_test.go

@@ -0,0 +1,80 @@
+package client_test
+
+import (
+	"testing"
+
+	"coopcloud.tech/abra/pkg/client"
+	dContext "github.com/docker/cli/cli/context"
+	dCliContextStore "github.com/docker/cli/cli/context/store"
+)
+
+type TestContext struct {
+	context           dCliContextStore.Metadata
+	expected_endpoint string
+}
+
+func dockerContext(host, key string) TestContext {
+	dockerContext := dCliContextStore.Metadata{
+		Name:     "foo",
+		Metadata: nil,
+		Endpoints: map[string]interface{}{
+			key: dContext.EndpointMetaBase{
+				Host:          host,
+				SkipTLSVerify: false,
+			},
+		},
+	}
+	return TestContext{
+		context:           dockerContext,
+		expected_endpoint: host,
+	}
+}
+
+func TestCreateContext(t *testing.T) {
+	err := client.CreateContext("testContext0", "wronguser", "wrongport")
+	if err == nil {
+		t.Error("client.CreateContext(\"testContextCreate\", \"wronguser\", \"wrongport\") should have failed but didn't return an error")
+	}
+	err = client.CreateContext("testContext0", "", "")
+	if err != nil {
+		t.Errorf("Couldn't create context: %s", err)
+	}
+}
+
+func TestDeleteContext(t *testing.T) {
+	ensureTestState(1, 1)
+	err := client.DeleteContext("default")
+	if err == nil {
+		t.Errorf("client.DeleteContext(\"default\") should have failed but didn't return an error")
+	}
+
+	err = client.DeleteContext("testContext0")
+	if err != nil {
+		t.Errorf("client.DeleteContext(\"testContext0\") failed: %s", err)
+	}
+	err = client.DeleteContext("testContextFail0")
+	if err == nil {
+		t.Errorf("client.DeleteContext(\"testContextFail0\") should have failed (attempt to delete non-existent context) but didn't return an error")
+	}
+}
+
+func TestGetContextEndpoint(t *testing.T) {
+	var testDockerContexts = []TestContext{
+		dockerContext("ssh://foobar", "docker"),
+		dockerContext("ssh://foobar", "k8"),
+	}
+	for _, context := range testDockerContexts {
+		endpoint, err := client.GetContextEndpoint(context.context)
+		if err != nil {
+			if err.Error() != "context lacks Docker endpoint" {
+				t.Error(err)
+			}
+		} else {
+			if endpoint != context.expected_endpoint {
+				t.Errorf("did not get correct context endpoint. Expected: %s, received: %s", context.expected_endpoint, endpoint)
+			}
+		}
+
+	}
+
+}

pkg/client/convert/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2013-2017 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/client/convert/README.md

@@ -0,0 +1,10 @@
+# github.com/docker/cli/cli/compose/convert
+
+DISCLAIMER: This is like the entire `github.com/docker/cli/cli/compose/convert`
+package. This should be an easy import but importing it creates DEPENDENCY
+HELL. I tried for an hour to fix it but it would work. TRY TO FIX AT YOUR OWN
+RISK!!!
+
+Due to this literally just being copy-pasted from the lib, the Apache license
+will be posted in this folder. Small edits to the source code have been to
+function names and parts we don't need deleted.

pkg/client/convert/compose.go

@@ -1,4 +1,4 @@
-package convert // https://github.com/docker/cli/blob/master/cli/command/container/tty.go
+package convert
 
 import (
 	"io/ioutil"

pkg/client/convert/compose_test.go

@@ -1,4 +1,4 @@
-package convert // https://github.com/docker/cli/blob/master/cli/compose/convert/compose_test.go
+package convert
 
 import (
 	"testing"

pkg/client/convert/service.go

@@ -1,4 +1,4 @@
-package convert // https://github.com/docker/cli/blob/master/cli/compose/convert/service.go
+package convert
 
 import (
 	"context"
@@ -399,6 +399,7 @@ func convertServiceNetworks(
 	return nets, nil
 }
 
+// TODO: fix secrets API so that SecretAPIClient is not required here
 func convertServiceSecrets(
 	client client.SecretAPIClient,
 	namespace Namespace,
@@ -420,12 +421,6 @@ func convertServiceSecrets(
 			return nil, err
 		}
 
-		// NOTE(d1): strip # length=... modifiers
-		if strings.Contains(obj.Name, "#") {
-			vals := strings.Split(obj.Name, "#")
-			obj.Name = strings.TrimSpace(vals[0])
-		}
-
 		file := swarm.SecretReferenceFileTarget(obj.File)
 		refs = append(refs, &swarm.SecretReference{
 			File:       &file,
@@ -447,6 +442,8 @@ func convertServiceSecrets(
 // required by the serivce. Unlike convertServiceSecrets, this takes the whole
 // ServiceConfig, because some Configs may be needed as a result of other
 // fields (like CredentialSpecs).
+//
+// TODO: fix configs API so that ConfigsAPIClient is not required here
 func convertServiceConfigObjs(
 	client client.ConfigAPIClient,
 	namespace Namespace,
@@ -629,6 +626,7 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 }
 
 func convertRestartPolicy(restart string, source *composetypes.RestartPolicy) (*swarm.RestartPolicy, error) {
+	// TODO: log if restart is being ignored
 	if source == nil {
 		policy, err := opts.ParseRestartPolicy(restart)
 		if err != nil {

pkg/client/convert/service_test.go

@@ -1,4 +1,4 @@
-package convert // https://github.com/docker/cli/blob/master/cli/compose/convert/service_test.go
+package convert
 
 import (
 	"context"

pkg/client/convert/volume.go

@@ -1,4 +1,4 @@
-package convert // https://github.com/docker/cli/blob/master/cli/compose/convert/volume.go
+package convert
 
 import (
 	composetypes "github.com/docker/cli/cli/compose/types"

pkg/client/convert/volume_test.go

@@ -1,4 +1,4 @@
-package convert // https://github.com/docker/cli/blob/master/cli/compose/convert/volume_test.go
+package convert
 
 import (
 	"testing"

pkg/client/registry.go

@@ -1,28 +1,170 @@
 package client
 
 import (
-	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
 
-	"github.com/containers/image/docker"
+	"coopcloud.tech/abra/pkg/web"
-	"github.com/containers/image/types"
 	"github.com/docker/distribution/reference"
 )
 
-// GetRegistryTags retrieves all tags of an image from a container registry.
+type RawTag struct {
-func GetRegistryTags(img reference.Named) ([]string, error) {
+	Layer string
-	var tags []string
+	Name  string
+}
 
-	ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
+type RawTags []RawTag
-	if err != nil {
-		return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
-	}
 
-	ctx := context.Background()
+var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
-	tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
+
-	if err != nil {
+func GetRegistryTags(image string) (RawTags, error) {
+	var tags RawTags
+
+	tagsUrl := fmt.Sprintf(registryURL, image)
+	if err := web.ReadJSON(tagsUrl, &tags); err != nil {
 		return tags, err
 	}
 
 	return tags, nil
 }
+
+// getRegv2Token retrieves a registry v2 authentication token.
+func getRegv2Token(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
+	req, err := http.NewRequest("GET", authTokenURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	client := &http.Client{Timeout: web.Timeout}
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", nil
+	}
+
+	tokenRes := struct {
+		Token  string
+		Expiry string
+		Issued string
+	}{}
+
+	if err := json.Unmarshal(body, &tokenRes); err != nil {
+		return "", err
+	}
+
+	return tokenRes.Token, nil
+}
+
+// GetTagDigest retrieves an image digest from a v2 registry
+func GetTagDigest(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	tag := image.(reference.NamedTagged).Tag()
+	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
+
+	req, err := http.NewRequest("GET", manifestURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := getRegv2Token(image)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header = http.Header{
+		"Accept": []string{
+			"application/vnd.docker.distribution.manifest.v2+json",
+			"application/vnd.docker.distribution.manifest.list.v2+json",
+		},
+		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
+	}
+
+	client := &http.Client{Timeout: web.Timeout}
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+
+	registryResT1 := struct {
+		SchemaVersion int
+		MediaType     string
+		Manifests     []struct {
+			MediaType string
+			Size      int
+			Digest    string
+			Platform  struct {
+				Architecture string
+				Os           string
+			}
+		}
+	}{}
+
+	registryResT2 := struct {
+		SchemaVersion int
+		MediaType     string
+		Config        struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+		Layers []struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+	}{}
+
+	if err := json.Unmarshal(body, &registryResT1); err != nil {
+		return "", err
+	}
+
+	var digest string
+	for _, manifest := range registryResT1.Manifests {
+		if string(manifest.Platform.Architecture) == "amd64" {
+			digest = strings.Split(manifest.Digest, ":")[1][:7]
+		}
+	}
+
+	if digest == "" {
+		if err := json.Unmarshal(body, &registryResT2); err != nil {
+			return "", err
+		}
+		digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
+	}
+
+	if digest == "" {
+		return "", fmt.Errorf("Unable to retrieve amd64 digest for '%s'", image)
+	}
+
+	return digest, nil
+}

pkg/client/secret.go

@@ -4,14 +4,20 @@ import (
 	"context"
 
 	"github.com/docker/docker/api/types/swarm"
-	"github.com/docker/docker/client"
 )
 
-func StoreSecret(cl *client.Client, secretName, secretValue, server string) error {
+func StoreSecret(secretName, secretValue, server string) error {
+	cl, err := New(server)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
 	ann := swarm.Annotations{Name: secretName}
 	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
 
-	if _, err := cl.SecretCreate(context.Background(), spec); err != nil {
+	// We don't bother with the secret IDs for now
+	if _, err := cl.SecretCreate(ctx, spec); err != nil {
 		return err
 	}
 

pkg/client/stack/loader.go

@@ -1,4 +1,4 @@
-package stack // https://github.com/docker/cli/blob/master/cli/command/stack/loader/loader.go
+package stack
 
 import (
 	"fmt"
@@ -13,11 +13,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// DontSkipValidation ensures validation is done for compose file loading
-func DontSkipValidation(opts *loader.Options) {
-	opts.SkipValidation = false
-}
-
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
 func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
@@ -26,30 +21,26 @@ func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Confi
 	}
 
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails, DontSkipValidation)
+	config, err := loader.Load(configDetails)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
-			return nil, fmt.Errorf("compose file contains unsupported options: %s",
+			return nil, fmt.Errorf("compose file contains unsupported options:\n\n%s",
 				propertyWarnings(fpe.Properties))
 		}
-		return nil, err
-	}
 
-	recipeName, exists := appEnv["RECIPE"]
+		return nil, err
-	if !exists {
-		recipeName, _ = appEnv["TYPE"]
 	}
 
 	unsupportedProperties := loader.GetUnsupportedProperties(dicts...)
 	if len(unsupportedProperties) > 0 {
-		logrus.Warnf("%s: ignoring unsupported options: %s",
+		logrus.Warnf("Ignoring unsupported options: %s\n\n",
-			recipeName, strings.Join(unsupportedProperties, ", "))
+			strings.Join(unsupportedProperties, ", "))
 	}
 
 	deprecatedProperties := loader.GetDeprecatedProperties(dicts...)
 	if len(deprecatedProperties) > 0 {
-		logrus.Warnf("%s: ignoring deprecated options: %s",
+		logrus.Warnf("Ignoring deprecated options:\n\n%s\n\n",
-			recipeName, propertyWarnings(deprecatedProperties))
+			propertyWarnings(deprecatedProperties))
 	}
 	return config, nil
 }