build: disable publish image for now

It's failing for unknown reasons and block releases.

See coop-cloud/recipes-catalogue-json#6

.dockerignore

@@ -0,0 +1,4 @@
+Dockerfile
+.dockerignore
+*.swp
+*.swo

.drone.yml

@@ -1,43 +1,69 @@
 ---
 kind: pipeline
-name: linters
+name: coopcloud.tech/abra
 steps:
-  - name: run shellcheck
-    image: koalaman/shellcheck-alpine:v0.7.1
+  - name: make check
+    image: golang:1.21
     commands:
-      - shellcheck abra
+      - make check
 
-  - name: run unit tests
-    image: decentral1se/docker-dind-bats-kcov
+  - name: make build
+    image: golang:1.21
     commands:
-      - bats tests
+      - make build
+    depends_on:
+      - make check
 
-  - name: collect code coverage
-    failure: ignore # until we fix this
-    image: decentral1se/docker-dind-bats-kcov
+  - name: make test
+    image: golang:1.21
     commands:
-      - kcov . bats tests || true
+      - make test
+    depends_on:
+      - make check
 
-  - name: send code coverage report to codecov
-    failure: ignore # until we fix this
-    image: plugins/codecov
-    settings:
-      token:
-        from_secret: codecov_token
-        required: true
-
-  - name: notify rocket chat
-    image: plugins/slack
-    settings:
-      webhook:
-        from_secret: rc_builds_url
-      username: comradebritney
-      channel: "internal.builds"
-      template: "{{repo.owner}}/{{repo.name}} build failed: {{build.link}}"
+  - name: fetch
+    image: docker:git
+    commands:
+      - git fetch --tags
+    depends_on:
+      - make check
+      - make build
+      - make test
     when:
-      status:
-        - failure
+      event: tag
 
-trigger:
-  branch:
-    - main
+  - name: release
+    image: goreleaser/goreleaser:v1.18.2
+    environment:
+      GITEA_TOKEN:
+        from_secret: goreleaser_gitea_token
+    volumes:
+      - name: deps
+        path: /go
+    commands:
+      - goreleaser release
+    depends_on:
+      - fetch
+    when:
+      event: tag
+
+  # - name: publish image
+  #   image: plugins/docker
+  #   settings:
+  #     auto_tag: true
+  #     username: 3wordchant
+  #     password:
+  #       from_secret: git_coopcloud_tech_token_3wc
+  #     repo: git.coopcloud.tech/coop-cloud/abra
+  #     tags: dev
+  #     registry: git.coopcloud.tech
+  #   when:
+  #     event:
+  #       exclude:
+  #         - pull_request
+  #   depends_on:
+  #     - make check
+
+volumes:
+  - name: deps
+    temp: {}

.envrc.sample

@@ -0,0 +1,7 @@
+go env -w GOPRIVATE=coopcloud.tech
+
+# export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
+
+# export ABRA_DIR="$HOME/.abra_test"
+# export ABRA_TEST_DOMAIN=test.example.com
+# export ABRA_SKIP_TEARDOWN=1 # for faster feedback when developing tests

.gitea/ISSUE_TEMPLATE.md

@@ -0,0 +1,8 @@
+---
+name: "Do not use this issue tracker"
+about: "Do not use this issue tracker"
+title: "Do not use this issue tracker"
+labels: []
+---
+
+Please report your issue on [`coop-cloud/organising`](https://git.coopcloud.tech/coop-cloud/organising)

.gitignore

@@ -1,2 +1,9 @@
-coverage/
-/.venv
+*fmtcoverage.html
+.e2e.env
+.envrc
+.vscode/
+/kadabra
+abra
+dist/
+tests/integration/.bats
+vendor/

.goreleaser.yml

@@ -0,0 +1,76 @@
+---
+gitea_urls:
+  api: https://git.coopcloud.tech/api/v1
+  download: https://git.coopcloud.tech/
+  skip_tls_verify: false
+
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - id: abra
+    binary: abra
+    dir: cmd/abra
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+
+  - id: kadabra
+    binary: kadabra
+    dir: cmd/kadabra
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+
+archives:
+  - replacements:
+      386: i386
+      amd64: x86_64
+    format: binary
+
+checksum:
+  name_template: "checksums.txt"
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: desc
+  filters:
+    exclude:
+      - "^Merge"
+      - "^Revert"
+      - "^WIP:"
+      - "^chore(deps):"
+      - "^style:"
+      - "^test:"
+      - "^tests:"

AUTHORS.md

@@ -0,0 +1,17 @@
+# authors
+
+> If you're looking at this and you hack on `abra` and you're not listed here,
+> please do add yourself! This is a community project, let's show some 💞
+
+- 3wordchant
+- cassowary
+- codegod100
+- decentral1se
+- frando
+- kawaiipunk
+- knoflook
+- moritz
+- rix
+- roxxers
+- vera
+- yksflip

CHANGELOG.md

@@ -1,61 +0,0 @@
-# abra x.x.x (UNRELEASED)
-
-# abra 0.5.0 (2021-03-01)
-
-- `secret auto` merged into `secret generate` and `app new --auto` is now `app new --secrets` ([#64](https://git.autonomic.zone/coop-cloud/abra/pulls/64))
-- Avoid outputting length during secret generation when not in use ([#67](https://git.autonomic.zone/coop-cloud/abra/issues/67))
-- Support graceful failure when missing secret generation commands ([44d3ac3](https://git.autonomic.zone/coop-cloud/abra/commit/44d3ac3a1cb86edc9b9e91eea1a00e70eae14965))
-- Fix secret detection when using new `.env` file format in apps ([5532452](https://git.autonomic.zone/coop-cloud/abra/commit/55324524ca77141666ffe6cc41b62cc71cf89ace))
-- Support choosing an `$EDITOR` when editing configs ([29cc392](https://git.autonomic.zone/coop-cloud/abra/commit/29cc392dff3e93e48e0e2edd3ce11b405c66a95a))
-- "server" shell completion fixed ([8839bd4](https://git.autonomic.zone/coop-cloud/abra/commit/8839bd45951d00dccf4ef81ece445bcc49e13ee6))
-- Drop `multilogs` command ([#56](https://git.autonomic.zone/coop-cloud/abra/pulls/56))
-- Remove `server use` command ([#51](https://git.autonomic.zone/coop-cloud/abra/issues/51))
-- `new <app>` becomes `new <type>` ([#48](https://git.autonomic.zone/coop-cloud/abra/issues/48))
-- `check` is run on `deploy` now and configurable ([77ba565](https://git.autonomic.zone/coop-cloud/abra/commit/77ba5652b2fe15820f5edfa0f642636f7b8eae7e))
-- App configurations are always updated now ([#42](https://git.autonomic.zone/coop-cloud/abra/issues/42))
-- We use docker format `.env` files (no "export" syntax) from now now ([#55](https://git.autonomic.zone/coop-cloud/abra/pulls/55))
-- Rename `<domain>` option to `<app>` and `APP` variable to `TYPE`, see ([#47](https://git.autonomic.zone/coop-cloud/abra/issues/47))
-- Use Docker-in-Docker (dind), and `dind-bats-kcov` Docker image, for `make test` ([1600b62](https://git.autonomic.zone/coop-cloud/abra/commit/1600b6277fbbffc4c6de1e4ba799c7bbe72ec6a0))
-- Add built-in documentation using `abra help <subcommand>...`, see ([#50](https://git.autonomic.zone/coop-cloud/abra/issues/50))
-- `version` subcommand ([e6b24fe](https://git.autonomic.zone/coop-cloud/abra/commit/e6b24fe))
-- Use `# length=x` comments to generate passwords with `pwgen` and drop `KEY`/`PASSWORD` logic ([#68](https://git.autonomic.zone/coop-cloud/abra/issues/68))
-- Global `--skip-update|-U` / `--skip-check|-C` options to make things quicker ([37e8b00](https://git.autonomic.zone/coop-cloud/abra/commit/37e8b00))
-- `app backup` and `app restore` commands; requires per-app definition ([#70](https://git.autonomic.zone/coop-cloud/abra/issues/70))
-- Rename per-type `abra-commands.sh` to `abra.sh`, and include config versions as type-level instead of app-level config ([#43](https://git.autonomic.zone/coop-cloud/abra/issues/43))
-- Show per-subcommand help by adding `-h/--help` to a command line ([#38](https://git.autonomic.zone/coop-cloud/abra/issues/78))
-
-# abra 0.4.1 (2020-12-24)
-
-- Bug-fixes on `app ls --status` & custom commands
-- Add `app ls --server=...` and alias
-
-# abra 0.4.0 (2020-12-24)
-
-- New command-line interface based on docopt
-- `~/.abra` directory instead of expecting local `.env` files
-- Integration tests & code coverage
-
-# abra 0.3.1 (2020-09-27)
-
-- Fix installer version
-
-# abra 0.3.0 (2020-09-27)
-
-- Add multilogs stack logs implementation ([#8](https://git.autonomic.zone/compose-stacks/abra/issues/8))
-- Add beginnings of "monorepo" functionality
-
-# abra 0.2.0 (2020-09-24)
-
-- Prepare for swarm install script using script.d ([#12](https://git.autonomic.zone/compose-stacks/planning/issues/12))
-
-# abra 0.1.2 (2020-09-22)
-
-- Add upgrade command ([#10](https://git.autonomic.zone/autonomic-cooperative/abra/issues/10))
-
-# abra 0.1.1 (2020-09-22)
-
-- Add installer script ([#9](https://git.autonomic.zone/autonomic-cooperative/abra/issues/9))
-
-# abra 0.1.0 (2020-09-22)
-
-- Initial pre-alpha release

Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.21-alpine AS build
+
+ENV GOPRIVATE coopcloud.tech
+
+RUN apk add --no-cache make git gcc musl-dev
+
+COPY . /app
+
+WORKDIR /app
+
+RUN CGO_ENABLED=0 make build
+
+FROM scratch
+
+COPY --from=build /app/abra /abra
+
+ENTRYPOINT ["/abra"]

LICENSE

@@ -0,0 +1,15 @@
+Abra: The Co-op Cloud utility belt
+Copyright (C) 2022 Co-op Cloud <helo@coopcloud.tech>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

Makefile

@@ -1,45 +1,49 @@
-.PHONY: test shellcheck docopt kcov codecov
+ABRA         := ./cmd/abra
+KADABRA      := ./cmd/kadabra
+COMMIT       := $(shell git rev-list -1 HEAD)
+GOPATH       := $(shell go env GOPATH)
+LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
+DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+
+export GOPRIVATE=coopcloud.tech
+
+all: format check build test
+
+run-abra:
+	@go run -ldflags=$(LDFLAGS) $(ABRA)
+
+run-kadabra:
+	@go run -ldflags=$(LDFLAGS) $(KADABRA)
+
+install-abra:
+	@go install -ldflags=$(LDFLAGS) $(ABRA)
+
+install-kadaabra:
+	@go install -ldflags=$(LDFLAGS) $(KADABRA)
+
+build-abra:
+	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
+
+build-kadabra:
+	@go build -v -ldflags=$(LDFLAGS) $(KADABRA)
+
+build:
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
+
+clean:
+	@rm '$(GOPATH)/bin/abra'
+	@rm '$(GOPATH)/bin/kadabra'
+
+format:
+	@gofmt -s -w .
+
+check:
+	@test -z $$(gofmt -l .) || \
+	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
 
 test:
-	@sudo DOCKER_CONTEXT=default docker run \
-		-v $$(pwd):/workdir \
-		--privileged \
-		-d \
-		--name=abra-test-dind \
-		-e DOCKER_TLS_CERTDIR="" \
-		decentral1se/docker-dind-bats-kcov
-	@DOCKER_CONTEXT=default docker exec \
-		-it \
-		abra-test-dind \
-		sh -c "cd /workdir && bats /workdir/tests"
-	@DOCKER_CONTEXT=default docker stop abra-test-dind
-	@DOCKER_CONTEXT=default docker rm abra-test-dind
+	@go test ./... -cover -v
 
-shellcheck:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		koalaman/shellcheck-alpine \
-		shellcheck /workdir/abra
-
-docopt:
-	@if [ ! -d ".venv" ]; then \
-		python3 -m venv .venv && \
-		.venv/bin/pip install -U pip setuptools wheel && \
-		.venv/bin/pip install docopt-sh; \
-	fi
-	.venv/bin/docopt.sh abra
-
-kcov:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		kcov/kcov:latest \
-		sh -c "kcov /workdir/coverage /workdir/abra || true"
-
-codecov: SHELL:=/bin/bash
-codecov:
-	@bash <(curl -s https://codecov.io/bash) \
-		-s coverage -t $$(pass show hosts/swarm.autonomic.zone/drone/codecov/token)
+loc:
+	@find . -name "*.go" | xargs wc -l

README.md

@@ -1,18 +1,13 @@
-# abra
+# `abra`
 
-[![Build Status](https://drone.autonomic.zone/api/badges/coop-cloud/abra/status.svg)](https://drone.autonomic.zone/coop-cloud/abra)
-[![codecov](https://codecov.io/gh/Autonomic-Cooperative/abra/branch/main/graph/badge.svg?token=aX3I5NMRsj)](undefined)
+[![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
+[![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
 
-> https://cloud.autonomic.zone
+The Co-op Cloud utility belt 🎩🐇
 
-The cooperative cloud utility belt 🎩🐇
+<a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-## Documentation
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community 💖
 
-> [docs.cloud.autonomic.zone](https://docs.cloud.autonomic.zone/)
-
-## Install
-
-```sh
-curl https://install.abra.autonomic.zone | bash
-```
+Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

cli/app/app.go

@@ -0,0 +1,37 @@
+package app
+
+import (
+	"github.com/urfave/cli"
+)
+
+var AppCommand = cli.Command{
+	Name:        "app",
+	Aliases:     []string{"a"},
+	Usage:       "Manage apps",
+	ArgsUsage:   "<domain>",
+	Description: "Functionality for managing the life cycle of your apps",
+	Subcommands: []cli.Command{
+		appBackupCommand,
+		appCheckCommand,
+		appCmdCommand,
+		appConfigCommand,
+		appCpCommand,
+		appDeployCommand,
+		appErrorsCommand,
+		appListCommand,
+		appLogsCommand,
+		appNewCommand,
+		appPsCommand,
+		appRemoveCommand,
+		appRestartCommand,
+		appRestoreCommand,
+		appRollbackCommand,
+		appRunCommand,
+		appSecretCommand,
+		appServicesCommand,
+		appUndeployCommand,
+		appUpgradeCommand,
+		appVersionCommand,
+		appVolumeCommand,
+	},
+}

cli/app/backup.go

@@ -0,0 +1,414 @@
+package app
+
+import (
+	"archive/tar"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/klauspost/pgzip"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type backupConfig struct {
+	preHookCmd  string
+	postHookCmd string
+	backupPaths []string
+}
+
+var appBackupCommand = cli.Command{
+	Name:      "backup",
+	Aliases:   []string{"bk"},
+	Usage:     "Run app backup",
+	ArgsUsage: "<domain> [<service>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app backup.
+
+A backup command and pre/post hook commands are defined in the recipe
+configuration. Abra reads this configuration and run the comands in the context
+of the deployed services. Pass <service> if you only want to back up a single
+service. All backups are placed in the ~/.abra/backups directory.
+
+A single backup file is produced for all backup paths specified for a service.
+If we have the following backup configuration:
+
+    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
+
+And we run "abra app backup example.com app", Abra will produce a file that
+looks like:
+
+    ~/.abra/backups/example_com_app_609341138.tar.gz
+
+This file is a compressed archive which contains all backup paths. To see paths, run:
+
+    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
+
+(Make sure to change the name of the backup file)
+
+This single file can be used to restore your app. See "abra app restore" for more.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		backupConfigs := make(map[string]backupConfig)
+		for _, service := range recipe.Config.Services {
+			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
+				if backupsEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					bkConfig := backupConfig{}
+
+					logrus.Debugf("backup config detected for %s", fullServiceName)
+
+					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
+						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
+						bkConfig.backupPaths = strings.Split(paths, ",")
+					}
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						bkConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						bkConfig.postHookCmd = postHookCmd
+					}
+
+					backupConfigs[service.Name] = bkConfig
+				}
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName != "" {
+			backupConfig, ok := backupConfigs[serviceName]
+			if !ok {
+				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
+			}
+
+			logrus.Infof("running backup for the %s service", serviceName)
+
+			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if len(backupConfigs) == 0 {
+				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
+			}
+
+			for serviceName, backupConfig := range backupConfigs {
+				logrus.Infof("running backup for the %s service", serviceName)
+
+				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+// TimeStamp generates a file name friendly timestamp.
+func TimeStamp() string {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	return strings.Replace(ts, ":", "-", -1)
+}
+
+// runBackup does the actual backup logic.
+func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
+	if len(bkConfig.backupPaths) == 0 {
+		return fmt.Errorf("backup paths are empty for %s?", serviceName)
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if bkConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
+	}
+
+	var tempBackupPaths []string
+	for _, remoteBackupPath := range bkConfig.backupPaths {
+		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
+		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
+
+		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
+
+		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		if err != nil {
+			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+		}
+		defer content.Close()
+
+		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
+		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
+		if err := copyToFile(localBackupPath, preArchive); err != nil {
+			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+		}
+
+		tempBackupPaths = append(tempBackupPaths, localBackupPath)
+	}
+
+	logrus.Infof("compressing and merging archives...")
+
+	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
+		logrus.Debugf("failed to merge archive files: %s", err.Error())
+		if err := cleanupTempArchives(tempBackupPaths); err != nil {
+			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+		}
+		return fmt.Errorf("failed to merge archive files: %s", err.Error())
+	}
+
+	if err := cleanupTempArchives(tempBackupPaths); err != nil {
+		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+	}
+
+	if bkConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
+	}
+
+	return nil
+}
+
+func copyToFile(outfile string, r io.Reader) error {
+	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
+	if err != nil {
+		return err
+	}
+
+	tmpPath := tmpFile.Name()
+
+	_, err = io.Copy(tmpFile, r)
+	tmpFile.Close()
+
+	if err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	if err = os.Rename(tmpPath, outfile); err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	return nil
+}
+
+func cleanupTempArchives(tarPaths []string) error {
+	for _, tarPath := range tarPaths {
+		if err := os.RemoveAll(tarPath); err != nil {
+			return err
+		}
+
+		logrus.Debugf("remove temporary archive file %s", tarPath)
+	}
+
+	return nil
+}
+
+func mergeArchives(tarPaths []string, serviceName string) error {
+	var out io.Writer
+	var cout *pgzip.Writer
+
+	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
+
+	fout, err := os.Create(localBackupPath)
+	if err != nil {
+		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
+	}
+
+	defer fout.Close()
+	out = fout
+
+	cout = pgzip.NewWriter(out)
+	out = cout
+
+	tw := tar.NewWriter(out)
+
+	for _, tarPath := range tarPaths {
+		if err := addTar(tw, tarPath); err != nil {
+			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		return fmt.Errorf("failed to close tar writer %v", err)
+	}
+
+	if cout != nil {
+		if err := cout.Flush(); err != nil {
+			return fmt.Errorf("failed to flush: %s", err)
+		} else if err = cout.Close(); err != nil {
+			return fmt.Errorf("failed to close compressed writer: %s", err)
+		}
+	}
+
+	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
+
+	return nil
+}
+
+func addTar(tw *tar.Writer, pth string) (err error) {
+	var tr *tar.Reader
+	var rc io.ReadCloser
+	var hdr *tar.Header
+
+	if tr, rc, err = openTarFile(pth); err != nil {
+		return
+	}
+
+	for {
+		if hdr, err = tr.Next(); err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			break
+		}
+		if err = tw.WriteHeader(hdr); err != nil {
+			break
+		} else if _, err = io.Copy(tw, tr); err != nil {
+			break
+		}
+	}
+	if err == nil {
+		err = rc.Close()
+	} else {
+		rc.Close()
+	}
+	return
+}
+
+func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
+	var fin *os.File
+	var n int
+	buff := make([]byte, 1024)
+
+	if fin, err = os.Open(pth); err != nil {
+		return
+	}
+
+	if n, err = fin.Read(buff); err != nil {
+		fin.Close()
+		return
+	} else if n == 0 {
+		fin.Close()
+		err = fmt.Errorf("%s is empty", pth)
+		return
+	}
+
+	if _, err = fin.Seek(0, 0); err != nil {
+		fin.Close()
+		return
+	}
+
+	rc = fin
+	tr = tar.NewReader(rc)
+
+	return tr, rc, nil
+}

cli/app/check.go

@@ -0,0 +1,81 @@
+package app
+
+import (
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appCheckCommand = cli.Command{
+	Name:      "check",
+	Aliases:   []string{"chk"},
+	Usage:     "Check if an app is configured correctly",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.ChaosFlag,
+		internal.OfflineFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
+		if _, err := os.Stat(envSamplePath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist?", envSamplePath)
+			}
+			logrus.Fatal(err)
+		}
+
+		envSample, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var missing []string
+		for k := range envSample {
+			if _, ok := app.Env[k]; !ok {
+				missing = append(missing, k)
+			}
+		}
+
+		if len(missing) > 0 {
+			missingEnvVars := strings.Join(missing, ", ")
+			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
+		}
+
+		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+
+		return nil
+	},
+}

cli/app/cmd.go

@@ -0,0 +1,188 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appCmdCommand = cli.Command{
+	Name:    "command",
+	Aliases: []string{"cmd"},
+	Usage:   "Run app commands",
+	Description: `
+Run an app specific command.
+
+These commands are bash functions, defined in the abra.sh of the recipe itself.
+They can be run within the context of a service (e.g. app) or locally on your
+work station by passing "--local". Arguments can be passed into these functions
+using the "-- <args>" syntax.
+
+Example:
+
+  abra app cmd example.com app create_user -- me@example.com
+`,
+	ArgsUsage: "<domain> [<service>] <command> [-- <args>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.LocalCmdFlag,
+		internal.RemoteUserFlag,
+		internal.TtyFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.LocalCmd && internal.RemoteUser != "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
+		}
+
+		hasCmdArgs, parsedCmdArgs := parseCmdArgs(c.Args(), internal.LocalCmd)
+
+		abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist for %s?", abraSh, app.Name)
+			}
+			logrus.Fatal(err)
+		}
+
+		if internal.LocalCmd {
+			if !(len(c.Args()) >= 2) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
+			cmdName := c.Args().Get(1)
+			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
+				logrus.Fatal(err)
+			}
+
+			logrus.Debugf("--local detected, running %s on local work station", cmdName)
+
+			var exportEnv string
+			for k, v := range app.Env {
+				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
+			}
+
+			var sourceAndExec string
+			if hasCmdArgs {
+				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName, parsedCmdArgs)
+			} else {
+				logrus.Debug("did not detect any command arguments")
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName)
+			}
+
+			shell := "/bin/bash"
+			if _, err := os.Stat(shell); errors.Is(err, os.ErrNotExist) {
+				logrus.Debugf("%s does not exist locally, use /bin/sh as fallback", shell)
+				shell = "/bin/sh"
+			}
+			cmd := exec.Command(shell, "-c", sourceAndExec)
+
+			if err := internal.RunCmd(cmd); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if !(len(c.Args()) >= 3) {
+				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+			}
+
+			targetServiceName := c.Args().Get(1)
+
+			cmdName := c.Args().Get(2)
+			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
+				logrus.Fatal(err)
+			}
+
+			serviceNames, err := config.GetAppServiceNames(app.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			matchingServiceName := false
+			for _, serviceName := range serviceNames {
+				if serviceName == targetServiceName {
+					matchingServiceName = true
+				}
+			}
+
+			if !matchingServiceName {
+				logrus.Fatalf("no service %s for %s?", targetServiceName, app.Name)
+			}
+
+			logrus.Debugf("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName)
+
+			if hasCmdArgs {
+				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
+			} else {
+				logrus.Debug("did not detect any command arguments")
+			}
+
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := internal.RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+func parseCmdArgs(args []string, isLocal bool) (bool, string) {
+	var (
+		parsedCmdArgs string
+		hasCmdArgs    bool
+	)
+
+	if isLocal {
+		if len(args) > 2 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[2:], " "))
+		}
+	} else {
+		if len(args) > 3 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[3:], " "))
+		}
+	}
+
+	return hasCmdArgs, parsedCmdArgs
+}

cli/app/cmd_test.go

@@ -0,0 +1,31 @@
+package app
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseCmdArgs(t *testing.T) {
+	tests := []struct {
+		input          []string
+		shouldParse    bool
+		expectedOutput string
+	}{
+		// `--` is not parsed when passed in from the command-line e.g. -- foo bar baz
+		// so we need to eumlate that as missing when testing if bash args are passed in
+		// see https://git.coopcloud.tech/coop-cloud/organising/issues/336 for more
+		{[]string{"foo.com", "app", "test"}, false, ""},
+		{[]string{"foo.com", "app", "test", "foo"}, true, "foo "},
+		{[]string{"foo.com", "app", "test", "foo", "bar", "baz"}, true, "foo bar baz "},
+	}
+
+	for _, test := range tests {
+		ok, parsed := parseCmdArgs(test.input, false)
+		if ok != test.shouldParse {
+			t.Fatalf("[%s] should not parse", strings.Join(test.input, " "))
+		}
+		if parsed != test.expectedOutput {
+			t.Fatalf("%s does not match %s", parsed, test.expectedOutput)
+		}
+	}
+}

cli/app/config.go

@@ -0,0 +1,64 @@
+package app
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appConfigCommand = cli.Command{
+	Name:      "config",
+	Aliases:   []string{"cfg"},
+	Usage:     "Edit app config",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		appName := c.Args().First()
+
+		if appName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+		}
+
+		files, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		appFile, exists := files[appName]
+		if !exists {
+			logrus.Fatalf("cannot find app with name %s", appName)
+		}
+
+		ed, ok := os.LookupEnv("EDITOR")
+		if !ok {
+			edPrompt := &survey.Select{
+				Message: "Which editor do you wish to use?",
+				Options: []string{"vi", "vim", "nvim", "nano", "pico", "emacs"},
+			}
+			if err := survey.AskOne(edPrompt, &ed); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		cmd := exec.Command(ed, appFile.Path)
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/cp.go

@@ -0,0 +1,149 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appCpCommand = cli.Command{
+	Name:      "cp",
+	Aliases:   []string{"c"},
+	ArgsUsage: "<domain> <src> <dst>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Usage:  "Copy files to/from a deployed app service",
+	Description: `
+Copy files to and from any app service file system.
+
+If you want to copy a myfile.txt to the root of the app service:
+
+    abra app cp <domain> myfile.txt app:/
+
+And if you want to copy that file back to your current working directory locally:
+
+    abra app cp <domain> app:/myfile.txt .
+`,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		src := c.Args().Get(1)
+		dst := c.Args().Get(2)
+		if src == "" {
+			logrus.Fatal("missing <src> argument")
+		} else if dst == "" {
+			logrus.Fatal("missing <dest> argument")
+		}
+
+		parsedSrc := strings.SplitN(src, ":", 2)
+		parsedDst := strings.SplitN(dst, ":", 2)
+		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
+		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
+			logrus.Fatal(errorMsg)
+		} else if len(parsedSrc) != 2 {
+			if len(parsedDst) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		} else if len(parsedDst) != 2 {
+			if len(parsedSrc) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		}
+
+		var service string
+		var srcPath string
+		var dstPath string
+		isToContainer := false // <container:src> <dst>
+		if len(parsedSrc) == 2 {
+			service = parsedSrc[0]
+			srcPath = parsedSrc[1]
+			dstPath = dst
+			logrus.Debugf("assuming transfer is coming FROM the container")
+		} else if len(parsedDst) == 2 {
+			service = parsedDst[0]
+			dstPath = parsedDst[1]
+			srcPath = src
+			isToContainer = true // <src> <container:dst>
+			logrus.Debugf("assuming transfer is going TO the container")
+		}
+
+		if isToContainer {
+			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", srcPath)
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+func configureAndCp(
+	c *cli.Context,
+	cl *dockerClient.Client,
+	app config.App,
+	srcPath string,
+	dstPath string,
+	service string,
+	isToContainer bool) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
+
+	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+
+	if isToContainer {
+		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		content, err := archive.TarWithOptions(srcPath, toTarOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		defer content.Close()
+		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	return nil
+}

cli/app/deploy.go

@@ -0,0 +1,227 @@
+package app
+
+import (
+	"context"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appDeployCommand = cli.Command{
+	Name:      "deploy",
+	Aliases:   []string{"d"},
+	Usage:     "Deploy an app",
+	ArgsUsage: "<domain> [<version>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.ForceFlag,
+		internal.ChaosFlag,
+		internal.NoDomainChecksFlag,
+		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Deploy an app. It does not support incrementing the version of a deployed app,
+for this you need to look at the "abra app upgrade <domain>" command.
+
+You may pass "--force" to re-deploy the same version again. This can be useful
+if the container runtime has gotten into a weird state.
+
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
+including unstaged changes and can be useful for live hacking and testing new
+recipes.
+`,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		r, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(r); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			if internal.Force || internal.Chaos {
+				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
+			} else {
+				logrus.Fatalf("%s is already deployed", app.Name)
+			}
+		}
+
+		version := deployedVersion
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			version = specificVersion
+			logrus.Debugf("choosing %s as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if !internal.Chaos && specificVersion == "" {
+			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if len(versions) == 0 && !internal.Chaos {
+				logrus.Warn("no published versions in catalogue, trying local recipe repository")
+				recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+				if err != nil {
+					logrus.Warn(err)
+				}
+				for _, recipeVersion := range recipeVersions {
+					for version := range recipeVersion {
+						versions = append(versions, version)
+					}
+				}
+			}
+
+			if len(versions) > 0 && !internal.Chaos {
+				version = versions[len(versions)-1]
+				logrus.Debugf("choosing %s as version to deploy", version)
+				if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				head, err := git.GetRecipeHead(app.Recipe)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				version = formatter.SmallSHA(head.String())
+				logrus.Warn("no versions detected, using latest commit")
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warnf("chaos mode engaged")
+			var err error
+			version, err = recipe.ChaosVersion(app.Recipe)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, version)
+		config.SetUpdateLabel(compose, stackName, app.Env)
+
+		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.NoDomainChecks {
+			domainName, ok := app.Env["DOMAIN"]
+			if ok {
+				if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
+			}
+		} else {
+			logrus.Warn("skipping domain checks as requested")
+		}
+
+		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_DEPLOY_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+			}
+		}
+		return nil
+	},
+}

cli/app/errors.go

@@ -0,0 +1,142 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appErrorsCommand = cli.Command{
+	Name:      "errors",
+	Usage:     "List errors for a deployed app",
+	ArgsUsage: "<domain>",
+	Description: `
+List errors for a deployed app.
+
+This is a best-effort implementation and an attempt to gather a number of tips
+& tricks for finding errors together into one convenient command. When an app
+is failing to deploy or having issues, it could be a lot of things.
+
+This command currently takes into account:
+
+    Is the service deployed?
+    Is the service killed by an OOM error?
+    Is the service reporting an error (like in "ps --no-trunc" output)
+    Is the service healthcheck failing? what are the healthcheck logs?
+
+Got any more ideas? Please let us know:
+
+    https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose
+
+This command is best accompanied by "abra app logs <domain>" which may reveal
+further information which can help you debug the cause of an app failure via
+the logs.
+`,
+	Aliases: []string{"e"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.WatchFlag,
+		internal.OfflineFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		if !internal.Watch {
+			if err := checkErrors(c, cl, app); err != nil {
+				logrus.Fatal(err)
+			}
+			return nil
+		}
+
+		for {
+			if err := checkErrors(c, cl, app); err != nil {
+				logrus.Fatal(err)
+			}
+			time.Sleep(2 * time.Second)
+		}
+	},
+}
+
+func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
+	recipe, err := recipe.Get(app.Recipe, internal.Offline)
+	if err != nil {
+		return err
+	}
+
+	for _, service := range recipe.Config.Services {
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
+
+		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			return err
+		}
+
+		if len(containers) == 0 {
+			logrus.Warnf("%s is not up, something seems wrong", service.Name)
+			continue
+		}
+
+		container := containers[0]
+		containerState, err := cl.ContainerInspect(context.Background(), container.ID)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if containerState.State.OOMKilled {
+			logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
+		}
+
+		if containerState.State.Error != "" {
+			logrus.Warnf("%s reports this error: %s", service.Name, containerState.State.Error)
+		}
+
+		if containerState.State.Health != nil {
+			if containerState.State.Health.Status != "healthy" {
+				logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
+				logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
+				for _, log := range containerState.State.Health.Log {
+					logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func getServiceName(names []string) string {
+	containerName := strings.Join(names, " ")
+	trimmed := strings.TrimPrefix(containerName, "/")
+	return strings.Split(trimmed, ".")[0]
+}

cli/app/list.go

@@ -0,0 +1,300 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var status bool
+var statusFlag = &cli.BoolFlag{
+	Name:        "status, S",
+	Usage:       "Show app deployment status",
+	Destination: &status,
+}
+
+var recipeFilter string
+var recipeFlag = &cli.StringFlag{
+	Name:        "recipe, r",
+	Value:       "",
+	Usage:       "Show apps of a specific recipe",
+	Destination: &recipeFilter,
+}
+
+var listAppServer string
+var listAppServerFlag = &cli.StringFlag{
+	Name:        "server, s",
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &listAppServer,
+}
+
+type appStatus struct {
+	Server       string `json:"server"`
+	Recipe       string `json:"recipe"`
+	AppName      string `json:"appName"`
+	Domain       string `json:"domain"`
+	Status       string `json:"status"`
+	Chaos        string `json:"chaos"`
+	ChaosVersion string `json:"chaosVersion"`
+	AutoUpdate   string `json:"autoUpdate"`
+	Version      string `json:"version"`
+	Upgrade      string `json:"upgrade"`
+}
+
+type serverStatus struct {
+	Apps             []appStatus `json:"apps"`
+	AppCount         int         `json:"appCount"`
+	VersionCount     int         `json:"versionCount"`
+	UnversionedCount int         `json:"unversionedCount"`
+	LatestCount      int         `json:"latestCount"`
+	UpgradeCount     int         `json:"upgradeCount"`
+}
+
+var appListCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
+	Usage:   "List all managed apps",
+	Description: `
+Read the local file system listing of apps and servers (e.g. ~/.abra/) to
+generate a report of all your apps.
+
+By passing the "--status/-S" flag, you can query all your servers for the
+actual live deployment status. Depending on how many servers you manage, this
+can take some time.
+`,
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.MachineReadableFlag,
+		statusFlag,
+		listAppServerFlag,
+		recipeFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		appFiles, err := config.LoadAppFiles(listAppServer)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		apps, err := config.GetApps(appFiles, recipeFilter)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		sort.Sort(config.ByServerAndRecipe(apps))
+
+		statuses := make(map[string]map[string]string)
+		var catl recipe.RecipeCatalogue
+		if status {
+			alreadySeen := make(map[string]bool)
+			for _, app := range apps {
+				if _, ok := alreadySeen[app.Server]; !ok {
+					alreadySeen[app.Server] = true
+				}
+			}
+
+			statuses, err = config.GetAppStatuses(apps, internal.MachineReadable)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			catl, err = recipe.ReadRecipeCatalogue(internal.Offline)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		var totalServersCount int
+		var totalAppsCount int
+		allStats := make(map[string]serverStatus)
+		for _, app := range apps {
+			var stats serverStatus
+			var ok bool
+			if stats, ok = allStats[app.Server]; !ok {
+				stats = serverStatus{}
+				if recipeFilter == "" {
+					// count server, no filtering
+					totalServersCount++
+				}
+			}
+
+			if app.Recipe == recipeFilter || recipeFilter == "" {
+				if recipeFilter != "" {
+					// only count server if matches filter
+					totalServersCount++
+				}
+
+				appStats := appStatus{}
+				stats.AppCount++
+				totalAppsCount++
+
+				if status {
+					status := "unknown"
+					version := "unknown"
+					chaos := "unknown"
+					chaosVersion := "unknown"
+					autoUpdate := "unknown"
+					if statusMeta, ok := statuses[app.StackName()]; ok {
+						if currentVersion, exists := statusMeta["version"]; exists {
+							if currentVersion != "" {
+								version = currentVersion
+							}
+						}
+						if chaosDeploy, exists := statusMeta["chaos"]; exists {
+							chaos = chaosDeploy
+						}
+						if chaosDeployVersion, exists := statusMeta["chaosVersion"]; exists {
+							chaosVersion = chaosDeployVersion
+						}
+						if autoUpdateState, exists := statusMeta["autoUpdate"]; exists {
+							autoUpdate = autoUpdateState
+						}
+						if statusMeta["status"] != "" {
+							status = statusMeta["status"]
+						}
+						stats.VersionCount++
+					} else {
+						stats.UnversionedCount++
+					}
+
+					appStats.Status = status
+					appStats.Chaos = chaos
+					appStats.ChaosVersion = chaosVersion
+					appStats.Version = version
+					appStats.AutoUpdate = autoUpdate
+
+					var newUpdates []string
+					if version != "unknown" {
+						updates, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+
+						parsedVersion, err := tagcmp.Parse(version)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+
+						for _, update := range updates {
+							parsedUpdate, err := tagcmp.Parse(update)
+							if err != nil {
+								logrus.Fatal(err)
+							}
+
+							if update != version && parsedUpdate.IsGreaterThan(parsedVersion) {
+								newUpdates = append(newUpdates, update)
+							}
+						}
+					}
+
+					if len(newUpdates) == 0 {
+						if version == "unknown" {
+							appStats.Upgrade = "unknown"
+						} else {
+							appStats.Upgrade = "latest"
+							stats.LatestCount++
+						}
+					} else {
+						newUpdates = internal.ReverseStringList(newUpdates)
+						appStats.Upgrade = strings.Join(newUpdates, "\n")
+						stats.UpgradeCount++
+					}
+				}
+
+				appStats.Server = app.Server
+				appStats.Recipe = app.Recipe
+				appStats.AppName = app.Name
+				appStats.Domain = app.Domain
+
+				stats.Apps = append(stats.Apps, appStats)
+			}
+			allStats[app.Server] = stats
+		}
+
+		if internal.MachineReadable {
+			jsonstring, err := json.Marshal(allStats)
+			if err != nil {
+				logrus.Fatal(err)
+			} else {
+				fmt.Println(string(jsonstring))
+			}
+			return nil
+		}
+
+		alreadySeen := make(map[string]bool)
+		for _, app := range apps {
+			if _, ok := alreadySeen[app.Server]; ok {
+				continue
+			}
+
+			serverStat := allStats[app.Server]
+
+			tableCol := []string{"recipe", "domain"}
+			if status {
+				tableCol = append(tableCol, []string{"status", "chaos", "version", "upgrade", "autoupdate"}...)
+			}
+
+			table := formatter.CreateTable(tableCol)
+
+			for _, appStat := range serverStat.Apps {
+				tableRow := []string{appStat.Recipe, appStat.Domain}
+				if status {
+					chaosStatus := appStat.Chaos
+					if chaosStatus != "unknown" {
+						chaosEnabled, err := strconv.ParseBool(chaosStatus)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+						if chaosEnabled && appStat.ChaosVersion != "unknown" {
+							chaosStatus = appStat.ChaosVersion
+						}
+					}
+					tableRow = append(tableRow, []string{appStat.Status, chaosStatus, appStat.Version, appStat.Upgrade, appStat.AutoUpdate}...)
+				}
+				table.Append(tableRow)
+			}
+
+			if table.NumLines() > 0 {
+				table.Render()
+
+				if status {
+					fmt.Println(fmt.Sprintf(
+						"server: %s | total apps: %v | versioned: %v | unversioned: %v | latest: %v | upgrade: %v",
+						app.Server,
+						serverStat.AppCount,
+						serverStat.VersionCount,
+						serverStat.UnversionedCount,
+						serverStat.LatestCount,
+						serverStat.UpgradeCount,
+					))
+				} else {
+					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.AppCount))
+				}
+			}
+
+			if len(allStats) > 1 && table.NumLines() > 0 {
+				fmt.Println() // newline separator for multiple servers
+			}
+
+			alreadySeen[app.Server] = true
+		}
+
+		if len(allStats) > 1 {
+			fmt.Println(fmt.Sprintf("total servers: %v | total apps: %v ", totalServersCount, totalAppsCount))
+		}
+
+		return nil
+	},
+}

cli/app/logs.go

@@ -0,0 +1,145 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var logOpts = types.ContainerLogsOptions{
+	ShowStderr: true,
+	ShowStdout: true,
+	Since:      "",
+	Until:      "",
+	Timestamps: true,
+	Follow:     true,
+	Tail:       "20",
+	Details:    false,
+}
+
+// stackLogs lists logs for all stack services
+func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
+	filters, err := app.Filters(true, false)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(context.Background(), serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			if internal.StdErrOnly {
+				logOpts.ShowStdout = false
+			}
+
+			logs, err := client.ServiceLogs(context.Background(), s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+
+	wg.Wait()
+
+	os.Exit(0)
+}
+
+var appLogsCommand = cli.Command{
+	Name:      "logs",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<domain> [<service>]",
+	Usage:     "Tail app logs",
+	Flags: []cli.Flag{
+		internal.StdErrOnlyFlag,
+		internal.SinceLogsFlag,
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		logOpts.Since = internal.SinceLogs
+
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			logrus.Debugf("tailing logs for all %s services", app.Recipe)
+			stackLogs(c, app, cl)
+		} else {
+			logrus.Debugf("tailing logs for %s", serviceName)
+			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
+
+	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if internal.StdErrOnly {
+		logOpts.ShowStdout = false
+	}
+
+	logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	defer logs.Close()
+
+	_, err = io.Copy(os.Stdout, logs)
+	if err != nil && err != io.EOF {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}

cli/app/new.go

@@ -0,0 +1,258 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/jsontable"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appNewDescription = `
+Take a recipe and uses it to create a new app. This new app configuration is
+stored in your ~/.abra directory under the appropriate server.
+
+This command does not deploy your app for you. You will need to run "abra app
+deploy <domain>" to do so.
+
+You can see what recipes are available (i.e. values for the <recipe> argument)
+by running "abra recipe ls".
+
+Passing the "--secrets/-S" flag will automatically generate secrets for your
+app and store them encrypted at rest on the chosen target server. These
+generated secrets are only visible at generation time, so please take care to
+store them somewhere safe.
+
+You can use the "--pass/-P" to store these generated passwords locally in a
+pass store (see passwordstore.org for more). The pass command must be available
+on your $PATH.
+`
+
+var appNewCommand = cli.Command{
+	Name:        "new",
+	Aliases:     []string{"n"},
+	Usage:       "Create a new app",
+	Description: appNewDescription,
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.NewAppServerFlag,
+		internal.DomainFlag,
+		internal.PassFlag,
+		internal.SecretsFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if err := ensureServerFlag(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := ensureDomainFlag(recipe, internal.NewAppServer); err != nil {
+			logrus.Fatal(err)
+		}
+
+		sanitisedAppName := config.SanitiseAppName(internal.Domain)
+		logrus.Debugf("%s sanitised as %s for new app", internal.Domain, sanitisedAppName)
+
+		if err := config.TemplateAppEnvSample(
+			recipe.Name,
+			internal.Domain,
+			internal.NewAppServer,
+			internal.Domain,
+		); err != nil {
+			logrus.Fatal(err)
+		}
+
+		var secrets AppSecrets
+		var secretTable *jsontable.JSONTable
+		if internal.Secrets {
+			sampleEnv, err := recipe.SampleEnv()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			composeFiles, err := config.GetComposeFiles(recipe.Name, sampleEnv)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secretsConfig, err := secret.ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+			if err != nil {
+				return err
+			}
+
+			if err := promptForSecrets(recipe.Name, secretsConfig); err != nil {
+				logrus.Fatal(err)
+			}
+
+			cl, err := client.New(internal.NewAppServer)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secretCols := []string{"Name", "Value"}
+			secretTable = formatter.CreateTable(secretCols)
+			for name, val := range secrets {
+				secretTable.Append([]string{name, val})
+			}
+		}
+
+		if internal.NewAppServer == "default" {
+			internal.NewAppServer = "local"
+		}
+
+		tableCol := []string{"server", "recipe", "domain"}
+		table := formatter.CreateTable(tableCol)
+		table.Append([]string{internal.NewAppServer, recipe.Name, internal.Domain})
+
+		fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
+		fmt.Println("")
+		table.Render()
+		fmt.Println("")
+		fmt.Println("You can configure this app by running the following:")
+		fmt.Println(fmt.Sprintf("\n    abra app config %s", internal.Domain))
+		fmt.Println("")
+		fmt.Println("You can deploy this app by running the following:")
+		fmt.Println(fmt.Sprintf("\n    abra app deploy %s", internal.Domain))
+
+		if len(secrets) > 0 {
+			fmt.Println("")
+			fmt.Println("Here are your generated secrets:")
+			fmt.Println("")
+			secretTable.Render()
+			logrus.Warn("generated secrets are not shown again, please take note of them NOW")
+		}
+
+		return nil
+	},
+}
+
+// AppSecrets represents all app secrest
+type AppSecrets map[string]string
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) {
+	// NOTE(d1): trim to match app.StackName() implementation
+	if len(sanitisedAppName) > 45 {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
+		sanitisedAppName = sanitisedAppName[:45]
+	}
+
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if internal.Pass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(
+				secretValue,
+				secretName,
+				internal.Domain,
+				internal.NewAppServer,
+			); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return secrets, nil
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag(recipe recipe.Recipe, server string) error {
+	if internal.Domain == "" && !internal.NoInput {
+		prompt := &survey.Input{
+			Message: "Specify app domain",
+			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
+		}
+		if err := survey.AskOne(prompt, &internal.Domain); err != nil {
+			return err
+		}
+	}
+
+	if internal.Domain == "" {
+		return fmt.Errorf("no domain provided")
+	}
+
+	return nil
+}
+
+// promptForSecrets asks if we should generate secrets for a new app.
+func promptForSecrets(recipeName string, secretsConfig map[string]string) error {
+	if len(secretsConfig) == 0 {
+		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
+		return nil
+	}
+
+	if !internal.Secrets && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: "Generate app secrets?",
+		}
+		if err := survey.AskOne(prompt, &internal.Secrets); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	servers, err := config.GetServers()
+	if err != nil {
+		return err
+	}
+
+	if internal.NewAppServer == "" && !internal.NoInput {
+		prompt := &survey.Select{
+			Message: "Select app server:",
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &internal.NewAppServer); err != nil {
+			return err
+		}
+	}
+
+	if internal.NewAppServer == "" {
+		return fmt.Errorf("no server provided")
+	}
+
+	return nil
+}

cli/app/ps.go

@@ -0,0 +1,101 @@
+package app
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/buger/goterm"
+	dockerFormatter "github.com/docker/cli/cli/command/formatter"
+	"github.com/docker/docker/api/types"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appPsCommand = cli.Command{
+	Name:        "ps",
+	Aliases:     []string{"p"},
+	Usage:       "Check app status",
+	ArgsUsage:   "<domain>",
+	Description: "Show a more detailed status output of a specific deployed app",
+	Flags: []cli.Flag{
+		internal.WatchFlag,
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		if !internal.Watch {
+			showPSOutput(c, app, cl)
+			return nil
+		}
+
+		goterm.Clear()
+		for {
+			goterm.MoveCursor(1, 1)
+			showPSOutput(c, app, cl)
+			goterm.Flush()
+			time.Sleep(2 * time.Second)
+		}
+	},
+}
+
+// showPSOutput renders ps output.
+func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
+	filters, err := app.Filters(true, true)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
+	table := formatter.CreateTable(tableCol)
+
+	for _, container := range containers {
+		var containerNames []string
+		for _, containerName := range container.Names {
+			trimmed := strings.TrimPrefix(containerName, "/")
+			containerNames = append(containerNames, trimmed)
+		}
+
+		tableRow := []string{
+			service.ContainerToServiceName(container.Names, app.StackName()),
+			formatter.RemoveSha(container.Image),
+			formatter.HumanDuration(container.Created),
+			container.Status,
+			container.State,
+			dockerFormatter.DisplayablePorts(container.Ports),
+		}
+		table.Append(tableRow)
+	}
+
+	table.Render()
+}

cli/app/remove.go

@@ -0,0 +1,145 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appRemoveCommand = cli.Command{
+	Name:      "remove",
+	Aliases:   []string{"rm"},
+	ArgsUsage: "<domain>",
+	Usage:     "Remove all app data, locally and remotely",
+	Description: `
+This command removes everything related to an app which is already undeployed.
+
+By default, it will prompt for confirmation before proceeding. All secrets,
+volumes and the local app env file will be deleted.
+
+Only run this command when you are sure you want to completely remove the app
+and all associated app data. This is a destructive action, Be Careful!
+
+If you would like to delete specific volumes or secrets, please use removal
+sub-commands under "app volume" and "app secret" instead.
+
+Please note, if you delete the local app env file without removing volumes and
+secrets first, Abra will *not* be able to help you remove them afterwards.
+
+To delete everything without prompt, use the "--force/-f" or the "--no-input/n"
+flag.
+`,
+	Flags: []cli.Flag{
+		internal.ForceFlag,
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.OfflineFlag,
+	},
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if !internal.Force && !internal.NoInput {
+			response := false
+			msg := "ALERTA ALERTA: this will completely remove %s data and configurations locally and remotely, are you sure?"
+			prompt := &survey.Confirm{Message: fmt.Sprintf(msg, app.Name)}
+			if err := survey.AskOne(prompt, &response); err != nil {
+				logrus.Fatal(err)
+			}
+			if !response {
+				logrus.Fatal("aborting as requested")
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if isDeployed {
+			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+		}
+
+		fs, err := app.Filters(false, false)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: fs})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secrets := make(map[string]string)
+		var secretNames []string
+
+		for _, cont := range secretList {
+			secrets[cont.Spec.Annotations.Name] = cont.ID // we have to map the names to ID's
+			secretNames = append(secretNames, cont.Spec.Annotations.Name)
+		}
+
+		if len(secrets) > 0 {
+			for _, name := range secretNames {
+				err := cl.SecretRemove(context.Background(), secrets[name])
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("secret: %s removed", name))
+			}
+		} else {
+			logrus.Info("no secrets to remove")
+		}
+
+		fs, err = app.Filters(false, true)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		volumeListOptions := volume.ListOptions{fs}
+		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
+		volumeList := volumeListOKBody.Volumes
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var vols []string
+		for _, vol := range volumeList {
+			vols = append(vols, vol.Name)
+		}
+
+		if len(vols) > 0 {
+			for _, vol := range vols {
+				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("volume %s removed", vol))
+			}
+		} else {
+			logrus.Info("no volumes to remove")
+		}
+
+		if err = os.Remove(app.Path); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Info(fmt.Sprintf("file: %s removed", app.Path))
+
+		return nil
+	},
+}

cli/app/restart.go

@@ -0,0 +1,80 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	upstream "coopcloud.tech/abra/pkg/upstream/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appRestartCommand = cli.Command{
+	Name:      "restart",
+	Aliases:   []string{"re"},
+	Usage:     "Restart an app",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Description:  `This command restarts a service within a deployed app.`,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		serviceNameShort := c.Args().Get(1)
+		if serviceNameShort == "" {
+			err := errors.New("missing service?")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		serviceName := fmt.Sprintf("%s_%s", app.StackName(), serviceNameShort)
+
+		logrus.Debugf("attempting to scale %s to 0 (restart logic)", serviceName)
+		if err := upstream.RunServiceScale(context.Background(), cl, serviceName, 0); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.WaitOnService(context.Background(), cl, serviceName, app.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("%s has been scaled to 0 (restart logic)", serviceName)
+
+		logrus.Debugf("attempting to scale %s to 1 (restart logic)", serviceName)
+		if err := upstream.RunServiceScale(context.Background(), cl, serviceName, 1); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.WaitOnService(context.Background(), cl, serviceName, app.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("%s has been scaled to 1 (restart logic)", serviceName)
+
+		logrus.Infof("%s service successfully restarted", serviceNameShort)
+
+		return nil
+	},
+}

cli/app/restore.go

@@ -0,0 +1,223 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type restoreConfig struct {
+	preHookCmd  string
+	postHookCmd string
+}
+
+var appRestoreCommand = cli.Command{
+	Name:      "restore",
+	Aliases:   []string{"rs"},
+	Usage:     "Run app restore",
+	ArgsUsage: "<domain> <service> <file>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app restore.
+
+Pre/post hook commands are defined in the recipe configuration. Abra reads this
+configuration and run the comands in the context of the service before
+restoring the backup.
+
+Unlike "abra app backup", restore must be run on a per-service basis. You can
+not restore all services in one go. Backup files produced by Abra are
+compressed archives which use absolute paths. This allows Abra to restore
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
+
+Example:
+
+    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
+		}
+
+		backupPath := c.Args().Get(2)
+		if backupPath == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
+		}
+
+		if _, err := os.Stat(backupPath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s doesn't exist?", backupPath)
+			}
+		}
+
+		restoreConfigs := make(map[string]restoreConfig)
+		for _, service := range recipe.Config.Services {
+			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
+				if restoreEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					rsConfig := restoreConfig{}
+
+					logrus.Debugf("restore config detected for %s", fullServiceName)
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						rsConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						rsConfig.postHookCmd = postHookCmd
+					}
+
+					restoreConfigs[service.Name] = rsConfig
+				}
+			}
+		}
+
+		rsConfig, ok := restoreConfigs[serviceName]
+		if !ok {
+			rsConfig = restoreConfig{}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+// runRestore does the actual restore logic.
+func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if rsConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
+	}
+
+	backupReader, err := os.Open(backupPath)
+	if err != nil {
+		return err
+	}
+
+	content, err := archive.DecompressStream(backupReader)
+	if err != nil {
+		return err
+	}
+
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
+	restorePath := "/"
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
+		return err
+	}
+
+	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
+
+	if rsConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
+	}
+
+	return nil
+}

cli/app/rollback.go

@@ -0,0 +1,236 @@
+package app
+
+import (
+	"context"
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appRollbackCommand = cli.Command{
+	Name:      "rollback",
+	Aliases:   []string{"rl"},
+	Usage:     "Roll an app back to a previous version",
+	ArgsUsage: "<domain> [<version>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.ForceFlag,
+		internal.ChaosFlag,
+		internal.NoDomainChecksFlag,
+		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+This command rolls an app back to a previous version if one exists.
+
+You may pass "--force/-f" to downgrade to the same version again. This can be
+useful if the container runtime has gotten into a weird state.
+
+This action could be destructive, please ensure you have a copy of your app
+data beforehand.
+
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
+including unstaged changes and can be useful for live hacking and testing new
+recipes.
+`,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		r, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(r); err != nil {
+			logrus.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(versions) == 0 && !internal.Chaos {
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
+		}
+
+		var availableDowngrades []string
+		if deployedVersion == "unknown" {
+			availableDowngrades = versions
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
+		}
+
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsGreaterThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not a downgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableDowngrades = append(availableDowngrades, specificVersion)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
+			for _, version := range versions {
+				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				parsedVersion, err := tagcmp.Parse(version)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if parsedVersion.IsLessThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
+					availableDowngrades = append(availableDowngrades, version)
+				}
+			}
+
+			if len(availableDowngrades) == 0 && !internal.Force {
+				logrus.Info("no available downgrades, you're on oldest ✌️")
+				return nil
+			}
+		}
+
+		var chosenDowngrade string
+		if len(availableDowngrades) > 0 && !internal.Chaos {
+			if internal.Force || internal.NoInput || specificVersion != "" {
+				chosenDowngrade = availableDowngrades[len(availableDowngrades)-1]
+				logrus.Debugf("choosing %s as version to downgrade to (--force/--no-input)", chosenDowngrade)
+			} else {
+				prompt := &survey.Select{
+					Message: fmt.Sprintf("Please select a downgrade (current version: %s):", deployedVersion),
+					Options: internal.ReverseStringList(availableDowngrades),
+				}
+				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
+					return err
+				}
+			}
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureVersion(app.Recipe, chosenDowngrade); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warn("chaos mode engaged")
+			var err error
+			chosenDowngrade, err = recipe.ChaosVersion(app.Recipe)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, chosenDowngrade)
+		config.SetUpdateLabel(compose, stackName, app.Env)
+
+		// NOTE(d1): no release notes implemeneted for rolling back
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade, ""); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, stackName, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/run.go

@@ -0,0 +1,100 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var user string
+var userFlag = &cli.StringFlag{
+	Name:        "user, u",
+	Value:       "",
+	Destination: &user,
+}
+
+var noTTY bool
+var noTTYFlag = &cli.BoolFlag{
+	Name:        "no-tty, t",
+	Destination: &noTTY,
+}
+
+var appRunCommand = cli.Command{
+	Name:    "run",
+	Aliases: []string{"r"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		noTTYFlag,
+		userFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	ArgsUsage:    "<domain> <service> <args>...",
+	Usage:        "Run a command in a service container",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if len(c.Args()) < 2 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no <service> provided?"))
+		}
+
+		if len(c.Args()) < 3 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no <args> provided?"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		stackAndServiceName := fmt.Sprintf("^%s_%s", app.StackName(), serviceName)
+		filters := filters.NewArgs()
+		filters.Add("name", stackAndServiceName)
+
+		targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		cmd := c.Args()[2:]
+		execCreateOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          cmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if user != "" {
+			execCreateOpts.User = user
+		}
+		if noTTY {
+			execCreateOpts.Tty = false
+		}
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/secret.go

@@ -0,0 +1,454 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/docker/docker/api/types"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var allSecrets bool
+var allSecretsFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Destination: &allSecrets,
+	Usage:       "Generate all secrets",
+}
+
+var rmAllSecrets bool
+var rmAllSecretsFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Destination: &rmAllSecrets,
+	Usage:       "Remove all secrets",
+}
+
+var appSecretGenerateCommand = cli.Command{
+	Name:      "generate",
+	Aliases:   []string{"g"},
+	Usage:     "Generate secrets",
+	ArgsUsage: "<domain> <secret> <version>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		allSecretsFlag,
+		internal.PassFlag,
+		internal.MachineReadableFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if len(c.Args()) == 1 && !allSecrets {
+			err := errors.New("missing arguments <secret>/<version> or '--all'")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		if c.Args().Get(1) != "" && allSecrets {
+			err := errors.New("cannot use '<secret> <version>' and '--all' together")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretsToCreate := make(map[string]string)
+		if allSecrets {
+			secretsToCreate = secretsConfig
+		} else {
+			secretName := c.Args().Get(1)
+			secretVersion := c.Args().Get(2)
+			matches := false
+			for name := range secretsConfig {
+				if secretName == name {
+					secretsToCreate[name] = secretVersion
+					matches = true
+				}
+			}
+
+			if !matches {
+				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if internal.Pass {
+			for name, data := range secretVals {
+				if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		if len(secretVals) == 0 {
+			logrus.Warn("no secrets generated")
+			os.Exit(1)
+		}
+
+		tableCol := []string{"name", "value"}
+		table := formatter.CreateTable(tableCol)
+		for name, val := range secretVals {
+			table.Append([]string{name, val})
+		}
+
+		if internal.MachineReadable {
+			table.JSONRender()
+		} else {
+			table.Render()
+		}
+		logrus.Warn("generated secrets are not shown again, please take note of them NOW")
+
+		return nil
+	},
+}
+
+var appSecretInsertCommand = cli.Command{
+	Name:    "insert",
+	Aliases: []string{"i"},
+	Usage:   "Insert secret",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.PassFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	ArgsUsage:    "<domain> <secret-name> <version> <data>",
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+This command inserts a secret into an app environment.
+
+This can be useful when you want to manually generate secrets for an app
+environment. Typically, you can let Abra generate them for you on app creation
+(see "abra app new --secrets" for more).
+
+Example:
+
+    abra app secret insert myapp db_pass v1 mySecretPassword
+
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if len(c.Args()) != 4 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		name := c.Args().Get(1)
+		version := c.Args().Get(2)
+		data := c.Args().Get(3)
+
+		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
+		if err := client.StoreSecret(cl, secretName, data, app.Server); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("%s successfully stored on server", secretName)
+
+		if internal.Pass {
+			if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+// secretRm removes a secret.
+func secretRm(cl *dockerClient.Client, app config.App, secretName, parsed string) error {
+	if err := cl.SecretRemove(context.Background(), secretName); err != nil {
+		return err
+	}
+
+	logrus.Infof("deleted %s successfully from server", secretName)
+
+	if internal.PassRemove {
+		if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+			return err
+		}
+
+		logrus.Infof("deleted %s successfully from local pass store", secretName)
+	}
+
+	return nil
+}
+
+var appSecretRmCommand = cli.Command{
+	Name:    "remove",
+	Aliases: []string{"rm"},
+	Usage:   "Remove a secret",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		rmAllSecretsFlag,
+		internal.PassRemoveFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	ArgsUsage:    "<domain> [<secret-name>]",
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+This command removes app secrets.
+
+Example:
+
+    abra app secret remove myapp db_pass
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if c.Args().Get(1) != "" && rmAllSecrets {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret-name>' and '--all' together"))
+		}
+
+		if c.Args().Get(1) == "" && !rmAllSecrets {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no secret(s) specified?"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters, err := app.Filters(false, false)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		remoteSecretNames := make(map[string]bool)
+		for _, cont := range secretList {
+			remoteSecretNames[cont.Spec.Annotations.Name] = true
+		}
+
+		match := false
+		secretToRm := c.Args().Get(1)
+		for secretName, secretValue := range secretsConfig {
+			val, err := secret.ParseSecretValue(secretValue)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+			if _, ok := remoteSecretNames[secretRemoteName]; ok {
+				if secretToRm != "" {
+					if secretName == secretToRm {
+						if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
+							logrus.Fatal(err)
+						}
+
+						return nil
+					}
+				} else {
+					match = true
+
+					if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
+						logrus.Fatal(err)
+					}
+				}
+			}
+		}
+
+		if !match && secretToRm != "" {
+			logrus.Fatalf("%s doesn't exist on server?", secretToRm)
+		}
+
+		if !match {
+			logrus.Fatal("no secrets to remove?")
+		}
+
+		return nil
+	},
+}
+
+var appSecretLsCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "List all secrets",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
+		table := formatter.CreateTable(tableCol)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters, err := app.Filters(false, false)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		remoteSecretNames := make(map[string]bool)
+		for _, cont := range secretList {
+			remoteSecretNames[cont.Spec.Annotations.Name] = true
+		}
+
+		for secretName, secretValue := range secretsConfig {
+			createdRemote := false
+			val, err := secret.ParseSecretValue(secretValue)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+			if _, ok := remoteSecretNames[secretRemoteName]; ok {
+				createdRemote = true
+			}
+			tableRow := []string{secretName, val.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
+			table.Append(tableRow)
+		}
+
+		if table.NumLines() > 0 {
+			table.Render()
+		} else {
+			logrus.Warnf("no secrets stored for %s", app.Name)
+		}
+
+		return nil
+	},
+}
+
+var appSecretCommand = cli.Command{
+	Name:      "secret",
+	Aliases:   []string{"s"},
+	Usage:     "Manage app secrets",
+	ArgsUsage: "<domain>",
+	Subcommands: []cli.Command{
+		appSecretGenerateCommand,
+		appSecretInsertCommand,
+		appSecretRmCommand,
+		appSecretLsCommand,
+	},
+}

cli/app/services.go

@@ -0,0 +1,80 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appServicesCommand = cli.Command{
+	Name:      "services",
+	Aliases:   []string{"sr"},
+	Usage:     "Display all services of an app",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		filters, err := app.Filters(true, true)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableCol := []string{"service name", "image"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, container := range containers {
+			var containerNames []string
+			for _, containerName := range container.Names {
+				trimmed := strings.TrimPrefix(containerName, "/")
+				containerNames = append(containerNames, trimmed)
+			}
+
+			serviceShortName := service.ContainerToServiceName(container.Names, app.StackName())
+			serviceLongName := fmt.Sprintf("%s_%s", app.StackName(), serviceShortName)
+
+			tableRow := []string{
+				serviceLongName,
+				formatter.RemoveSha(container.Image),
+			}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+
+		return nil
+	},
+}

cli/app/undeploy.go

@@ -0,0 +1,138 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var prune bool
+
+var pruneFlag = &cli.BoolFlag{
+	Name:        "prune, p",
+	Destination: &prune,
+	Usage:       "Prunes unused containers, networks, and dangling images for an app",
+}
+
+// pruneApp runs the equivalent of a "docker system prune" but only filtering
+// against resources connected with the app deployment. It is not a system wide
+// prune. Volumes are not pruned to avoid unwated data loss.
+func pruneApp(c *cli.Context, cl *dockerClient.Client, app config.App) error {
+	stackName := app.StackName()
+	ctx := context.Background()
+
+	for {
+		logrus.Debugf("polling for %s stack, waiting to be undeployed...", stackName)
+
+		services, err := stack.GetStackServices(ctx, cl, stackName)
+		if err != nil {
+			return err
+		}
+
+		if len(services) == 0 {
+			logrus.Debugf("%s undeployed, moving on with pruning logic", stackName)
+			time.Sleep(time.Second) // give runtime more time to tear down related state
+			break
+		}
+
+		time.Sleep(time.Second)
+	}
+
+	pruneFilters := filters.NewArgs()
+	stackSearch := fmt.Sprintf("%s*", stackName)
+	pruneFilters.Add("label", stackSearch)
+	cr, err := cl.ContainersPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	cntSpaceReclaimed := formatter.ByteCountSI(cr.SpaceReclaimed)
+	logrus.Infof("containers pruned: %d; space reclaimed: %s", len(cr.ContainersDeleted), cntSpaceReclaimed)
+
+	nr, err := cl.NetworksPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("networks pruned: %d", len(nr.NetworksDeleted))
+
+	ir, err := cl.ImagesPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
+	logrus.Infof("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed)
+
+	return nil
+}
+
+var appUndeployCommand = cli.Command{
+	Name:      "undeploy",
+	Aliases:   []string{"un"},
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		pruneFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Undeploy an app",
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+This does not destroy any of the application data.
+
+However, you should remain vigilant, as your swarm installation will consider
+any previously attached volumes as eligible for pruning once undeployed.
+
+Passing "-p/--prune" does not remove those volumes.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		if err := internal.DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
+			logrus.Fatal(err)
+		}
+
+		rmOpts := stack.Remove{Namespaces: []string{app.StackName()}}
+		if err := stack.RunRemove(context.Background(), cl, rmOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if prune {
+			if err := pruneApp(c, cl, app); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}

cli/app/upgrade.go

@@ -0,0 +1,281 @@
+package app
+
+import (
+	"context"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appUpgradeCommand = cli.Command{
+	Name:      "upgrade",
+	Aliases:   []string{"up"},
+	Usage:     "Upgrade an app",
+	ArgsUsage: "<domain> [<version>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.ForceFlag,
+		internal.ChaosFlag,
+		internal.NoDomainChecksFlag,
+		internal.DontWaitConvergeFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Upgrade an app. You can use it to choose and roll out a new upgrade to an
+existing app.
+
+This command specifically supports incrementing the version of running apps, as
+opposed to "abra app deploy <domain>" which will not change the version of a
+deployed app.
+
+You may pass "--force/-f" to upgrade to the same version again. This can be
+useful if the container runtime has gotten into a weird state.
+
+This action could be destructive, please ensure you have a copy of your app
+data beforehand.
+
+Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
+including unstaged changes and can be useful for live hacking and testing new
+recipes.
+`,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := recipePkg.GetRecipeCatalogueVersions(app.Recipe, catl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(versions) == 0 && !internal.Chaos {
+			logrus.Warn("no published versions in catalogue, trying local recipe repository")
+			recipeVersions, err := recipePkg.GetRecipeVersions(app.Recipe, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, recipeVersion := range recipeVersions {
+				for version := range recipeVersion {
+					versions = append(versions, version)
+				}
+			}
+		}
+
+		var availableUpgrades []string
+		if deployedVersion == "unknown" {
+			availableUpgrades = versions
+			logrus.Warnf("failed to determine deployed version of %s", app.Name)
+		}
+
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" {
+			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if parsedSpecificVersion.IsLessThan(parsedDeployedVersion) || parsedSpecificVersion.Equals(parsedDeployedVersion) {
+				logrus.Fatalf("%s is not an upgrade for %s?", deployedVersion, specificVersion)
+			}
+			availableUpgrades = append(availableUpgrades, specificVersion)
+		}
+
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos && specificVersion == "" {
+			for _, version := range versions {
+				parsedVersion, err := tagcmp.Parse(version)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if parsedVersion.IsGreaterThan(parsedDeployedVersion) && !(parsedVersion.Equals(parsedDeployedVersion)) {
+					availableUpgrades = append(availableUpgrades, version)
+				}
+			}
+
+			if len(availableUpgrades) == 0 && !internal.Force {
+				logrus.Infof("no available upgrades, you're on latest (%s) ✌️", deployedVersion)
+				return nil
+			}
+		}
+
+		var chosenUpgrade string
+		if len(availableUpgrades) > 0 && !internal.Chaos {
+			if internal.Force || internal.NoInput || specificVersion != "" {
+				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+				logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
+			} else {
+				prompt := &survey.Select{
+					Message: fmt.Sprintf("Please select an upgrade (current version: %s):", deployedVersion),
+					Options: internal.ReverseStringList(availableUpgrades),
+				}
+				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
+					return err
+				}
+			}
+		}
+
+		if internal.Force && chosenUpgrade == "" {
+			logrus.Warnf("%s is already upgraded to latest but continuing (--force/--chaos)", app.Name)
+			chosenUpgrade = deployedVersion
+		}
+
+		// if release notes written after git tag published, read them before we
+		// check out the tag and then they'll appear to be missing. this covers
+		// when we obviously will forget to write release notes before publishing
+		var releaseNotes string
+		for _, version := range versions {
+			parsedVersion, err := tagcmp.Parse(version)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !(parsedVersion.Equals(parsedDeployedVersion)) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
+				note, err := internal.GetReleaseNotes(app.Recipe, version)
+				if err != nil {
+					return err
+				}
+				if note != "" {
+					releaseNotes += fmt.Sprintf("%s\n", note)
+				}
+			}
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warn("chaos mode engaged")
+			var err error
+			chosenUpgrade, err = recipePkg.ChaosVersion(app.Recipe)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		config.ExposeAllEnv(stackName, compose, app.Env)
+		config.SetRecipeLabel(compose, stackName, app.Recipe)
+		config.SetChaosLabel(compose, stackName, internal.Chaos)
+		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
+		config.SetUpdateLabel(compose, stackName, app.Env)
+
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
+			logrus.Fatal(err)
+		}
+
+		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
+
+		if err := stack.RunDeploy(cl, deployOpts, compose, stackName, internal.DontWaitConverge); err != nil {
+			logrus.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_UPGRADE_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+			}
+		}
+
+		return nil
+	},
+}

cli/app/version.go

@@ -0,0 +1,117 @@
+package app
+
+import (
+	"context"
+	"sort"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/distribution/reference"
+	"github.com/olekukonko/tablewriter"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
+// getImagePath returns the image name
+func getImagePath(image string) (string, error) {
+	img, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return "", err
+	}
+
+	path := reference.Path(img)
+
+	path = formatter.StripTagMeta(path)
+
+	logrus.Debugf("parsed %s from %s", path, image)
+
+	return path, nil
+}
+
+var appVersionCommand = cli.Command{
+	Name:      "version",
+	Aliases:   []string{"v"},
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.OfflineFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Show version info of a deployed app",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether %s is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("%s is not deployed?", app.Name)
+		}
+
+		if deployedVersion == "unknown" {
+			logrus.Fatalf("failed to determine version of deployed %s", app.Name)
+		}
+
+		recipeMeta, err := recipe.GetRecipeMeta(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versionsMeta := make(map[string]recipe.ServiceMeta)
+		for _, recipeVersion := range recipeMeta.Versions {
+			if currentVersion, exists := recipeVersion[deployedVersion]; exists {
+				versionsMeta = currentVersion
+			}
+		}
+
+		if len(versionsMeta) == 0 {
+			logrus.Fatalf("could not retrieve deployed version (%s) from recipe catalogue?", deployedVersion)
+		}
+
+		tableCol := []string{"version", "service", "image", "tag"}
+		table := formatter.CreateTable(tableCol)
+
+		var versions [][]string
+		for serviceName, versionMeta := range versionsMeta {
+			versions = append(versions, []string{deployedVersion, serviceName, versionMeta.Image, versionMeta.Tag})
+		}
+
+		sort.Slice(versions, sortServiceByName(versions))
+
+		for _, version := range versions {
+			table.Append(version)
+		}
+
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+		table.SetAlignment(tablewriter.ALIGN_LEFT)
+		table.Render()
+
+		return nil
+	},
+}

cli/app/volume.go

@@ -0,0 +1,157 @@
+package app
+
+import (
+	"context"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var appVolumeListCommand = cli.Command{
+	Name:      "list",
+	Aliases:   []string{"ls"},
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "List volumes associated with an app",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters, err := app.Filters(false, true)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		table := formatter.CreateTable([]string{"name", "created", "mounted"})
+		var volTable [][]string
+		for _, volume := range volumeList {
+			volRow := []string{volume.Name, volume.CreatedAt, volume.Mountpoint}
+			volTable = append(volTable, volRow)
+		}
+
+		table.AppendBulk(volTable)
+
+		if table.NumLines() > 0 {
+			table.Render()
+		} else {
+			logrus.Warnf("no volumes created for %s", app.Name)
+		}
+
+		return nil
+	},
+}
+
+var appVolumeRemoveCommand = cli.Command{
+	Name:  "remove",
+	Usage: "Remove volume(s) associated with an app",
+	Description: `
+This command supports removing volumes associated with an app. The app in
+question must be undeployed before you try to remove volumes. See "abra app
+undeploy <domain>" for more.
+
+The command is interactive and will show a multiple select input which allows
+you to make a seclection. Use the "?" key to see more help on navigating this
+interface.
+
+Passing "--force/-f" will select all volumes for removal. Be careful.
+`,
+	ArgsUsage: "<domain>",
+	Aliases:   []string{"rm"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.ForceFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if isDeployed {
+			logrus.Fatalf("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name)
+		}
+
+		filters, err := app.Filters(false, true)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		volumeNames := client.GetVolumeNames(volumeList)
+
+		var volumesToRemove []string
+		if !internal.Force && !internal.NoInput {
+			volumesPrompt := &survey.MultiSelect{
+				Message: "which volumes do you want to remove?",
+				Help:    "'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled",
+				VimMode: true,
+				Options: volumeNames,
+				Default: volumeNames,
+			}
+			if err := survey.AskOne(volumesPrompt, &volumesToRemove); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.Force || internal.NoInput {
+			volumesToRemove = volumeNames
+		}
+
+		if len(volumesToRemove) > 0 {
+			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			logrus.Info("volumes removed successfully")
+		} else {
+			logrus.Info("no volumes removed")
+		}
+
+		return nil
+	},
+}
+
+var appVolumeCommand = cli.Command{
+	Name:      "volume",
+	Aliases:   []string{"vl"},
+	Usage:     "Manage app volumes",
+	ArgsUsage: "<domain>",
+	Subcommands: []cli.Command{
+		appVolumeListCommand,
+		appVolumeRemoveCommand,
+	},
+}

cli/catalogue/catalogue.go

@@ -0,0 +1,228 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var catalogueGenerateCommand = cli.Command{
+	Name:    "generate",
+	Aliases: []string{"g"},
+	Usage:   "Generate the recipe catalogue",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.PublishFlag,
+		internal.DryFlag,
+		internal.SkipUpdatesFlag,
+		internal.ChaosFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Generate a new copy of the recipe catalogue which can be found on:
+
+    https://recipes.coopcloud.tech (website that humans read)
+    https://recipes.coopcloud.tech/recipes.json (JSON that Abra reads)
+
+It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
+listing, parses README.md and git tags to produce recipe metadata which is
+loaded into the catalogue JSON file.
+
+It is possible to generate new metadata for a single recipe by passing
+<recipe>. The existing local catalogue will be updated, not overwritten.
+
+It is quite easy to get rate limited by Docker Hub when running this command.
+If you have a Hub account you can have Abra log you in to avoid this. Pass
+"--user" and "--pass".
+
+Push your new release to git.coopcloud.tech with "-p/--publish". This requires
+that you have permission to git push to these repositories and have your SSH
+keys configured on your account.
+`,
+	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		if !internal.Chaos {
+			if err := catalogue.EnsureIsClean(); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		repos, err := recipe.ReadReposMetadata()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var barLength int
+		var logMsg string
+		if recipeName != "" {
+			barLength = 1
+			logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
+		} else {
+			barLength = len(repos)
+			logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
+		}
+
+		if !internal.SkipUpdates {
+			logrus.Warn(logMsg)
+			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		catl := make(recipe.RecipeCatalogue)
+		catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
+		for _, recipeMeta := range repos {
+			if recipeName != "" && recipeName != recipeMeta.Name {
+				catlBar.Add(1)
+				continue
+			}
+
+			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
+				catlBar.Add(1)
+				continue
+			}
+
+			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
+			features, category, err := recipe.GetRecipeFeaturesAndCategory(recipeMeta.Name)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
+			catl[recipeMeta.Name] = recipe.RecipeMeta{
+				Name:          recipeMeta.Name,
+				Repository:    recipeMeta.CloneURL,
+				SSHURL:        recipeMeta.SSHURL,
+				Icon:          recipeMeta.AvatarURL,
+				DefaultBranch: recipeMeta.DefaultBranch,
+				Description:   recipeMeta.Description,
+				Website:       recipeMeta.Website,
+				Versions:      versions,
+				Category:      category,
+				Features:      features,
+			}
+
+			catlBar.Add(1)
+		}
+
+		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if recipeName == "" {
+			if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			catlFS, err := recipe.ReadRecipeCatalogue(internal.Offline)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			catlFS[recipeName] = catl[recipeName]
+
+			updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		logrus.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)
+
+		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
+		if internal.Publish {
+
+			isClean, err := gitPkg.IsClean(cataloguePath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if isClean {
+				if !internal.Dry {
+					logrus.Fatalf("no changes discovered in %s, nothing to publish?", cataloguePath)
+				}
+			}
+
+			msg := "chore: publish new catalogue release changes"
+			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
+				logrus.Fatal(err)
+			}
+
+			repo, err := git.PlainOpen(cataloguePath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			sshURL := fmt.Sprintf(config.SSH_URL_TEMPLATE, config.CATALOGUE_JSON_REPO_NAME)
+			if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		repo, err := git.PlainOpen(cataloguePath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		head, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Dry && internal.Publish {
+			url := fmt.Sprintf("%s/%s/commit/%s", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME, head.Hash())
+			logrus.Infof("new changes published: %s", url)
+		}
+
+		if internal.Dry {
+			logrus.Info("dry run: no changes published")
+		}
+
+		return nil
+	},
+}
+
+// CatalogueCommand defines the `abra catalogue` command and sub-commands.
+var CatalogueCommand = cli.Command{
+	Name:        "catalogue",
+	Usage:       "Manage the recipe catalogue",
+	Aliases:     []string{"c"},
+	ArgsUsage:   "<recipe>",
+	Description: "This command helps recipe packagers interact with the recipe catalogue",
+	Subcommands: []cli.Command{
+		catalogueGenerateCommand,
+	},
+}

cli/cli.go

@@ -0,0 +1,210 @@
+// Package cli provides the interface for the command-line.
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+
+	"coopcloud.tech/abra/cli/app"
+	"coopcloud.tech/abra/cli/catalogue"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/recipe"
+	"coopcloud.tech/abra/cli/server"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/web"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// AutoCompleteCommand helps people set up auto-complete in their shells
+var AutoCompleteCommand = cli.Command{
+	Name:    "autocomplete",
+	Aliases: []string{"ac"},
+	Usage:   "Configure shell autocompletion (recommended)",
+	Description: `
+Set up auto-completion in your shell by downloading the relevant files and
+laying out what additional information must be loaded. Supported shells are as
+follows: bash, fish, fizsh & zsh.
+
+Example:
+
+    abra autocomplete bash
+`,
+	ArgsUsage: "<shell>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Action: func(c *cli.Context) error {
+		shellType := c.Args().First()
+
+		if shellType == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
+		}
+
+		supportedShells := map[string]bool{
+			"bash":  true,
+			"zsh":   true,
+			"fizsh": true,
+			"fish":  true,
+		}
+
+		if _, ok := supportedShells[shellType]; !ok {
+			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
+		}
+
+		if shellType == "fizsh" {
+			shellType = "zsh" // handled the same on the autocompletion side
+		}
+
+		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
+		if err := os.Mkdir(autocompletionDir, 0764); err != nil {
+			if !os.IsExist(err) {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("%s already created", autocompletionDir)
+		}
+
+		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
+		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
+			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
+			logrus.Infof("fetching %s", url)
+			if err := web.GetFile(autocompletionFile, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		switch shellType {
+		case "bash":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install auto-completion
+sudo mkdir /etc/bash_completion.d/
+sudo cp %s /etc/bash_completion.d/abra
+echo "source /etc/bash_completion.d/abra" >> ~/.bashrc
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
+`, autocompletionFile))
+		case "zsh":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install auto-completion
+sudo mkdir /etc/zsh/completion.d/
+sudo cp %s /etc/zsh/completion.d/abra
+echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
+`, autocompletionFile))
+		case "fish":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install auto-completion
+sudo mkdir -p /etc/fish/completions
+sudo cp %s /etc/fish/completions/abra
+echo "source /etc/fish/completions/abra" >> ~/.config/fish/config.fish
+# To test, run the following: "abra app <hit tab key>" - you should see command completion!
+`, autocompletionFile))
+		}
+
+		return nil
+	},
+}
+
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = cli.Command{
+	Name:    "upgrade",
+	Aliases: []string{"u"},
+	Usage:   "Upgrade Abra itself",
+	Description: `
+Upgrade Abra in-place with the latest stable or release candidate.
+
+Pass "-r/--rc" to install the latest release candidate. Please bear in mind
+that it may contain catastrophic bugs. Thank you very much for the testing
+efforts!
+`,
+	Flags: []cli.Flag{internal.RCFlag},
+	Action: func(c *cli.Context) error {
+		mainURL := "https://install.abra.coopcloud.tech"
+		cmd := exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash", mainURL))
+
+		if internal.RC {
+			releaseCandidateURL := "https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer"
+			cmd = exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash -s -- --rc", releaseCandidateURL))
+		}
+
+		logrus.Debugf("attempting to run %s", cmd)
+
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+func newAbraApp(version, commit string) *cli.App {
+	app := &cli.App{
+		Name: "abra",
+		Usage: `The Co-op Cloud command-line utility belt 🎩🐇
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []cli.Command{
+			app.AppCommand,
+			server.ServerCommand,
+			recipe.RecipeCommand,
+			catalogue.CatalogueCommand,
+			UpgradeCommand,
+			AutoCompleteCommand,
+		},
+		BashComplete: autocomplete.SubcommandComplete,
+	}
+
+	app.EnableBashCompletion = true
+
+	app.Before = func(c *cli.Context) error {
+		paths := []string{
+			config.ABRA_DIR,
+			config.SERVERS_DIR,
+			config.RECIPES_DIR,
+			config.VENDOR_DIR,
+			config.BACKUP_DIR,
+		}
+
+		for _, path := range paths {
+			if err := os.Mkdir(path, 0764); err != nil {
+				if !os.IsExist(err) {
+					logrus.Fatal(err)
+				}
+				continue
+			}
+		}
+
+		if _, err := os.Stat(config.CATALOGUE_DIR); os.IsNotExist(err) {
+			url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+			logrus.Warnf("local recipe catalogue is missing, retrieving now")
+			if err := git.Clone(config.CATALOGUE_DIR, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		logrus.Debugf("abra version %s, commit %s", version, commit)
+
+		return nil
+	}
+
+	return app
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := newAbraApp(version, commit)
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cli/internal/backup.go

@@ -0,0 +1,35 @@
+package internal
+
+import (
+	"strings"
+)
+
+// SafeSplit splits up a string into a list of commands safely.
+func SafeSplit(s string) []string {
+	split := strings.Split(s, " ")
+
+	var result []string
+	var inquote string
+	var block string
+	for _, i := range split {
+		if inquote == "" {
+			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
+				inquote = string(i[0])
+				block = strings.TrimPrefix(i, inquote) + " "
+			} else {
+				result = append(result, i)
+			}
+		} else {
+			if !strings.HasSuffix(i, inquote) {
+				block += i + " "
+			} else {
+				block += strings.TrimSuffix(i, inquote)
+				inquote = ""
+				result = append(result, block)
+				block = ""
+			}
+		}
+	}
+
+	return result
+}

cli/internal/cli.go

@@ -0,0 +1,251 @@
+package internal
+
+import (
+	"os"
+
+	logrusStack "github.com/Gurpartap/logrus-stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// Secrets stores the variable from SecretsFlag
+var Secrets bool
+
+// SecretsFlag turns on/off automatically generating secrets
+var SecretsFlag = &cli.BoolFlag{
+	Name:        "secrets, S",
+	Usage:       "Automatically generate secrets",
+	Destination: &Secrets,
+}
+
+// Pass stores the variable from PassFlag
+var Pass bool
+
+// PassFlag turns on/off storing generated secrets in pass
+var PassFlag = &cli.BoolFlag{
+	Name:        "pass, p",
+	Usage:       "Store the generated secrets in a local pass store",
+	Destination: &Pass,
+}
+
+// PassRemove stores the variable for PassRemoveFlag
+var PassRemove bool
+
+// PassRemoveFlag turns on/off removing generated secrets from pass
+var PassRemoveFlag = &cli.BoolFlag{
+	Name:        "pass, p",
+	Usage:       "Remove generated secrets from a local pass store",
+	Destination: &PassRemove,
+}
+
+// Force force functionality without asking.
+var Force bool
+
+// ForceFlag turns on/off force functionality.
+var ForceFlag = &cli.BoolFlag{
+	Name:        "force, f",
+	Usage:       "Perform action without further prompt. Use with care!",
+	Destination: &Force,
+}
+
+// Chaos engages chaos mode.
+var Chaos bool
+
+// ChaosFlag turns on/off chaos functionality.
+var ChaosFlag = &cli.BoolFlag{
+	Name:        "chaos, C",
+	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Destination: &Chaos,
+}
+
+// Disable tty to run commands from script
+var Tty bool
+
+// TtyFlag turns on/off tty mode.
+var TtyFlag = &cli.BoolFlag{
+	Name:        "tty, T",
+	Usage:       "Disables TTY mode to run this command from a script.",
+	Destination: &Tty,
+}
+
+var NoInput bool
+var NoInputFlag = &cli.BoolFlag{
+	Name:        "no-input, n",
+	Usage:       "Toggle non-interactive mode",
+	Destination: &NoInput,
+}
+
+// Debug stores the variable from DebugFlag.
+var Debug bool
+
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
+var DebugFlag = &cli.BoolFlag{
+	Name:        "debug, d",
+	Destination: &Debug,
+	Usage:       "Show DEBUG messages",
+}
+
+// Offline stores the variable from OfflineFlag.
+var Offline bool
+
+// DebugFlag turns on/off offline mode.
+var OfflineFlag = &cli.BoolFlag{
+	Name:        "offline, o",
+	Destination: &Offline,
+	Usage:       "Prefer offline & filesystem access when possible",
+}
+
+// MachineReadable stores the variable from MachineReadableFlag
+var MachineReadable bool
+
+// MachineReadableFlag turns on/off machine readable output where supported
+var MachineReadableFlag = &cli.BoolFlag{
+	Name:        "machine, m",
+	Destination: &MachineReadable,
+	Usage:       "Output in a machine-readable format (where supported)",
+}
+
+// RC signifies the latest release candidate
+var RC bool
+
+// RCFlag chooses the latest release candidate for install
+var RCFlag = &cli.BoolFlag{
+	Name:        "rc, r",
+	Destination: &RC,
+	Usage:       "Install the latest release candidate",
+}
+
+var Major bool
+var MajorFlag = &cli.BoolFlag{
+	Name:        "major, x",
+	Usage:       "Increase the major part of the version",
+	Destination: &Major,
+}
+
+var Minor bool
+var MinorFlag = &cli.BoolFlag{
+	Name:        "minor, y",
+	Usage:       "Increase the minor part of the version",
+	Destination: &Minor,
+}
+
+var Patch bool
+var PatchFlag = &cli.BoolFlag{
+	Name:        "patch, z",
+	Usage:       "Increase the patch part of the version",
+	Destination: &Patch,
+}
+
+var Dry bool
+var DryFlag = &cli.BoolFlag{
+	Name:        "dry-run, r",
+	Usage:       "Only reports changes that would be made",
+	Destination: &Dry,
+}
+
+var Publish bool
+var PublishFlag = &cli.BoolFlag{
+	Name:        "publish, p",
+	Usage:       "Publish changes to git.coopcloud.tech",
+	Destination: &Publish,
+}
+
+var Domain string
+var DomainFlag = &cli.StringFlag{
+	Name:        "domain, D",
+	Value:       "",
+	Usage:       "Choose a domain name",
+	Destination: &Domain,
+}
+
+var NewAppServer string
+var NewAppServerFlag = &cli.StringFlag{
+	Name:        "server, s",
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &NewAppServer,
+}
+
+var NoDomainChecks bool
+var NoDomainChecksFlag = &cli.BoolFlag{
+	Name:        "no-domain-checks, D",
+	Usage:       "Disable app domain sanity checks",
+	Destination: &NoDomainChecks,
+}
+
+var StdErrOnly bool
+var StdErrOnlyFlag = &cli.BoolFlag{
+	Name:        "stderr, s",
+	Usage:       "Only tail stderr",
+	Destination: &StdErrOnly,
+}
+
+var SinceLogs string
+var SinceLogsFlag = &cli.StringFlag{
+	Name:        "since, S",
+	Value:       "",
+	Usage:       "tail logs since YYYY-MM-DDTHH:MM:SSZ",
+	Destination: &SinceLogs,
+}
+
+var DontWaitConverge bool
+var DontWaitConvergeFlag = &cli.BoolFlag{
+	Name:        "no-converge-checks, c",
+	Usage:       "Don't wait for converge logic checks",
+	Destination: &DontWaitConverge,
+}
+
+var Watch bool
+var WatchFlag = &cli.BoolFlag{
+	Name:        "watch, w",
+	Usage:       "Watch status by polling repeatedly",
+	Destination: &Watch,
+}
+
+var OnlyErrors bool
+var OnlyErrorFlag = &cli.BoolFlag{
+	Name:        "errors, e",
+	Usage:       "Only show errors",
+	Destination: &OnlyErrors,
+}
+
+var SkipUpdates bool
+var SkipUpdatesFlag = &cli.BoolFlag{
+	Name:        "skip-updates, s",
+	Usage:       "Skip updating recipe repositories",
+	Destination: &SkipUpdates,
+}
+
+var AllTags bool
+var AllTagsFlag = &cli.BoolFlag{
+	Name:        "all-tags, a",
+	Usage:       "List all tags, not just upgrades",
+	Destination: &AllTags,
+}
+
+var LocalCmd bool
+var LocalCmdFlag = &cli.BoolFlag{
+	Name:        "local, l",
+	Usage:       "Run command locally",
+	Destination: &LocalCmd,
+}
+
+var RemoteUser string
+var RemoteUserFlag = &cli.StringFlag{
+	Name:        "user, u",
+	Value:       "",
+	Usage:       "User to run command within a service context",
+	Destination: &RemoteUser,
+}
+
+// SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
+func SubCommandBefore(c *cli.Context) error {
+	if Debug {
+		logrus.SetLevel(logrus.DebugLevel)
+		logrus.SetFormatter(&logrus.TextFormatter{})
+		logrus.SetOutput(os.Stderr)
+		logrus.AddHook(logrusStack.StandardHook())
+	}
+
+	return nil
+}

cli/internal/command.go

@@ -0,0 +1,138 @@
+package internal
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+// RunCmdRemote executes an abra.sh command in the target service
+func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
+
+	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(abraSh, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
+		return err
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	shell := "/bin/bash"
+	findShell := []string{"test", "-e", shell}
+	execCreateOpts := types.ExecConfig{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          findShell,
+		Detach:       false,
+		Tty:          false,
+	}
+
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
+		shell = "/bin/sh"
+	}
+
+	var cmd []string
+	if cmdArgs != "" {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.Name, app.StackName(), cmdName, cmdArgs)}
+	} else {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.Name, app.StackName(), cmdName)}
+	}
+
+	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
+
+	if RemoteUser != "" {
+		logrus.Debugf("running command with user %s", RemoteUser)
+		execCreateOpts.User = RemoteUser
+	}
+
+	execCreateOpts.Cmd = cmd
+	execCreateOpts.Tty = true
+	if Tty {
+		execCreateOpts.Tty = false
+	}
+
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func EnsureCommand(abraSh, recipeName, execCmd string) error {
+	bytes, err := ioutil.ReadFile(abraSh)
+	if err != nil {
+		return err
+	}
+
+	if !strings.Contains(string(bytes), execCmd) {
+		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
+	}
+
+	return nil
+}
+
+// RunCmd runs a shell command and streams stdout/stderr in real-time.
+func RunCmd(cmd *exec.Cmd) error {
+	r, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	cmd.Stderr = cmd.Stdout
+	done := make(chan struct{})
+	scanner := bufio.NewScanner(r)
+
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			fmt.Println(line)
+		}
+		done <- struct{}{}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	<-done
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/internal/deploy.go

@@ -0,0 +1,173 @@
+package internal
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// NewVersionOverview shows an upgrade or downgrade overview
+func NewVersionOverview(app config.App, currentVersion, newVersion, releaseNotes string) error {
+	tableCol := []string{"server", "recipe", "config", "domain", "current version", "to be deployed"}
+	table := formatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, currentVersion, newVersion})
+	table.Render()
+
+	if releaseNotes != "" && newVersion != "" {
+		fmt.Println()
+		fmt.Print(releaseNotes)
+	} else {
+		logrus.Warnf("no release notes available for %s", newVersion)
+	}
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with deployment?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}
+
+// GetReleaseNotes prints release notes for a recipe version
+func GetReleaseNotes(recipeName, version string) (string, error) {
+	if version == "" {
+		return "", nil
+	}
+
+	fpath := path.Join(config.RECIPES_DIR, recipeName, "release", version)
+
+	if _, err := os.Stat(fpath); !os.IsNotExist(err) {
+		releaseNotes, err := ioutil.ReadFile(fpath)
+		if err != nil {
+			return "", err
+		}
+		withTitle := fmt.Sprintf("%s release notes:\n%s", version, string(releaseNotes))
+		return withTitle, nil
+	}
+
+	return "", nil
+}
+
+// PostCmds parses a string of commands and executes them inside of the respective services
+// the commands string must have the following format:
+// "<service> <command> <arguments>|<service> <command> <arguments>|... "
+func PostCmds(cl *dockerClient.Client, app config.App, commands string) error {
+	abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
+	if _, err := os.Stat(abraSh); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf(fmt.Sprintf("%s does not exist for %s?", abraSh, app.Name))
+		}
+		return err
+	}
+
+	for _, command := range strings.Split(commands, "|") {
+		commandParts := strings.Split(command, " ")
+		if len(commandParts) < 2 {
+			return fmt.Errorf(fmt.Sprintf("not enough arguments: %s", command))
+		}
+		targetServiceName := commandParts[0]
+		cmdName := commandParts[1]
+		parsedCmdArgs := ""
+		if len(commandParts) > 2 {
+			parsedCmdArgs = fmt.Sprintf("%s ", strings.Join(commandParts[2:], " "))
+		}
+		logrus.Infof("running post-command '%s %s' in container %s", cmdName, parsedCmdArgs, targetServiceName)
+
+		if err := EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
+			return err
+		}
+
+		serviceNames, err := config.GetAppServiceNames(app.Name)
+		if err != nil {
+			return err
+		}
+
+		matchingServiceName := false
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
+			}
+		}
+
+		if !matchingServiceName {
+			return fmt.Errorf(fmt.Sprintf("no service %s for %s?", targetServiceName, app.Name))
+		}
+
+		logrus.Debugf("running command %s %s within the context of %s_%s", cmdName, parsedCmdArgs, app.StackName(), targetServiceName)
+
+		Tty = true
+		if err := RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DeployOverview shows a deployment overview
+func DeployOverview(app config.App, version, message string) error {
+	tableCol := []string{"server", "recipe", "config", "domain", "version"}
+	table := formatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, app.Recipe, deployConfig, app.Domain, version})
+	table.Render()
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: message,
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}

cli/internal/errors.go

@@ -0,0 +1,18 @@
+package internal
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// ShowSubcommandHelpAndError exits the program on error, logs the error to the
+// terminal, and shows the help command.
+func ShowSubcommandHelpAndError(c *cli.Context, err interface{}) {
+	if err2 := cli.ShowSubcommandHelp(c); err2 != nil {
+		logrus.Error(err2)
+	}
+	logrus.Error(err)
+	os.Exit(1)
+}

cli/internal/list.go

@@ -0,0 +1,10 @@
+package internal
+
+// ReverseStringList reverses a list of a strings. Roll on Go generics.
+func ReverseStringList(strings []string) []string {
+	for i, j := 0, len(strings)-1; i < j; i, j = i+1, j-1 {
+		strings[i], strings[j] = strings[j], strings[i]
+	}
+
+	return strings
+}

cli/internal/recipe.go

@@ -0,0 +1,110 @@
+package internal
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+)
+
+// PromptBumpType prompts for version bump type
+func PromptBumpType(tagString, latestRelease string) error {
+	if (!Major && !Minor && !Patch) && tagString == "" {
+		fmt.Printf(`
+You need to make a decision about what kind of an update this new recipe
+version is. If someone else performs this upgrade, do they have to do some
+migration work or take care of some breaking changes? This can be signaled in
+the version you specify on the recipe deploy label and is called a semantic
+version.
+
+The latest published version is %s.
+
+Here is a semver cheat sheet (more on https://semver.org):
+
+    major: new features/bug fixes, backwards incompatible (e.g 1.0.0 -> 2.0.0).
+           the upgrade won't work without some preparation work and others need
+           to take care when performing it. "it could go wrong".
+
+    minor: new features/bug fixes, backwards compatible (e.g. 0.1.0 -> 0.2.0).
+           the upgrade should Just Work and there are no breaking changes in
+           the app and the recipe config. "it should go fine".
+
+    patch: bug fixes, backwards compatible (e.g. 0.0.1 -> 0.0.2). this upgrade
+           should also Just Work and is mostly to do with minor bug fixes
+           and/or security patches. "nothing to worry about".
+
+`, latestRelease)
+
+		var chosenBumpType string
+		prompt := &survey.Select{
+			Message: fmt.Sprintf("select recipe version increment type"),
+			Options: []string{"major", "minor", "patch"},
+		}
+
+		if err := survey.AskOne(prompt, &chosenBumpType); err != nil {
+			return err
+		}
+
+		SetBumpType(chosenBumpType)
+	}
+
+	return nil
+}
+
+// GetBumpType figures out which bump type is specified
+func GetBumpType() string {
+	var bumpType string
+
+	if Major {
+		bumpType = "major"
+	} else if Minor {
+		bumpType = "minor"
+	} else if Patch {
+		bumpType = "patch"
+	} else {
+		logrus.Fatal("no version bump type specififed?")
+	}
+
+	return bumpType
+}
+
+// SetBumpType figures out which bump type is specified
+func SetBumpType(bumpType string) {
+	if bumpType == "major" {
+		Major = true
+	} else if bumpType == "minor" {
+		Minor = true
+	} else if bumpType == "patch" {
+		Patch = true
+	} else {
+		logrus.Fatal("no version bump type specififed?")
+	}
+}
+
+// GetMainAppImage retrieves the main 'app' image name
+func GetMainAppImage(recipe recipe.Recipe) (string, error) {
+	var path string
+
+	for _, service := range recipe.Config.Services {
+		if service.Name == "app" {
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return "", err
+			}
+
+			path = reference.Path(img)
+			path = formatter.StripTagMeta(path)
+
+			return path, nil
+		}
+	}
+
+	if path == "" {
+		return path, fmt.Errorf("%s has no main 'app' service?", recipe.Name)
+	}
+
+	return path, nil
+}

cli/internal/validate.go

@@ -0,0 +1,172 @@
+package internal
+
+import (
+	"errors"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// ValidateRecipe ensures the recipe arg is valid.
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
+	recipeName := c.Args().First()
+
+	if recipeName == "" && !NoInput {
+		var recipes []string
+
+		catl, err := recipe.ReadRecipeCatalogue(Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		knownRecipes := make(map[string]bool)
+		for name := range catl {
+			knownRecipes[name] = true
+		}
+
+		localRecipes, err := recipe.GetRecipesLocal()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, recipeLocal := range localRecipes {
+			if _, ok := knownRecipes[recipeLocal]; !ok {
+				knownRecipes[recipeLocal] = true
+			}
+		}
+
+		for recipeName := range knownRecipes {
+			recipes = append(recipes, recipeName)
+		}
+
+		prompt := &survey.Select{
+			Message: "Select recipe",
+			Options: recipes,
+		}
+		if err := survey.AskOne(prompt, &recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if recipeName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
+	}
+
+	chosenRecipe, err := recipe.Get(recipeName, Offline)
+	if err != nil {
+		if c.Command.Name == "generate" {
+			if strings.Contains(err.Error(), "missing a compose") {
+				logrus.Fatal(err)
+			}
+			logrus.Warn(err)
+		} else {
+			if strings.Contains(err.Error(), "template_driver is not allowed") {
+				logrus.Warnf("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName)
+			}
+			logrus.Fatalf("unable to validate recipe: %s", err)
+		}
+	}
+
+	logrus.Debugf("validated %s as recipe argument", recipeName)
+
+	return chosenRecipe
+}
+
+// ValidateApp ensures the app name arg is valid.
+func ValidateApp(c *cli.Context) config.App {
+	appName := c.Args().First()
+
+	if appName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+	}
+
+	app, err := app.Get(appName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated %s as app argument", appName)
+
+	return app
+}
+
+// ValidateDomain ensures the domain name arg is valid.
+func ValidateDomain(c *cli.Context) string {
+	domainName := c.Args().First()
+
+	if domainName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify a domain name",
+			Default: "example.com",
+		}
+		if err := survey.AskOne(prompt, &domainName); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if domainName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no domain provided"))
+	}
+
+	logrus.Debugf("validated %s as domain argument", domainName)
+
+	return domainName
+}
+
+// ValidateSubCmdFlags ensures flag order conforms to correct order
+func ValidateSubCmdFlags(c *cli.Context) bool {
+	for argIdx, arg := range c.Args() {
+		if !strings.HasPrefix(arg, "--") {
+			for _, flag := range c.Args()[argIdx:] {
+				if strings.HasPrefix(flag, "--") {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+// ValidateServer ensures the server name arg is valid.
+func ValidateServer(c *cli.Context) string {
+	serverName := c.Args().First()
+
+	serverNames, err := config.ReadServerNames()
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if serverName == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Specify a server name",
+			Options: serverNames,
+		}
+		if err := survey.AskOne(prompt, &serverName); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	matched := false
+	for _, name := range serverNames {
+		if name == serverName {
+			matched = true
+		}
+	}
+
+	if serverName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
+	}
+
+	if !matched {
+		ShowSubcommandHelpAndError(c, errors.New("server doesn't exist?"))
+	}
+
+	logrus.Debugf("validated %s as server argument", serverName)
+
+	return serverName
+}

cli/recipe/fetch.go

@@ -0,0 +1,44 @@
+package recipe
+
+import (
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeFetchCommand = cli.Command{
+	Name:        "fetch",
+	Usage:       "Fetch recipe(s)",
+	Aliases:     []string{"f"},
+	ArgsUsage:   "[<recipe>]",
+	Description: "Retrieves all recipes if no <recipe> argument is passed",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		if err := recipe.EnsureExists(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.EnsureUpToDate(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipe.EnsureLatest(recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/recipe/lint.go

@@ -0,0 +1,122 @@
+package recipe
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/lint"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeLintCommand = cli.Command{
+	Name:      "lint",
+	Usage:     "Lint a recipe",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OnlyErrorFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		tableCol := []string{"ref", "rule", "severity", "satisfied", "skipped", "resolve"}
+		table := formatter.CreateTable(tableCol)
+
+		hasError := false
+		bar := formatter.CreateProgressbar(-1, "running recipe lint rules...")
+		for level := range lint.LintRules {
+			for _, rule := range lint.LintRules[level] {
+				if internal.OnlyErrors && rule.Level != "error" {
+					logrus.Debugf("skipping %s, does not have level \"error\"", rule.Ref)
+					continue
+				}
+
+				skipped := false
+				if rule.Skip(recipe) {
+					skipped = true
+				}
+
+				skippedOutput := "-"
+				if skipped {
+					skippedOutput = "yes"
+				}
+
+				satisfied := false
+				if !skipped {
+					ok, err := rule.Function(recipe)
+					if err != nil {
+						logrus.Warn(err)
+					}
+
+					if !ok && rule.Level == "error" {
+						hasError = true
+					}
+
+					if ok {
+						satisfied = true
+					}
+				}
+
+				satisfiedOutput := "yes"
+				if !satisfied {
+					satisfiedOutput = "NO"
+					if skipped {
+						satisfiedOutput = "-"
+					}
+				}
+
+				table.Append([]string{
+					rule.Ref,
+					rule.Description,
+					rule.Level,
+					satisfiedOutput,
+					skippedOutput,
+					rule.HowToResolve,
+				})
+
+				bar.Add(1)
+			}
+		}
+
+		if table.NumLines() > 0 {
+			fmt.Println()
+			table.Render()
+		}
+
+		if hasError {
+			logrus.Warn("watch out, some critical errors are present in your recipe config")
+		}
+
+		return nil
+	},
+}

cli/recipe/list.go

@@ -0,0 +1,83 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var pattern string
+var patternFlag = &cli.StringFlag{
+	Name:        "pattern, p",
+	Value:       "",
+	Usage:       "Simple string to filter recipes",
+	Destination: &pattern,
+}
+
+var recipeListCommand = cli.Command{
+	Name:    "list",
+	Usage:   "List available recipes",
+	Aliases: []string{"ls"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.MachineReadableFlag,
+		patternFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			logrus.Fatal(err.Error())
+		}
+
+		recipes := catl.Flatten()
+		sort.Sort(recipe.ByRecipeName(recipes))
+
+		tableCol := []string{"name", "category", "status", "healthcheck", "backups", "email", "tests", "SSO"}
+		table := formatter.CreateTable(tableCol)
+
+		len := 0
+		for _, recipe := range recipes {
+			tableRow := []string{
+				recipe.Name,
+				recipe.Category,
+				strconv.Itoa(recipe.Features.Status),
+				recipe.Features.Healthcheck,
+				recipe.Features.Backups,
+				recipe.Features.Email,
+				recipe.Features.Tests,
+				recipe.Features.SSO,
+			}
+
+			if pattern != "" {
+				if strings.Contains(recipe.Name, pattern) {
+					table.Append(tableRow)
+					len++
+				}
+			} else {
+				table.Append(tableRow)
+				len++
+			}
+		}
+
+		if table.NumLines() > 0 {
+			if internal.MachineReadable {
+				table.SetCaption(false, "")
+				table.JSONRender()
+			} else {
+				table.SetCaption(true, fmt.Sprintf("total recipes: %v", len))
+				table.Render()
+			}
+		}
+
+		return nil
+	},
+}

cli/recipe/new.go

@@ -0,0 +1,141 @@
+package recipe
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"text/template"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// recipeMetadata is the recipe metadata for the README.md
+type recipeMetadata struct {
+	Name        string
+	Description string
+	Category    string
+	Status      string
+	Image       string
+	Healthcheck string
+	Backups     string
+	Email       string
+	Tests       string
+	SSO         string
+}
+
+var recipeNewCommand = cli.Command{
+	Name:    "new",
+	Aliases: []string{"n"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.OfflineFlag,
+	},
+	Before:    internal.SubCommandBefore,
+	Usage:     "Create a new recipe",
+	ArgsUsage: "<recipe>",
+	Description: `
+Create a new recipe.
+
+Abra uses the built-in example repository which is available here:
+
+    https://git.coopcloud.tech/coop-cloud/example
+
+Files within the example repository make use of the Golang templating system
+which Abra uses to inject values into the generated recipe folder (e.g. name of
+recipe and domain in the sample environment config).
+`,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe name provided"))
+		}
+
+		directory := path.Join(config.RECIPES_DIR, recipeName)
+		if _, err := os.Stat(directory); !os.IsNotExist(err) {
+			logrus.Fatalf("%s recipe directory already exists?", directory)
+		}
+
+		url := fmt.Sprintf("%s/example.git", config.REPOS_BASE_URL)
+		if err := git.Clone(directory, url); err != nil {
+			logrus.Fatal(err)
+		}
+
+		gitRepo := path.Join(config.RECIPES_DIR, recipeName, ".git")
+		if err := os.RemoveAll(gitRepo); err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("removed example git repo in %s", gitRepo)
+
+		meta := newRecipeMeta(recipeName)
+
+		toParse := []string{
+			path.Join(config.RECIPES_DIR, recipeName, "README.md"),
+			path.Join(config.RECIPES_DIR, recipeName, ".env.sample"),
+		}
+		for _, path := range toParse {
+			tpl, err := template.ParseFiles(path)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			var templated bytes.Buffer
+			if err := tpl.Execute(&templated, meta); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := ioutil.WriteFile(path, templated.Bytes(), 0644); err != nil {
+				logrus.Fatal(err)
+			}
+
+		}
+
+		newGitRepo := path.Join(config.RECIPES_DIR, recipeName)
+		if err := git.Init(newGitRepo, true); err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Print(fmt.Sprintf(`
+Your new %s recipe has been created in %s.
+
+In order to share your recipe, you can upload it the git repository to:
+
+    https://git.coopcloud.tech/coop-cloud/%s
+
+If you're not sure how to do that, come chat with us:
+
+    https://docs.coopcloud.tech/intro/contact
+
+See "abra recipe -h" for additional recipe maintainer commands.
+
+Happy Hacking!
+
+`, recipeName, path.Join(config.RECIPES_DIR, recipeName), recipeName))
+
+		return nil
+	},
+}
+
+// newRecipeMeta creates a new recipeMetadata instance with defaults
+func newRecipeMeta(recipeName string) recipeMetadata {
+	return recipeMetadata{
+		Name:        recipeName,
+		Description: "> One line description of the recipe",
+		Category:    "Apps",
+		Status:      "0",
+		Image:       fmt.Sprintf("[`%s`](https://hub.docker.com/r/%s), 4, upstream", recipeName, recipeName),
+		Healthcheck: "No",
+		Backups:     "No",
+		Email:       "No",
+		Tests:       "No",
+		SSO:         "No",
+	}
+}

cli/recipe/recipe.go

@@ -0,0 +1,34 @@
+package recipe
+
+import (
+	"github.com/urfave/cli"
+)
+
+// RecipeCommand defines all recipe related sub-commands.
+var RecipeCommand = cli.Command{
+	Name:      "recipe",
+	Aliases:   []string{"r"},
+	Usage:     "Manage recipes",
+	ArgsUsage: "<recipe>",
+	Description: `
+A recipe is a blueprint for an app. It is a bunch of config files which
+describe how to deploy and maintain an app. Recipes are maintained by the Co-op
+Cloud community and you can use Abra to read them, deploy them and create apps
+for you.
+
+Anyone who uses a recipe can become a maintainer. Maintainers typically make
+sure the recipe is in good working order and the config upgraded in a timely
+manner. Abra supports convenient automation for recipe maintainenace, see the
+"abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands.
+`,
+	Subcommands: []cli.Command{
+		recipeFetchCommand,
+		recipeLintCommand,
+		recipeListCommand,
+		recipeNewCommand,
+		recipeReleaseCommand,
+		recipeSyncCommand,
+		recipeUpgradeCommand,
+		recipeVersionCommand,
+	},
+}

cli/recipe/release.go

@@ -0,0 +1,454 @@
+package recipe
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeReleaseCommand = cli.Command{
+	Name:      "release",
+	Aliases:   []string{"rl"},
+	Usage:     "Release a new recipe version",
+	ArgsUsage: "<recipe> [<version>]",
+	Description: `
+Create a new version of a recipe. These versions are then published on the
+Co-op Cloud recipe catalogue. These versions take the following form:
+
+    a.b.c+x.y.z
+
+Where the "a.b.c" part is a semantic version determined by the maintainer. The
+"x.y.z" part is the image tag of the recipe "app" service (the main container
+which contains the software to be used, by naming convention).
+
+We maintain a semantic versioning scheme ("a.b.c") alongside the recipe
+versioning scheme ("x.y.z") in order to maximise the chances that the nature of
+recipe updates are properly communicated. I.e. developers of an app might
+publish a minor version but that might lead to changes in the recipe which are
+major and therefore require intervention while doing the upgrade work.
+
+Publish your new release to git.coopcloud.tech with "-p/--publish". This
+requires that you have permission to git push to these repositories and have
+your SSH keys configured on your account.
+`,
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.DryFlag,
+		internal.MajorFlag,
+		internal.MinorFlag,
+		internal.PatchFlag,
+		internal.PublishFlag,
+		internal.OfflineFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		imagesTmp, err := getImageVersions(recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		mainApp, err := internal.GetMainAppImage(recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		mainAppVersion := imagesTmp[mainApp]
+		if mainAppVersion == "" {
+			logrus.Fatalf("main app service version for %s is empty?", recipe.Name)
+		}
+
+		tagString := c.Args().Get(1)
+		if tagString != "" {
+			if _, err := tagcmp.Parse(tagString); err != nil {
+				logrus.Fatalf("cannot parse %s, invalid tag specified?", tagString)
+			}
+		}
+
+		if (internal.Major || internal.Minor || internal.Patch) && tagString != "" {
+			logrus.Fatal("cannot specify tag and bump type at the same time")
+		}
+
+		if tagString != "" {
+			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		tags, err := recipe.Tags()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if tagString == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+			var err error
+			tagString, err = getLabelVersion(recipe, false)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if len(tags) > 0 {
+			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
+			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			logrus.Warnf("no tag specified and no previous tag available for %s, assuming this is the initial release", recipe.Name)
+
+			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+				if cleanUpErr := cleanUpTag(tagString, recipe.Name); err != nil {
+					logrus.Fatal(cleanUpErr)
+				}
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+// getImageVersions retrieves image versions for a recipe
+func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
+	var services = make(map[string]string)
+
+	missingTag := false
+	for _, service := range recipe.Config.Services {
+		if service.Image == "" {
+			continue
+		}
+
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return services, err
+		}
+
+		path := reference.Path(img)
+
+		path = formatter.StripTagMeta(path)
+
+		var tag string
+		switch img.(type) {
+		case reference.NamedTagged:
+			tag = img.(reference.NamedTagged).Tag()
+		case reference.Named:
+			if service.Name == "app" {
+				missingTag = true
+			}
+			continue
+		}
+
+		services[path] = tag
+	}
+
+	if missingTag {
+		return services, fmt.Errorf("app service is missing image tag?")
+	}
+
+	return services, nil
+}
+
+// createReleaseFromTag creates a new release based on a supplied recipe version string
+func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string) error {
+	var err error
+
+	directory := path.Join(config.RECIPES_DIR, recipe.Name)
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return err
+	}
+
+	tag, err := tagcmp.Parse(tagString)
+	if err != nil {
+		return err
+	}
+
+	if tag.MissingMinor {
+		tag.Minor = "0"
+		tag.MissingMinor = false
+	}
+
+	if tag.MissingPatch {
+		tag.Patch = "0"
+		tag.MissingPatch = false
+	}
+
+	if tagString == "" {
+		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
+	}
+
+	if err := commitRelease(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := tagRelease(tagString, repo); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := pushRelease(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}
+
+// btoi converts a boolean value into an integer
+func btoi(b bool) int {
+	if b {
+		return 1
+	}
+
+	return 0
+}
+
+// getTagCreateOptions constructs git tag create options
+func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
+	msg := fmt.Sprintf("chore: publish %s release", tag)
+	return git.CreateTagOptions{Message: msg}, nil
+}
+
+func commitRelease(recipe recipe.Recipe, tag string) error {
+	if internal.Dry {
+		logrus.Debugf("dry run: no changes committed")
+		return nil
+	}
+
+	isClean, err := gitPkg.IsClean(recipe.Dir())
+	if err != nil {
+		return err
+	}
+
+	if isClean {
+		if !internal.Dry {
+			return fmt.Errorf("no changes discovered in %s, nothing to publish?", recipe.Dir())
+		}
+	}
+
+	msg := fmt.Sprintf("chore: publish %s release", tag)
+	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
+	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func tagRelease(tagString string, repo *git.Repository) error {
+	if internal.Dry {
+		logrus.Debugf("dry run: no git tag created (%s)", tagString)
+		return nil
+	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return err
+	}
+
+	createTagOptions, err := getTagCreateOptions(tagString)
+	if err != nil {
+		return err
+	}
+
+	_, err = repo.CreateTag(tagString, head.Hash(), &createTagOptions)
+	if err != nil {
+		return err
+	}
+
+	hash := formatter.SmallSHA(head.Hash().String())
+	logrus.Debugf(fmt.Sprintf("created tag %s at %s", tagString, hash))
+
+	return nil
+}
+
+func pushRelease(recipe recipe.Recipe, tagString string) error {
+	if internal.Dry {
+		logrus.Info("dry run: no changes published")
+		return nil
+	}
+
+	if !internal.Publish && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: "publish new release?",
+		}
+
+		if err := survey.AskOne(prompt, &internal.Publish); err != nil {
+			return err
+		}
+	}
+
+	if internal.Publish {
+		if err := recipe.Push(internal.Dry); err != nil {
+			return err
+		}
+		url := fmt.Sprintf("%s/%s/src/tag/%s", config.REPOS_BASE_URL, recipe.Name, tagString)
+		logrus.Infof("new release published: %s", url)
+	} else {
+		logrus.Info("no -p/--publish passed, not publishing")
+	}
+
+	return nil
+}
+
+func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recipe.Recipe, tags []string) error {
+	directory := path.Join(config.RECIPES_DIR, recipe.Name)
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return err
+	}
+
+	bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+	if bumpType != 0 {
+		if (bumpType & (bumpType - 1)) != 0 {
+			return fmt.Errorf("you can only use one of: --major, --minor, --patch")
+		}
+	}
+
+	var lastGitTag tagcmp.Tag
+	for _, tag := range tags {
+		parsed, err := tagcmp.Parse(tag)
+		if err != nil {
+			return err
+		}
+
+		if (lastGitTag == tagcmp.Tag{}) {
+			lastGitTag = parsed
+		} else if parsed.IsGreaterThan(lastGitTag) {
+			lastGitTag = parsed
+		}
+	}
+
+	newTag := lastGitTag
+	if internal.Patch {
+		now, err := strconv.Atoi(newTag.Patch)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = strconv.Itoa(now + 1)
+	} else if internal.Minor {
+		now, err := strconv.Atoi(newTag.Minor)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = "0"
+		newTag.Minor = strconv.Itoa(now + 1)
+	} else if internal.Major {
+		now, err := strconv.Atoi(newTag.Major)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = "0"
+		newTag.Minor = "0"
+		newTag.Major = strconv.Itoa(now + 1)
+	}
+
+	if tagString == "" {
+		if err := internal.PromptBumpType(tagString, lastGitTag.String()); err != nil {
+			return err
+		}
+	}
+
+	if internal.Major || internal.Minor || internal.Patch {
+		newTag.Metadata = mainAppVersion
+		tagString = newTag.String()
+	}
+
+	if lastGitTag.String() == tagString {
+		logrus.Fatalf("latest git tag (%s) and synced label (%s) are the same?", lastGitTag, tagString)
+	}
+
+	if !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: fmt.Sprintf("current: %s, new: %s, correct?", lastGitTag, tagString),
+		}
+
+		var ok bool
+		if err := survey.AskOne(prompt, &ok); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !ok {
+			logrus.Fatal("exiting as requested")
+		}
+	}
+
+	if err := commitRelease(recipe, tagString); err != nil {
+		logrus.Fatalf("failed to commit changes: %s", err.Error())
+	}
+
+	if err := tagRelease(tagString, repo); err != nil {
+		logrus.Fatalf("failed to tag release: %s", err.Error())
+	}
+
+	if err := pushRelease(recipe, tagString); err != nil {
+		logrus.Fatalf("failed to publish new release: %s", err.Error())
+	}
+
+	return nil
+}
+
+// cleanUpTag removes a freshly created tag
+func cleanUpTag(tag, recipeName string) error {
+	directory := path.Join(config.RECIPES_DIR, recipeName)
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return err
+	}
+
+	if err := repo.DeleteTag(tag); err != nil {
+		if !strings.Contains(err.Error(), "not found") {
+			return err
+		}
+	}
+
+	logrus.Debugf("removed freshly created tag %s", tag)
+
+	return nil
+}
+
+func getLabelVersion(recipe recipe.Recipe, prompt bool) (string, error) {
+	initTag, err := recipePkg.GetVersionLabelLocal(recipe)
+	if err != nil {
+		return "", err
+	}
+
+	if initTag == "" {
+		logrus.Fatalf("unable to read version for %s from synced label. Did you try running \"abra recipe sync %s\" already?", recipe.Name, recipe.Name)
+	}
+
+	logrus.Warnf("discovered %s as currently synced recipe label", initTag)
+
+	if prompt && !internal.NoInput {
+		var response bool
+		prompt := &survey.Confirm{Message: fmt.Sprintf("use %s as the new version?", initTag)}
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return "", err
+		}
+
+		if !response {
+			return "", fmt.Errorf("please fix your synced label for %s and re-run this command", recipe.Name)
+		}
+	}
+
+	return initTag, nil
+}

cli/recipe/sync.go

@@ -0,0 +1,203 @@
+package recipe
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeSyncCommand = cli.Command{
+	Name:      "sync",
+	Aliases:   []string{"s"},
+	Usage:     "Sync recipe version label",
+	ArgsUsage: "<recipe> [<version>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.DryFlag,
+		internal.MajorFlag,
+		internal.MinorFlag,
+		internal.PatchFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Generate labels for the main recipe service (i.e. by convention, the service
+named "app") which corresponds to the following format:
+
+    coop-cloud.${STACK_NAME}.version=<version>
+
+Where <version> can be specifed on the command-line or Abra can attempt to
+auto-generate it for you. The <recipe> configuration will be updated on the
+local file system.
+`,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		mainApp, err := internal.GetMainAppImage(recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		imagesTmp, err := getImageVersions(recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		mainAppVersion := imagesTmp[mainApp]
+
+		tags, err := recipe.Tags()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		nextTag := c.Args().Get(1)
+		if len(tags) == 0 && nextTag == "" {
+			logrus.Warnf("no git tags found for %s", recipe.Name)
+			if internal.NoInput {
+				logrus.Fatalf("unable to continue, input required for initial version")
+			}
+			fmt.Println(fmt.Sprintf(`
+The following options are two types of initial semantic version that you can
+pick for %s that will be published in the recipe catalogue. This follows the
+semver convention (more on https://semver.org), here is a short cheatsheet
+
+    0.1.0: development release, still hacking. when you make a major upgrade
+           you increment the "y" part (i.e. 0.1.0 -> 0.2.0) and only move to
+           using the "x" part when things are stable.
+
+    1.0.0: public release, assumed to be working. you already have a stable
+           and reliable deployment of this app and feel relatively confident
+           about it.
+
+If you want people to be able alpha test your current config for %s but don't
+think it is quite reliable, go with 0.1.0 and people will know that things are
+likely to change.
+
+`, recipe.Name, recipe.Name))
+			var chosenVersion string
+			edPrompt := &survey.Select{
+				Message: "which version do you want to begin with?",
+				Options: []string{"0.1.0", "1.0.0"},
+			}
+
+			if err := survey.AskOne(edPrompt, &chosenVersion); err != nil {
+				logrus.Fatal(err)
+			}
+
+			nextTag = fmt.Sprintf("%s+%s", chosenVersion, mainAppVersion)
+		}
+
+		if nextTag == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+			latestRelease := tags[len(tags)-1]
+			if err := internal.PromptBumpType("", latestRelease); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if nextTag == "" {
+			recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+			repo, err := git.PlainOpen(recipeDir)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			var lastGitTag tagcmp.Tag
+			iter, err := repo.Tags()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := iter.ForEach(func(ref *plumbing.Reference) error {
+				obj, err := repo.TagObject(ref.Hash())
+				if err != nil {
+					logrus.Fatal("Tag at commit ", ref.Hash(), " is unannotated or otherwise broken. Please fix it.")
+					return err
+				}
+
+				tagcmpTag, err := tagcmp.Parse(obj.Name)
+				if err != nil {
+					return err
+				}
+
+				if (lastGitTag == tagcmp.Tag{}) {
+					lastGitTag = tagcmpTag
+				} else if tagcmpTag.IsGreaterThan(lastGitTag) {
+					lastGitTag = tagcmpTag
+				}
+
+				return nil
+			}); err != nil {
+				logrus.Fatal(err)
+			}
+
+			// bumpType is used to decide what part of the tag should be incremented
+			bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+			if bumpType != 0 {
+				// a bitwise check if the number is a power of 2
+				if (bumpType & (bumpType - 1)) != 0 {
+					logrus.Fatal("you can only use one version flag: --major, --minor or --patch")
+				}
+			}
+
+			newTag := lastGitTag
+			if bumpType > 0 {
+				if internal.Patch {
+					now, err := strconv.Atoi(newTag.Patch)
+					if err != nil {
+						logrus.Fatal(err)
+					}
+
+					newTag.Patch = strconv.Itoa(now + 1)
+				} else if internal.Minor {
+					now, err := strconv.Atoi(newTag.Minor)
+					if err != nil {
+						logrus.Fatal(err)
+					}
+
+					newTag.Patch = "0"
+					newTag.Minor = strconv.Itoa(now + 1)
+				} else if internal.Major {
+					now, err := strconv.Atoi(newTag.Major)
+					if err != nil {
+						logrus.Fatal(err)
+					}
+
+					newTag.Patch = "0"
+					newTag.Minor = "0"
+					newTag.Major = strconv.Itoa(now + 1)
+				}
+			}
+
+			newTag.Metadata = mainAppVersion
+			logrus.Debugf("choosing %s as new version for %s", newTag.String(), recipe.Name)
+			nextTag = newTag.String()
+		}
+
+		if _, err := tagcmp.Parse(nextTag); err != nil {
+			logrus.Fatalf("invalid version %s specified", nextTag)
+		}
+
+		mainService := "app"
+		label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", nextTag)
+		if !internal.Dry {
+			if err := recipe.UpdateLabel("compose.y*ml", mainService, label); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
+		}
+
+		return nil
+	},
+}

cli/recipe/upgrade.go

@@ -0,0 +1,341 @@
+package recipe
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type imgPin struct {
+	image   string
+	version tagcmp.Tag
+}
+
+// anUpgrade represents a single service upgrade (as within a recipe), and the list of tags that it can be upgraded to,
+// for serialization purposes.
+type anUpgrade struct {
+	Service     string   `json:"service"`
+	Image       string   `json:"image"`
+	Tag         string   `json:"tag"`
+	UpgradeTags []string `json:"upgrades"`
+}
+
+var recipeUpgradeCommand = cli.Command{
+	Name:    "upgrade",
+	Aliases: []string{"u"},
+	Usage:   "Upgrade recipe image tags",
+	Description: `
+Parse all image tags within the given <recipe> configuration and prompt with
+more recent tags to upgrade to. It will update the relevant compose file tags
+on the local file system.
+
+Some image tags cannot be parsed because they do not follow some sort of
+semver-like convention. In this case, all possible tags will be listed and it
+is up to the end-user to decide.
+
+The command is interactive and will show a select input which allows you to
+make a seclection. Use the "?" key to see more help on navigating this
+interface.
+
+You may invoke this command in "wizard" mode and be prompted for input:
+
+    abra recipe upgrade
+`,
+	ArgsUsage: "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.PatchFlag,
+		internal.MinorFlag,
+		internal.MajorFlag,
+		internal.MachineReadableFlag,
+		internal.AllTagsFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+		if bumpType != 0 {
+			// a bitwise check if the number is a power of 2
+			if (bumpType & (bumpType - 1)) != 0 {
+				logrus.Fatal("you can only use one of: --major, --minor, --patch.")
+			}
+		}
+
+		if internal.MachineReadable {
+			// -m implies -n in this case
+			internal.NoInput = true
+		}
+
+		upgradeList := make(map[string]anUpgrade)
+
+		// check for versions file and load pinned versions
+		versionsPresent := false
+		recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+		versionsPath := path.Join(recipeDir, "versions")
+		var servicePins = make(map[string]imgPin)
+		if _, err := os.Stat(versionsPath); err == nil {
+			logrus.Debugf("found versions file for %s", recipe.Name)
+			file, err := os.Open(versionsPath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := scanner.Text()
+				splitLine := strings.Split(line, " ")
+				if splitLine[0] != "pin" || len(splitLine) != 3 {
+					logrus.Fatalf("malformed version pin specification: %s", line)
+				}
+				pinSlice := strings.Split(splitLine[2], ":")
+				pinTag, err := tagcmp.Parse(pinSlice[1])
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				pin := imgPin{
+					image:   pinSlice[0],
+					version: pinTag,
+				}
+				servicePins[splitLine[1]] = pin
+			}
+			if err := scanner.Err(); err != nil {
+				logrus.Error(err)
+			}
+			versionsPresent = true
+		} else {
+			logrus.Debugf("did not find versions file for %s", recipe.Name)
+		}
+
+		for _, service := range recipe.Config.Services {
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			regVersions, err := client.GetRegistryTags(img)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			image := reference.Path(img)
+			logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
+			image = formatter.StripTagMeta(image)
+
+			switch img.(type) {
+			case reference.NamedTagged:
+				if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+					logrus.Debugf("%s not considered semver-like", img.(reference.NamedTagged).Tag())
+				}
+			default:
+				logrus.Warnf("unable to read tag for image %s, is it missing? skipping upgrade for %s", image, service.Name)
+				continue
+			}
+
+			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
+			if err != nil {
+				logrus.Warnf("unable to parse %s, error was: %s, skipping upgrade for %s", image, err.Error(), service.Name)
+				continue
+			}
+
+			logrus.Debugf("parsed %s for %s", tag, service.Name)
+
+			var compatible []tagcmp.Tag
+			for _, regVersion := range regVersions {
+				other, err := tagcmp.Parse(regVersion)
+				if err != nil {
+					continue // skip tags that cannot be parsed
+				}
+
+				if tag.IsCompatible(other) && tag.IsLessThan(other) && !tag.Equals(other) {
+					compatible = append(compatible, other)
+				}
+			}
+
+			logrus.Debugf("detected potential upgradable tags %s for %s", compatible, service.Name)
+
+			sort.Sort(tagcmp.ByTagDesc(compatible))
+
+			if len(compatible) == 0 && !internal.AllTags {
+				logrus.Info(fmt.Sprintf("no new versions available for %s, assuming %s is the latest (use -a/--all-tags to see all anyway)", image, tag))
+				continue // skip on to the next tag and don't update any compose files
+			}
+
+			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, internal.Offline)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			compatibleStrings := []string{"skip"}
+			for _, compat := range compatible {
+				skip := false
+				for _, catlVersion := range catlVersions {
+					if compat.String() == catlVersion {
+						skip = true
+					}
+				}
+				if !skip {
+					compatibleStrings = append(compatibleStrings, compat.String())
+				}
+			}
+
+			logrus.Debugf("detected compatible upgradable tags %s for %s", compatibleStrings, service.Name)
+
+			var upgradeTag string
+			_, ok := servicePins[service.Name]
+			if versionsPresent && ok {
+				pinnedTag := servicePins[service.Name].version
+				if tag.IsLessThan(pinnedTag) {
+					pinnedTagString := pinnedTag.String()
+					contains := false
+					for _, v := range compatible {
+						if pinnedTag.IsUpgradeCompatible(v) {
+							contains = true
+							upgradeTag = v.String()
+							break
+						}
+					}
+					if contains {
+						logrus.Infof("upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
+					} else {
+						logrus.Infof("service %s, image %s pinned to %s, no compatible upgrade found", service.Name, servicePins[service.Name].image, pinnedTagString)
+						continue
+					}
+				} else {
+					logrus.Fatalf("service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
+					continue
+				}
+			} else {
+				if bumpType != 0 {
+					for _, upTag := range compatible {
+						upElement, err := tag.UpgradeDelta(upTag)
+						if err != nil {
+							return err
+						}
+						delta := upElement.UpgradeType()
+						if delta <= bumpType {
+							upgradeTag = upTag.String()
+							break
+						}
+					}
+					if upgradeTag == "" {
+						logrus.Warnf("not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants", tag.String(), compatible[0].String(), image)
+						continue
+					}
+				} else {
+					msg := fmt.Sprintf("upgrade to which tag? (service: %s, image: %s, tag: %s)", service.Name, image, tag)
+					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) || internal.AllTags {
+						tag := img.(reference.NamedTagged).Tag()
+						if !internal.AllTags {
+							logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of %s, listing all tags", tag))
+						}
+						msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
+						compatibleStrings = []string{"skip"}
+						for _, regVersion := range regVersions {
+							compatibleStrings = append(compatibleStrings, regVersion)
+						}
+					}
+
+					// there is always at least the item "skip" in compatibleStrings (a list of
+					// possible upgradable tags) and at least one other tag.
+					upgradableTags := compatibleStrings[1:]
+					upgrade := anUpgrade{
+						Service:     service.Name,
+						Image:       image,
+						Tag:         tag.String(),
+						UpgradeTags: make([]string, len(upgradableTags)),
+					}
+
+					for n, s := range upgradableTags {
+						var sb strings.Builder
+						if _, err := sb.WriteString(s); err != nil {
+						}
+						upgrade.UpgradeTags[n] = sb.String()
+					}
+
+					upgradeList[upgrade.Service] = upgrade
+
+					if internal.NoInput {
+						upgradeTag = "skip"
+					} else {
+						prompt := &survey.Select{
+							Message: msg,
+							Help:    "enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled",
+							VimMode: true,
+							Options: compatibleStrings,
+						}
+						if err := survey.AskOne(prompt, &upgradeTag); err != nil {
+							logrus.Fatal(err)
+						}
+					}
+				}
+			}
+			if upgradeTag != "skip" {
+				ok, err := recipe.UpdateTag(image, upgradeTag)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if ok {
+					logrus.Infof("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image)
+				}
+			} else {
+				if !internal.NoInput {
+					logrus.Warnf("not upgrading %s, skipping as requested", image)
+				}
+			}
+		}
+
+		if internal.NoInput {
+			if internal.MachineReadable {
+				jsonstring, err := json.Marshal(upgradeList)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				fmt.Println(string(jsonstring))
+				return nil
+			}
+
+			for _, upgrade := range upgradeList {
+				logrus.Infof("can upgrade service: %s, image: %s, tag: %s ::\n", upgrade.Service, upgrade.Image, upgrade.Tag)
+				for _, utag := range upgrade.UpgradeTags {
+					logrus.Infof("    %s\n", utag)
+				}
+			}
+		}
+		return nil
+	},
+}

cli/recipe/version.go

@@ -0,0 +1,85 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/olekukonko/tablewriter"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		// NOTE(d1): corresponds to the `tableCol` definition below
+		if versions[i][1] == "app" {
+			return true
+		}
+		return versions[i][1] < versions[j][1]
+	}
+}
+
+var recipeVersionCommand = cli.Command{
+	Name:        "versions",
+	Aliases:     []string{"v"},
+	Usage:       "List recipe versions",
+	ArgsUsage:   "<recipe>",
+	Description: "Versions are read from the recipe catalogue.",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+		internal.MachineReadableFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		recipeMeta, ok := catl[recipe.Name]
+		if !ok {
+			logrus.Fatalf("%s is not published on the catalogue?", recipe.Name)
+		}
+
+		if len(recipeMeta.Versions) == 0 {
+			logrus.Fatalf("%s has no catalogue published versions?", recipe.Name)
+		}
+
+		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
+			tableCols := []string{"version", "service", "image", "tag"}
+			table := formatter.CreateTable(tableCols)
+			for version, meta := range recipeMeta.Versions[i] {
+				var versions [][]string
+				for service, serviceMeta := range meta {
+					versions = append(versions, []string{version, service, serviceMeta.Image, serviceMeta.Tag})
+				}
+
+				sort.Slice(versions, sortServiceByName(versions))
+
+				for _, version := range versions {
+					table.Append(version)
+				}
+
+				if internal.MachineReadable {
+					table.JSONRender()
+				} else {
+					table.SetAutoMergeCellsByColumnIndex([]int{0})
+					table.SetAlignment(tablewriter.ALIGN_LEFT)
+					table.Render()
+					fmt.Println()
+				}
+			}
+		}
+
+		return nil
+	},
+}

cli/server/add.go

@@ -0,0 +1,181 @@
+package server
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/server"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var local bool
+var localFlag = &cli.BoolFlag{
+	Name:        "local, l",
+	Usage:       "Use local server",
+	Destination: &local,
+}
+
+func cleanUp(domainName string) {
+	if domainName != "default" {
+		logrus.Infof("cleaning up context for %s", domainName)
+		if err := client.DeleteContext(domainName); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	logrus.Infof("attempting to clean up server directory for %s", domainName)
+
+	serverDir := filepath.Join(config.SERVERS_DIR, domainName)
+	files, err := config.GetAllFilesInDirectory(serverDir)
+	if err != nil {
+		logrus.Fatalf("unable to list files in %s: %s", serverDir, err)
+	}
+
+	if len(files) > 0 {
+		logrus.Warnf("aborting clean up of %s because it is not empty", serverDir)
+		return
+	}
+
+	if err := os.RemoveAll(serverDir); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+// newContext creates a new internal Docker context for a server. This is how
+// Docker manages SSH connection details. These are stored to disk in
+// ~/.docker. Abra can manage this completely for the user, so it's an
+// implementation detail.
+func newContext(c *cli.Context, domainName, username, port string) error {
+	store := contextPkg.NewDefaultDockerContextStore()
+	contexts, err := store.Store.List()
+	if err != nil {
+		return err
+	}
+
+	for _, context := range contexts {
+		if context.Name == domainName {
+			logrus.Debugf("context for %s already exists", domainName)
+			return nil
+		}
+	}
+
+	logrus.Debugf("creating context with domain %s, username %s and port %s", domainName, username, port)
+
+	if err := client.CreateContext(domainName, username, port); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// createServerDir creates the ~/.abra/servers/... directory for a new server.
+func createServerDir(domainName string) error {
+	if err := server.CreateServerDir(domainName); err != nil {
+		if !os.IsExist(err) {
+			return err
+		}
+		logrus.Debugf("server dir for %s already created", domainName)
+	}
+
+	return nil
+}
+
+var serverAddCommand = cli.Command{
+	Name:    "add",
+	Aliases: []string{"a"},
+	Usage:   "Add a server to your configuration",
+	Description: `
+Add a new server to your configuration so that it can be managed by Abra.
+
+Abra uses the SSH command-line to discover connection details for your server.
+It is advised to configure an entry per-host in your ~/.ssh/config for each
+server. For example:
+
+Host example.com
+  Hostname example.com
+  User exampleUser
+  Port 12345
+  IdentityFile ~/.ssh/example@somewhere
+
+Abra can then load SSH connection details from this configuratiion with:
+
+    abra server add example.com
+
+If "--local" is passed, then Abra assumes that the current local server is
+intended as the target server. This is useful when you want to have your entire
+Co-op Cloud config located on the server itself, and not on your local
+developer machine.
+`,
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		localFlag,
+	},
+	Before:    internal.SubCommandBefore,
+	ArgsUsage: "<domain>",
+	Action: func(c *cli.Context) error {
+		if len(c.Args()) > 0 && local || !internal.ValidateSubCmdFlags(c) {
+			err := errors.New("cannot use <domain> and --local together")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		var domainName string
+		if local {
+			domainName = "default"
+		} else {
+			domainName = internal.ValidateDomain(c)
+		}
+
+		if local {
+			if err := createServerDir(domainName); err != nil {
+				logrus.Fatal(err)
+			}
+
+			logrus.Infof("attempting to create client for %s", domainName)
+			if _, err := client.New(domainName); err != nil {
+				cleanUp(domainName)
+				logrus.Fatal(err)
+			}
+
+			logrus.Info("local server added")
+
+			return nil
+		}
+
+		if _, err := dns.EnsureIPv4(domainName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := createServerDir(domainName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		hostConfig, err := sshPkg.GetHostConfig(domainName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("attempting to create client for %s", domainName)
+		if _, err := client.New(domainName); err != nil {
+			cleanUp(domainName)
+			logrus.Debugf("failed to construct client for %s, saw %s", domainName, err.Error())
+			logrus.Fatal(sshPkg.Fatal(domainName, err))
+		}
+
+		logrus.Infof("%s added", domainName)
+
+		return nil
+	},
+}

cli/server/list.go

@@ -0,0 +1,110 @@
+package server
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/docker/cli/cli/connhelper/ssh"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var problemsFilter bool
+
+var problemsFilterFlag = &cli.BoolFlag{
+	Name:        "problems, p",
+	Usage:       "Show only servers with potential connection problems",
+	Destination: &problemsFilter,
+}
+
+var serverListCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
+	Usage:   "List managed servers",
+	Flags: []cli.Flag{
+		problemsFilterFlag,
+		internal.DebugFlag,
+		internal.MachineReadableFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		dockerContextStore := context.NewDefaultDockerContextStore()
+		contexts, err := dockerContextStore.Store.List()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableColumns := []string{"name", "host", "user", "port"}
+		table := formatter.CreateTable(tableColumns)
+
+		serverNames, err := config.ReadServerNames()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, serverName := range serverNames {
+			var row []string
+			for _, ctx := range contexts {
+				endpoint, err := context.GetContextEndpoint(ctx)
+				if err != nil && strings.Contains(err.Error(), "does not exist") {
+					// No local context found, we can continue safely
+					continue
+				}
+
+				if ctx.Name == serverName {
+					sp, err := ssh.ParseURL(endpoint)
+					if err != nil {
+						logrus.Fatal(err)
+					}
+
+					if sp.Host == "" {
+						sp.Host = "unknown"
+					}
+					if sp.User == "" {
+						sp.User = "unknown"
+					}
+					if sp.Port == "" {
+						sp.Port = "unknown"
+					}
+
+					row = []string{serverName, sp.Host, sp.User, sp.Port}
+				}
+			}
+
+			if len(row) == 0 {
+				if serverName == "default" {
+					row = []string{serverName, "local", "n/a", "n/a"}
+				} else {
+					row = []string{serverName, "unknown", "unknown", "unknown"}
+				}
+			}
+
+			if problemsFilter {
+				for _, val := range row {
+					if val == "unknown" {
+						table.Append(row)
+						break
+					}
+				}
+			} else {
+				table.Append(row)
+			}
+		}
+
+		if internal.MachineReadable {
+			table.JSONRender()
+		} else {
+			if problemsFilter && table.NumLines() == 0 {
+				logrus.Info("all servers wired up correctly 👏")
+			} else {
+				table.Render()
+			}
+		}
+
+		return nil
+	},
+}

cli/server/prune.go

@@ -0,0 +1,103 @@
+package server
+
+import (
+	"context"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var allFilter bool
+
+var allFilterFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Usage:       "Remove all unused images not just dangling ones",
+	Destination: &allFilter,
+}
+
+var volumesFilter bool
+
+var volumesFilterFlag = &cli.BoolFlag{
+	Name:        "volumes, v",
+	Usage:       "Prune volumes. This will remove app data, Be Careful!",
+	Destination: &volumesFilter,
+}
+
+var serverPruneCommand = cli.Command{
+	Name:    "prune",
+	Aliases: []string{"p"},
+	Usage:   "Prune a managed server; Runs a docker system prune",
+	Description: `
+Prunes unused containers, networks, and dangling images.
+
+If passing "-v/--volumes" then volumes not connected to a deployed app will
+also be removed. This can result in unwanted data loss if not used carefully.
+	`,
+	ArgsUsage: "[<server>]",
+	Flags: []cli.Flag{
+		allFilterFlag,
+		volumesFilterFlag,
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.ServerNameComplete,
+	Action: func(c *cli.Context) error {
+		serverName := internal.ValidateServer(c)
+
+		cl, err := client.New(serverName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var args filters.Args
+
+		ctx := context.Background()
+		cr, err := cl.ContainersPrune(ctx, args)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		cntSpaceReclaimed := formatter.ByteCountSI(cr.SpaceReclaimed)
+		logrus.Infof("containers pruned: %d; space reclaimed: %s", len(cr.ContainersDeleted), cntSpaceReclaimed)
+
+		nr, err := cl.NetworksPrune(ctx, args)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("networks pruned: %d", len(nr.NetworksDeleted))
+
+		pruneFilters := filters.NewArgs()
+		if allFilter {
+			logrus.Debugf("removing all images, not only dangling ones")
+			pruneFilters.Add("dangling", "false")
+		}
+
+		ir, err := cl.ImagesPrune(ctx, pruneFilters)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
+		logrus.Infof("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed)
+
+		if volumesFilter {
+			vr, err := cl.VolumesPrune(ctx, args)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			volSpaceReclaimed := formatter.ByteCountSI(vr.SpaceReclaimed)
+			logrus.Infof("volumes pruned: %d; space reclaimed: %s", len(vr.VolumesDeleted), volSpaceReclaimed)
+		}
+
+		return nil
+	},
+}

cli/server/remove.go

@@ -0,0 +1,48 @@
+package server
+
+import (
+	"os"
+	"path/filepath"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var serverRemoveCommand = cli.Command{
+	Name:      "remove",
+	Aliases:   []string{"rm"},
+	ArgsUsage: "<server>",
+	Usage:     "Remove a managed server",
+	Description: `Remove a managed server.
+
+Abra will remove the internal bookkeeping (~/.abra/servers/...) and underlying
+client connection context. This server will then be lost in time, like tears in
+rain.
+`,
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+		internal.OfflineFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.ServerNameComplete,
+	Action: func(c *cli.Context) error {
+		serverName := internal.ValidateServer(c)
+
+		if err := client.DeleteContext(serverName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, serverName)); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("server at %s has been lost in time, like tears in rain", serverName)
+
+		return nil
+	},
+}

cli/server/server.go

@@ -0,0 +1,18 @@
+package server
+
+import (
+	"github.com/urfave/cli"
+)
+
+// ServerCommand defines the `abra server` command and its subcommands
+var ServerCommand = cli.Command{
+	Name:    "server",
+	Aliases: []string{"s"},
+	Usage:   "Manage servers",
+	Subcommands: []cli.Command{
+		serverAddCommand,
+		serverListCommand,
+		serverRemoveCommand,
+		serverPruneCommand,
+	},
+}

cli/updater/updater.go

@@ -0,0 +1,498 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerclient "github.com/docker/docker/client"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+const SERVER = "localhost"
+
+var majorUpdate bool
+var majorFlag = &cli.BoolFlag{
+	Name:        "major, m",
+	Usage:       "Also check for major updates",
+	Destination: &majorUpdate,
+}
+
+var updateAll bool
+var allFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Usage:       "Update all deployed apps",
+	Destination: &updateAll,
+}
+
+// Notify checks for available upgrades
+var Notify = cli.Command{
+	Name:    "notify",
+	Aliases: []string{"n"},
+	Usage:   "Check for available upgrades",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		majorFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+It reads the deployed app versions and looks for new versions in the recipe
+catalogue. If a new patch/minor version is available, a notification is
+printed. To include major versions use the --major flag.
+`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if recipeName != "" {
+				_, err = getLatestUpgrade(cl, stackName, recipeName)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+// UpgradeApp upgrades apps.
+var UpgradeApp = cli.Command{
+	Name:      "upgrade",
+	Aliases:   []string{"u"},
+	Usage:     "Upgrade apps",
+	ArgsUsage: "<stack-name> <recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.ChaosFlag,
+		majorFlag,
+		allFlag,
+		internal.OfflineFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Description: `
+Upgrade an app by specifying its stack name and recipe. By passing "--all"
+instead, every deployed app is upgraded. For each apps with enabled auto
+updates the deployed version is compared with the current recipe catalogue
+version. If a new patch/minor version is available, the app is upgraded. To
+include major versions use the "--major" flag. Don't do that, it will probably
+break things. Only apps that are not deployed with "--chaos" are upgraded, to
+update chaos deployments use the "--chaos" flag. Use it with care.
+`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !updateAll {
+			stackName := c.Args().Get(0)
+			recipeName := c.Args().Get(1)
+			err = tryUpgrade(cl, stackName, recipeName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			return nil
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			err = tryUpgrade(cl, stackName, recipeName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+// getLabel reads docker labels from running services in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
+func getLabel(cl *dockerclient.Client, stackName string, label string) (string, error) {
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return "", err
+	}
+
+	for _, service := range services {
+		labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+		if labelValue, ok := service.Spec.Labels[labelKey]; ok {
+			return labelValue, nil
+		}
+	}
+
+	logrus.Debugf("no %s label found for %s", label, stackName)
+
+	return "", nil
+}
+
+// getBoolLabel reads a boolean docker label from running services
+func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool, error) {
+	lableValue, err := getLabel(cl, stackName, label)
+	if err != nil {
+		return false, err
+	}
+
+	if lableValue != "" {
+		value, err := strconv.ParseBool(lableValue)
+		if err != nil {
+			return false, err
+		}
+
+		return value, nil
+	}
+
+	logrus.Debugf("Boolean label %s could not be found for %s, set default to false.", label, stackName)
+
+	return false, nil
+}
+
+// getEnv reads env variables from docker services.
+func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) {
+	envMap := make(map[string]string)
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, service := range services {
+		envList := service.Spec.TaskTemplate.ContainerSpec.Env
+		for _, envString := range envList {
+			splitString := strings.SplitN(envString, "=", 2)
+			if len(splitString) != 2 {
+				logrus.Debugf("can't separate key from value: %s (this variable is probably unset)", envString)
+				continue
+			}
+			k := splitString[0]
+			v := splitString[1]
+			logrus.Debugf("For %s read env %s with value: %s from docker service", stackName, k, v)
+			envMap[k] = v
+		}
+	}
+
+	return envMap, nil
+}
+
+// getLatestUpgrade returns the latest available version for an app respecting
+// the "--major" flag if it is newer than the currently deployed version.
+func getLatestUpgrade(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	deployedVersion, err := getDeployedVersion(cl, stackName, recipeName)
+	if err != nil {
+		return "", err
+	}
+
+	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion)
+	if err != nil {
+		return "", err
+	}
+
+	if len(availableUpgrades) == 0 {
+		logrus.Debugf("no available upgrades for %s", stackName)
+		return "", nil
+	}
+
+	var chosenUpgrade string
+	if len(availableUpgrades) > 0 {
+		chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+		logrus.Infof("%s (%s) can be upgraded from version %s to %s", stackName, recipeName, deployedVersion, chosenUpgrade)
+	}
+
+	return chosenUpgrade, nil
+}
+
+// getDeployedVersion returns the currently deployed version of an app.
+func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	logrus.Debugf("Retrieve deployed version whether %s is already deployed", stackName)
+
+	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+	if err != nil {
+		return "", err
+	}
+
+	if !isDeployed {
+		return "", fmt.Errorf("%s is not deployed?", stackName)
+	}
+
+	if deployedVersion == "unknown" {
+		return "", fmt.Errorf("failed to determine deployed version of %s", stackName)
+	}
+
+	return deployedVersion, nil
+}
+
+// getAvailableUpgrades returns all available versions of an app that are newer
+// than the deployed version. It only includes major upgrades if the "--major"
+// flag is set.
+func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
+	deployedVersion string) ([]string, error) {
+	catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+	if err != nil {
+		return nil, err
+	}
+
+	versions, err := recipe.GetRecipeCatalogueVersions(recipeName, catl)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(versions) == 0 {
+		logrus.Warnf("no published releases for %s in the recipe catalogue?", recipeName)
+		return nil, nil
+	}
+
+	var availableUpgrades []string
+	for _, version := range versions {
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return nil, err
+		}
+
+		versionDelta, err := parsedDeployedVersion.UpgradeDelta(parsedVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		if 0 < versionDelta.UpgradeType() && (versionDelta.UpgradeType() < 4 || majorUpdate) {
+			availableUpgrades = append(availableUpgrades, version)
+		}
+	}
+
+	logrus.Debugf("Available updates for %s: %s", stackName, availableUpgrades)
+
+	return availableUpgrades, nil
+}
+
+// processRecipeRepoVersion clones, pulls, checks out the version and lints the
+// recipe repository.
+func processRecipeRepoVersion(recipeName, version string) error {
+	if err := recipe.EnsureExists(recipeName); err != nil {
+		return err
+	}
+
+	if err := recipe.EnsureUpToDate(recipeName); err != nil {
+		return err
+	}
+
+	if err := recipe.EnsureVersion(recipeName, version); err != nil {
+		return err
+	}
+
+	if r, err := recipe.Get(recipeName, internal.Offline); err != nil {
+		return err
+	} else if err := lint.LintForErrors(r); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// mergeAbraShEnv merges abra.sh env vars into the app env vars.
+func mergeAbraShEnv(recipeName string, env config.AppEnv) error {
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		return err
+	}
+
+	for k, v := range abraShEnv {
+		logrus.Debugf("read v:%s k: %s", v, k)
+		env[k] = v
+	}
+
+	return nil
+}
+
+// createDeployConfig merges and enriches the compose config for the deployment.
+func createDeployConfig(recipeName string, stackName string, env config.AppEnv) (*composetypes.Config, stack.Deploy, error) {
+	env["STACK_NAME"] = stackName
+
+	deployOpts := stack.Deploy{
+		Namespace:    stackName,
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+
+	composeFiles, err := config.GetComposeFiles(recipeName, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+
+	deployOpts.Composefiles = composeFiles
+	compose, err := config.GetAppComposeConfig(stackName, deployOpts, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+
+	config.ExposeAllEnv(stackName, compose, env)
+
+	// after the upgrade the deployment won't be in chaos state anymore
+	config.SetChaosLabel(compose, stackName, false)
+	config.SetRecipeLabel(compose, stackName, recipeName)
+	config.SetUpdateLabel(compose, stackName, env)
+
+	return compose, deployOpts, nil
+}
+
+// tryUpgrade performs the upgrade if all the requirements are fulfilled.
+func tryUpgrade(cl *dockerclient.Client, stackName, recipeName string) error {
+	if recipeName == "" {
+		logrus.Debugf("don't update %s due to missing recipe name", stackName)
+		return nil
+	}
+
+	chaos, err := getBoolLabel(cl, stackName, "chaos")
+	if err != nil {
+		return err
+	}
+
+	if chaos && !internal.Chaos {
+		logrus.Debugf("don't update %s due to chaos deployment", stackName)
+		return nil
+	}
+
+	updatesEnabled, err := getBoolLabel(cl, stackName, "autoupdate")
+	if err != nil {
+		return err
+	}
+
+	if !updatesEnabled {
+		logrus.Debugf("don't update %s due to disabled auto updates or missing ENABLE_AUTO_UPDATE env", stackName)
+		return nil
+	}
+
+	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName)
+	if err != nil {
+		return err
+	}
+
+	if upgradeVersion == "" {
+		logrus.Debugf("don't update %s due to no new version", stackName)
+		return nil
+	}
+
+	err = upgrade(cl, stackName, recipeName, upgradeVersion)
+
+	return err
+}
+
+// upgrade performs all necessary steps to upgrade an app.
+func upgrade(cl *dockerclient.Client, stackName, recipeName,
+	upgradeVersion string) error {
+	env, err := getEnv(cl, stackName)
+	if err != nil {
+		return err
+	}
+
+	app := config.App{
+		Name:   stackName,
+		Recipe: recipeName,
+		Server: SERVER,
+		Env:    env,
+	}
+
+	if err = processRecipeRepoVersion(recipeName, upgradeVersion); err != nil {
+		return err
+	}
+
+	if err = mergeAbraShEnv(recipeName, app.Env); err != nil {
+		return err
+	}
+
+	compose, deployOpts, err := createDeployConfig(recipeName, stackName, app.Env)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
+
+	err = stack.RunDeploy(cl, deployOpts, compose, stackName, true)
+
+	return err
+}
+
+func newAbraApp(version, commit string) *cli.App {
+	app := &cli.App{
+		Name: "kadabra",
+		Usage: `The Co-op Cloud auto-updater
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []cli.Command{
+			Notify,
+			UpgradeApp,
+		},
+	}
+
+	app.Before = func(c *cli.Context) error {
+		logrus.Debugf("kadabra version %s, commit %s", version, commit)
+		return nil
+	}
+
+	return app
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := newAbraApp(version, commit)
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cmd/abra/main.go

@@ -0,0 +1,23 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli"
+)
+
+// Version is the current version of Abra.
+var Version string
+
+// Commit is the current git commit of Abra.
+var Commit string
+
+func main() {
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	cli.RunApp(Version, Commit)
+}

cmd/kadabra/main.go

@@ -0,0 +1,23 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli/updater"
+)
+
+// Version is the current version of Kadabra.
+var Version string
+
+// Commit is the current git commit of Kadabra.
+var Commit string
+
+func main() {
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	updater.RunApp(Version, Commit)
+}

completion/_abra

@@ -1,52 +0,0 @@
-#compdef abra
-
-_abra () {
-	local context state line curcontext="$curcontext" ret=1
-	_arguments -n : \
-		{-h,--help}'[Help message]' \
-		'1:commands:(app server)' \
-		'*::arguments:->arguments' \
-		&& ret=0
-
-	case $state in
-		(arguments)
-			curcontext="${curcontext%:*:*}:abra-arguments-$words[1]:"
-			case $words[1] in
-				(app)
-					_arguments \
-						'1: :_abra_apps' \
-					&& ret=0
-				;;
-				(server)
-					_arguments \
-						'1:servers:_abra_servers' \
-					&& ret=0
-				;;
-			esac
-		;;
-	esac
-
-	return ret
-}
-
-_abra_servers() {
-	_path_files -/W $HOME/.abra/servers
-}
-
-_abra_apps()
-{
-	local newapps apps=($HOME/.abra/servers/*/*.env)
-	typeset -a apps
-	newapps=()
-	for app in $apps; do
-		newapps+=($(_abra_basename "${app}"))
-	done
-	_describe -t apps 'app' newapps
-}
-
-_abra_basename()
-{
-    printf -- "${1##*/}"
-}
-
-_abra "$@"

completion/abra.bash

@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-_abra_basename()
-{
-    echo "${1##*/}"
-}
-
-_abra_servers()
-{
-    # FIXME 3wc: copied from abra/get_servers()
-    shopt -s nullglob dotglob
-    local SERVERS=(~/.abra/servers/*)
-    shopt -u nullglob dotglob
-
-    for SERVER in "${SERVERS[@]}"; do
-        _abra_basename "${SERVER}"
-    done
-}
-
-_abra_complete_servers()
-{
-    mapfile -t COMPREPLY < <(compgen -W "$(_abra_servers)" -- "$1")
-}
-
-_abra_apps()
-{
-    shopt -s nullglob dotglob
-    local APPS=(~/.abra/servers/*/*.env)
-    shopt -u nullglob dotglob
-
-    for APP in "${APPS[@]}"; do
-        _abra_basename "${APP%.env}"
-    done
-}
-
-_abra_complete_apps()
-{
-    mapfile -t COMPREPLY < <(compgen -W "$(_abra_apps)" -- "$1")
-}
-
-_abra_complete()
-{
-    compopt +o default +o nospace
-    COMPREPLY=()
-
-    local -r cmds='
-        app
-        server
-    '
-    local -r short_opts='-e      -h     -s      -v'
-    local -r long_opts='--env --help --stack --version'
-
-    # Scan through the command line and find the abra command
-    # (if present), as well as its expected position.
-    local cmd
-    local cmd_index=1 # Expected index of the command token.
-    local i
-    for (( i = 1; i < ${#COMP_WORDS[@]}; i++ )); do
-        local word="${COMP_WORDS[i]}"
-        case "$word" in
-            -*)
-                ((cmd_index++))
-                ;;
-            *)
-                cmd="$word"
-                break
-                ;;
-        esac
-    done
-
-    local cur="${COMP_WORDS[COMP_CWORD]}"
-
-    if (( COMP_CWORD < cmd_index )); then
-        # Offer option completions.
-        case "$cur" in
-            --*)
-                mapfile -t COMPREPLY < <(compgen -W "$long_opts" -- "$cur")
-                ;;
-            -*)
-                mapfile -t COMPREPLY < <(compgen -W "$short_opts" -- "$cur")
-                ;;
-            *)
-                # Skip completion; we should never get here.
-                ;;
-        esac
-    elif (( COMP_CWORD == cmd_index )); then
-        # Offer command name completions.
-        mapfile -t COMPREPLY < <(compgen -W "$cmds" -- "$cur")
-    else
-        # Offer command argument completions.
-        case "$cmd" in
-            server)
-                # Offer exactly one server name completion.
-                if (( COMP_CWORD == cmd_index + 1 )); then
-                    _abra_complete_servers "$cur"
-                fi
-                ;;
-            app)
-                # Offer exactly one app completion.
-                if (( COMP_CWORD == cmd_index + 1 )); then
-                    _abra_complete_apps "$cur"
-                fi
-                ;;
-            #help)
-            #    # Offer exactly one command name completion.
-            #    if (( COMP_CWORD == cmd_index + 1 )); then
-            #        COMPREPLY=($(compgen -W "$cmds" -- "$cur"))
-            #    fi
-            #    ;;
-            *)
-                # Unknown command or unknowable argument.
-                ;;
-        esac
-    fi
-}
-
-complete -o default -F _abra_complete abra

go.mod

@@ -0,0 +1,119 @@
+module coopcloud.tech/abra
+
+go 1.18
+
+require (
+	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	github.com/AlecAivazis/survey/v2 v2.3.7
+	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
+	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
+	github.com/docker/cli v24.0.6+incompatible
+	github.com/docker/distribution v2.8.2+incompatible
+	github.com/docker/docker v24.0.6+incompatible
+	github.com/docker/go-units v0.5.0
+	github.com/go-git/go-git/v5 v5.9.0
+	github.com/moby/sys/signal v0.7.0
+	github.com/moby/term v0.5.0
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/pkg/errors v0.9.1
+	github.com/schollz/progressbar/v3 v3.13.1
+	github.com/sirupsen/logrus v1.9.3
+	gotest.tools/v3 v3.5.1
+)
+
+require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/BurntSushi/toml v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.9.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/miekg/pkcs11 v1.0.3 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/runc v1.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/skeema/knownhosts v1.2.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	golang.org/x/tools v0.13.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
+	github.com/buger/goterm v1.0.4
+	github.com/containerd/containerd v1.5.9 // indirect
+	github.com/containers/image v3.0.2+incompatible
+	github.com/containers/storage v1.38.2 // indirect
+	github.com/decentral1se/passgen v1.0.1
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4
+	github.com/klauspost/pgzip v1.2.6
+	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/stretchr/testify v1.8.4
+	github.com/theupdateframework/notary v0.7.0 // indirect
+	github.com/urfave/cli v1.22.9
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
+	golang.org/x/sys v0.12.0
+)

installer/installer

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-ABRA_VERSION="0.5.0"
-ABRA_SRC="https://git.autonomic.zone/coop-cloud/abra/raw/tag/$ABRA_VERSION/abra"
-
-function install_abra {
-  mkdir -p "$HOME/.local/bin"
-  curl "$ABRA_SRC" > "$HOME/.local/bin/abra"
-  chmod +x "$HOME/.local/bin/abra"
-  echo "abra installed to $HOME/.local/bin/abra"
-}
-
-function run_installation {
-  install_abra
-}
-
-run_installation
-exit 0

pkg/app/app.go

@@ -0,0 +1,42 @@
+package app
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+)
+
+// Get retrieves an app
+func Get(appName string) (config.App, error) {
+	files, err := config.LoadAppFiles("")
+	if err != nil {
+		return config.App{}, err
+	}
+
+	app, err := config.GetApp(files, appName)
+	if err != nil {
+		return config.App{}, err
+	}
+
+	logrus.Debugf("retrieved %s for %s", app, appName)
+
+	return app, nil
+}
+
+// deployedServiceSpec represents a deployed service of an app.
+type deployedServiceSpec struct {
+	Name    string
+	Version string
+}
+
+// VersionSpec represents a deployed app and associated metadata.
+type VersionSpec map[string]deployedServiceSpec
+
+// ParseServiceName parses a $STACK_NAME_$SERVICE_NAME service label.
+func ParseServiceName(label string) string {
+	idx := strings.LastIndex(label, "_")
+	serviceName := label[idx+1:]
+	logrus.Debugf("parsed %s as service name from %s", serviceName, label)
+	return serviceName
+}

pkg/autocomplete/autocomplete.go

@@ -0,0 +1,78 @@
+package autocomplete
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// AppNameComplete copletes app names.
+func AppNameComplete(c *cli.Context) {
+	appNames, err := config.GetAppNames()
+	if err != nil {
+		logrus.Warn(err)
+	}
+
+	if c.NArg() > 0 {
+		return
+	}
+
+	for _, a := range appNames {
+		fmt.Println(a)
+	}
+}
+
+// RecipeNameComplete completes recipe names.
+func RecipeNameComplete(c *cli.Context) {
+	catl, err := recipe.ReadRecipeCatalogue(false)
+	if err != nil {
+		logrus.Warn(err)
+	}
+
+	if c.NArg() > 0 {
+		return
+	}
+
+	for name := range catl {
+		fmt.Println(name)
+	}
+}
+
+// ServerNameComplete completes server names.
+func ServerNameComplete(c *cli.Context) {
+	files, err := config.LoadAppFiles("")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if c.NArg() > 0 {
+		return
+	}
+
+	for _, appFile := range files {
+		fmt.Println(appFile.Server)
+	}
+}
+
+// SubcommandComplete completes sub-commands.
+func SubcommandComplete(c *cli.Context) {
+	if c.NArg() > 0 {
+		return
+	}
+
+	subcmds := []string{
+		"app",
+		"autocomplete",
+		"catalogue",
+		"recipe",
+		"server",
+		"upgrade",
+	}
+
+	for _, cmd := range subcmds {
+		fmt.Println(cmd)
+	}
+}

pkg/catalogue/catalogue.go

@@ -0,0 +1,127 @@
+package catalogue
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                         true,
+	"abra-apps":                    true,
+	"abra-aur":                     true,
+	"abra-bash":                    true,
+	"abra-capsul":                  true,
+	"abra-gandi":                   true,
+	"abra-hetzner":                 true,
+	"abra-integration-test-recipe": true,
+	"apps":                         true,
+	"aur-abra-git":                 true,
+	"auto-mirror":                  true,
+	"auto-recipes-catalogue-json":  true,
+	"backup-bot":                   true,
+	"backup-bot-two":               true,
+	"beta.coopcloud.tech":          true,
+	"comrade-renovate-bot":         true,
+	"coopcloud.tech":               true,
+	"coturn":                       true,
+	"docker-cp-deploy":             true,
+	"docker-dind-bats-kcov":        true,
+	"docs.coopcloud.tech":          true,
+	"drone-abra":                   true,
+	"example":                      true,
+	"gardening":                    true,
+	"go-abra":                      true,
+	"organising":                   true,
+	"pyabra":                       true,
+	"radicle-seed-node":            true,
+	"recipes-catalogue-json":       true,
+	"recipes-wishlist":             true,
+	"recipes.coopcloud.tech":       true,
+	"stack-ssh-deploy":             true,
+	"swarm-cronjob":                true,
+	"tagcmp":                       true,
+	"traefik-cert-dumper":          true,
+	"tyop":                         true,
+}
+
+// EnsureCatalogue ensures that the catalogue is cloned locally & present.
+func EnsureCatalogue() error {
+	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+
+		logrus.Debugf("cloned catalogue repository to %s", catalogueDir)
+	}
+
+	return nil
+}
+
+// EnsureIsClean makes sure that the catalogue has no unstaged changes.
+func EnsureIsClean() error {
+	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		msg := "%s has locally unstaged changes? please commit/remove your changes before proceeding"
+		return fmt.Errorf(msg, config.CATALOGUE_DIR)
+	}
+
+	return nil
+}
+
+// EnsureUpToDate ensures that the local catalogue is up to date.
+func EnsureUpToDate() error {
+	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	remotes, err := repo.Remotes()
+	if err != nil {
+		return err
+	}
+
+	if len(remotes) == 0 {
+		msg := "cannot ensure %s is up-to-date, no git remotes configured"
+		logrus.Debugf(msg, config.CATALOGUE_DIR)
+		return nil
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	branch, err := gitPkg.CheckoutDefaultBranch(repo, config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	opts := &git.PullOptions{
+		Force:         true,
+		ReferenceName: branch,
+	}
+
+	if err := worktree.Pull(opts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return err
+		}
+	}
+
+	logrus.Debugf("fetched latest git changes for %s", config.CATALOGUE_DIR)
+
+	return nil
+}

pkg/client/client.go

@@ -0,0 +1,84 @@
+// Package client provides Docker client initiatialisation functions.
+package client
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	contextPkg "coopcloud.tech/abra/pkg/context"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
+	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// New initiates a new Docker client. New client connections are validated so
+// that we ensure connections via SSH to the daemon can succeed. It takes into
+// account that you may only want the local client and not communicate via SSH.
+// For this use-case, please pass "default" as the contextName.
+func New(serverName string) (*client.Client, error) {
+	var clientOpts []client.Opt
+
+	if serverName != "default" {
+		context, err := GetContext(serverName)
+		if err != nil {
+			return nil, fmt.Errorf("unknown server, run \"abra server add %s\"?", serverName)
+		}
+
+		ctxEndpoint, err := contextPkg.GetContextEndpoint(context)
+		if err != nil {
+			return nil, err
+		}
+
+		helper, err := commandconnPkg.NewConnectionHelper(ctxEndpoint)
+		if err != nil {
+			return nil, err
+		}
+
+		httpClient := &http.Client{
+			Transport: &http.Transport{
+				DialContext:     helper.Dialer,
+				IdleConnTimeout: 30 * time.Second,
+			},
+		}
+
+		clientOpts = append(clientOpts,
+			client.WithHTTPClient(httpClient),
+			client.WithHost(helper.Host),
+			client.WithDialContext(helper.Dialer),
+		)
+	}
+
+	version := os.Getenv("DOCKER_API_VERSION")
+	if version != "" {
+		clientOpts = append(clientOpts, client.WithVersion(version))
+	} else {
+		clientOpts = append(clientOpts, client.WithAPIVersionNegotiation())
+	}
+
+	cl, err := client.NewClientWithOpts(clientOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("created client for %s", serverName)
+
+	info, err := cl.Info(context.Background())
+	if err != nil {
+		return cl, sshPkg.Fatal(serverName, err)
+	}
+
+	if info.Swarm.LocalNodeState == "inactive" {
+		if serverName != "default" {
+			return cl, fmt.Errorf("swarm mode not enabled on %s?", serverName)
+		} else {
+			return cl, errors.New("swarm mode not enabled on local server?")
+		}
+	}
+
+	return cl, nil
+}

pkg/client/context.go

@@ -0,0 +1,91 @@
+package client
+
+import (
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/context"
+	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
+	dConfig "github.com/docker/cli/cli/config"
+	"github.com/docker/cli/cli/context/docker"
+	contextStore "github.com/docker/cli/cli/context/store"
+	"github.com/sirupsen/logrus"
+)
+
+type Context = contextStore.Metadata
+
+func CreateContext(contextName string, user string, port string) error {
+	host := contextName
+	if user != "" {
+		host = fmt.Sprintf("%s@%s", user, host)
+	}
+	if port != "" {
+		host = fmt.Sprintf("%s:%s", host, port)
+	}
+	host = fmt.Sprintf("ssh://%s", host)
+	if err := createContext(contextName, host); err != nil {
+		return err
+	}
+	logrus.Debugf("created the %s context", contextName)
+	return nil
+}
+
+// createContext interacts with Docker Context to create a Docker context config
+func createContext(name string, host string) error {
+	s := context.NewDefaultDockerContextStore()
+	contextMetadata := contextStore.Metadata{
+		Endpoints: make(map[string]interface{}),
+		Name:      name,
+	}
+
+	contextTLSData := contextStore.ContextTLSData{
+		Endpoints: make(map[string]contextStore.EndpointTLSData),
+	}
+
+	dockerEP, dockerTLS, err := commandconnPkg.GetDockerEndpointMetadataAndTLS(host)
+	if err != nil {
+		return err
+	}
+
+	contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP
+	if dockerTLS != nil {
+		contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerTLS
+	}
+
+	if err := s.CreateOrUpdate(contextMetadata); err != nil {
+		return err
+	}
+
+	if err := s.ResetTLSMaterial(name, &contextTLSData); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DeleteContext(name string) error {
+	if name == "default" {
+		return errors.New("context 'default' cannot be removed")
+	}
+
+	if _, err := GetContext(name); err != nil {
+		return err
+	}
+
+	cfg := dConfig.LoadDefaultConfigFile(nil)
+	cfg.CurrentContext = ""
+	if err := cfg.Save(); err != nil {
+		return err
+	}
+
+	return context.NewDefaultDockerContextStore().Remove(name)
+}
+
+func GetContext(contextName string) (contextStore.Metadata, error) {
+	ctx, err := context.NewDefaultDockerContextStore().GetMetadata(contextName)
+	if err != nil {
+		return contextStore.Metadata{}, err
+	}
+
+	return ctx, nil
+}

pkg/client/registry.go

@@ -0,0 +1,28 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/containers/image/docker"
+	"github.com/containers/image/types"
+	"github.com/docker/distribution/reference"
+)
+
+// GetRegistryTags retrieves all tags of an image from a container registry.
+func GetRegistryTags(img reference.Named) ([]string, error) {
+	var tags []string
+
+	ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
+	if err != nil {
+		return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
+	}
+
+	ctx := context.Background()
+	tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
+	if err != nil {
+		return tags, err
+	}
+
+	return tags, nil
+}

pkg/client/secret.go

@@ -0,0 +1,19 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+)
+
+func StoreSecret(cl *client.Client, secretName, secretValue, server string) error {
+	ann := swarm.Annotations{Name: secretName}
+	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
+
+	if _, err := cl.SecretCreate(context.Background(), spec); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/client/volumes.go

@@ -0,0 +1,41 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
+)
+
+func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
+	volumeListOptions := volume.ListOptions{fs}
+	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
+	volumeList := volumeListOKBody.Volumes
+	if err != nil {
+		return volumeList, err
+	}
+
+	return volumeList, nil
+}
+
+func GetVolumeNames(volumes []*volume.Volume) []string {
+	var volumeNames []string
+
+	for _, vol := range volumes {
+		volumeNames = append(volumeNames, vol.Name)
+	}
+
+	return volumeNames
+}
+
+func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
+	for _, volName := range volumeNames {
+		err := cl.VolumeRemove(ctx, volName, force)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/compose/compose.go

@@ -0,0 +1,158 @@
+package compose
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+)
+
+// UpdateTag updates an image tag in-place on file system local compose files.
+func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return false, err
+	}
+
+	logrus.Debugf("considering %s config(s) for tag update", strings.Join(composeFiles, ", "))
+
+	for _, composeFile := range composeFiles {
+		opts := stack.Deploy{Composefiles: []string{composeFile}}
+
+		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
+		sampleEnv, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			return false, err
+		}
+
+		compose, err := loader.LoadComposefile(opts, sampleEnv)
+		if err != nil {
+			return false, err
+		}
+
+		for _, service := range compose.Services {
+			if service.Image == "" {
+				continue // may be a compose.$optional.yml file
+			}
+
+			img, _ := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return false, err
+			}
+
+			var composeTag string
+			switch img.(type) {
+			case reference.NamedTagged:
+				composeTag = img.(reference.NamedTagged).Tag()
+			default:
+				logrus.Debugf("unable to parse %s, skipping", img)
+				continue
+			}
+
+			composeImage := formatter.StripTagMeta(reference.Path(img))
+
+			logrus.Debugf("parsed %s from %s", composeTag, service.Image)
+
+			if image == composeImage {
+				bytes, err := ioutil.ReadFile(composeFile)
+				if err != nil {
+					return false, err
+				}
+
+				old := fmt.Sprintf("%s:%s", composeImage, composeTag)
+				new := fmt.Sprintf("%s:%s", composeImage, tag)
+				replacedBytes := strings.Replace(string(bytes), old, new, -1)
+
+				logrus.Debugf("updating %s to %s in %s", old, new, compose.Filename)
+
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0764); err != nil {
+					return false, err
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
+// UpdateLabel updates a label in-place on file system local compose files.
+func UpdateLabel(pattern, serviceName, label, recipeName string) error {
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("considering %s config(s) for label update", strings.Join(composeFiles, ", "))
+
+	for _, composeFile := range composeFiles {
+		opts := stack.Deploy{Composefiles: []string{composeFile}}
+
+		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
+		sampleEnv, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			return err
+		}
+
+		compose, err := loader.LoadComposefile(opts, sampleEnv)
+		if err != nil {
+			return err
+		}
+
+		serviceExists := false
+		var service composetypes.ServiceConfig
+		for _, s := range compose.Services {
+			if s.Name == serviceName {
+				service = s
+				serviceExists = true
+			}
+		}
+
+		if !serviceExists {
+			continue
+		}
+
+		discovered := false
+		for oldLabel, value := range service.Deploy.Labels {
+			if strings.HasPrefix(oldLabel, "coop-cloud") && strings.Contains(oldLabel, "version") {
+				discovered = true
+
+				bytes, err := ioutil.ReadFile(composeFile)
+				if err != nil {
+					return err
+				}
+
+				old := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", value)
+				replacedBytes := strings.Replace(string(bytes), old, label, -1)
+
+				if old == label {
+					logrus.Warnf("%s is already set, nothing to do?", label)
+					return nil
+				}
+
+				logrus.Debugf("updating %s to %s in %s", old, label, compose.Filename)
+
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0764); err != nil {
+					return err
+				}
+
+				logrus.Infof("synced label %s to service %s", label, serviceName)
+			}
+		}
+
+		if !discovered {
+			logrus.Warn("no existing label found, automagic insertion not supported yet")
+			logrus.Fatalf("add '- \"%s\"' manually to the 'app' service in %s", label, composeFile)
+		}
+	}
+
+	return nil
+}

pkg/config/app.go

@@ -0,0 +1,573 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/schollz/progressbar/v3"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+)
+
+// Type aliases to make code hints easier to understand
+
+// AppEnv is a map of the values in an apps env config
+type AppEnv = map[string]string
+
+// AppName is AppName
+type AppName = string
+
+// AppFile represents app env files on disk without reading the contents
+type AppFile struct {
+	Path   string
+	Server string
+}
+
+// AppFiles is a slice of appfiles
+type AppFiles map[AppName]AppFile
+
+// App reprents an app with its env file read into memory
+type App struct {
+	Name   AppName
+	Recipe string
+	Domain string
+	Env    AppEnv
+	Server string
+	Path   string
+}
+
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
+func (a App) StackName() string {
+	if _, exists := a.Env["STACK_NAME"]; exists {
+		return a.Env["STACK_NAME"]
+	}
+
+	stackName := SanitiseAppName(a.Name)
+
+	if len(stackName) > 45 {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:45])
+		stackName = stackName[:45]
+	}
+
+	a.Env["STACK_NAME"] = stackName
+
+	return stackName
+}
+
+// Filters retrieves exact app filters for querying the container runtime. Due
+// to upstream issues, filtering works different depending on what you're
+// querying. So, for example, secrets don't work with regex! The caller needs
+// to implement their own validation that the right secrets are matched. In
+// order to handle these cases, we provide the `appendServiceNames` /
+// `exactMatch` modifiers.
+func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) {
+	filters := filters.NewArgs()
+
+	composeFiles, err := GetComposeFiles(a.Recipe, a.Env)
+	if err != nil {
+		return filters, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	compose, err := GetAppComposeConfig(a.Recipe, opts, a.Env)
+	if err != nil {
+		return filters, err
+	}
+
+	for _, service := range compose.Services {
+		var filter string
+
+		if appendServiceNames {
+			if exactMatch {
+				filter = fmt.Sprintf("^%s_%s", a.StackName(), service.Name)
+			} else {
+				filter = fmt.Sprintf("%s_%s", a.StackName(), service.Name)
+			}
+		} else {
+			if exactMatch {
+				filter = fmt.Sprintf("^%s", a.StackName())
+			} else {
+				filter = fmt.Sprintf("%s", a.StackName())
+			}
+		}
+
+		filters.Add("name", filter)
+	}
+
+	return filters, nil
+}
+
+// ByServer sort a slice of Apps
+type ByServer []App
+
+func (a ByServer) Len() int      { return len(a) }
+func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServer) Less(i, j int) bool {
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByServerAndRecipe sort a slice of Apps
+type ByServerAndRecipe []App
+
+func (a ByServerAndRecipe) Len() int      { return len(a) }
+func (a ByServerAndRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServerAndRecipe) Less(i, j int) bool {
+	if a[i].Server == a[j].Server {
+		return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe)
+	}
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByRecipe sort a slice of Apps
+type ByRecipe []App
+
+func (a ByRecipe) Len() int      { return len(a) }
+func (a ByRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByRecipe) Less(i, j int) bool {
+	return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe)
+}
+
+// ByName sort a slice of Apps
+type ByName []App
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
+}
+
+func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
+	env, err := ReadEnv(appFile.Path)
+	if err != nil {
+		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
+	}
+
+	logrus.Debugf("read env %s from %s", env, appFile.Path)
+
+	app, err := newApp(env, name, appFile)
+	if err != nil {
+		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
+	}
+
+	return app, nil
+}
+
+// newApp creates new App object
+func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+	domain := env["DOMAIN"]
+
+	recipe, exists := env["RECIPE"]
+	if !exists {
+		recipe, exists = env["TYPE"]
+		if !exists {
+			return App{}, fmt.Errorf("%s is missing the TYPE env var?", name)
+		}
+	}
+
+	return App{
+		Name:   name,
+		Domain: domain,
+		Recipe: recipe,
+		Env:    env,
+		Server: appFile.Server,
+		Path:   appFile.Path,
+	}, nil
+}
+
+// LoadAppFiles gets all app files for a given set of servers or all servers.
+func LoadAppFiles(servers ...string) (AppFiles, error) {
+	appFiles := make(AppFiles)
+	if len(servers) == 1 {
+		if servers[0] == "" {
+			// Empty servers flag, one string will always be passed
+			var err error
+			servers, err = GetAllFoldersInDirectory(SERVERS_DIR)
+			if err != nil {
+				return appFiles, err
+			}
+		}
+	}
+
+	logrus.Debugf("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", "))
+
+	for _, server := range servers {
+		serverDir := path.Join(SERVERS_DIR, server)
+		files, err := GetAllFilesInDirectory(serverDir)
+		if err != nil {
+			return appFiles, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server)
+		}
+
+		for _, file := range files {
+			appName := strings.TrimSuffix(file.Name(), ".env")
+			appFilePath := path.Join(SERVERS_DIR, server, file.Name())
+			appFiles[appName] = AppFile{
+				Path:   appFilePath,
+				Server: server,
+			}
+		}
+	}
+
+	return appFiles, nil
+}
+
+// GetApp loads an apps settings, reading it from file, in preparation to use
+// it. It should only be used when ready to use the env file to keep IO
+// operations down.
+func GetApp(apps AppFiles, name AppName) (App, error) {
+	appFile, exists := apps[name]
+	if !exists {
+		return App{}, fmt.Errorf("cannot find app with name %s", name)
+	}
+
+	app, err := readAppEnvFile(appFile, name)
+	if err != nil {
+		return App{}, err
+	}
+
+	return app, nil
+}
+
+// GetApps returns a slice of Apps with their env files read from a given
+// slice of AppFiles.
+func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) {
+	var apps []App
+
+	for name := range appFiles {
+		app, err := GetApp(appFiles, name)
+		if err != nil {
+			return nil, err
+		}
+
+		if recipeFilter != "" {
+			if app.Recipe == recipeFilter {
+				apps = append(apps, app)
+			}
+		} else {
+			apps = append(apps, app)
+		}
+	}
+
+	return apps, nil
+}
+
+// GetAppServiceNames retrieves a list of app service names.
+func GetAppServiceNames(appName string) ([]string, error) {
+	var serviceNames []string
+
+	appFiles, err := LoadAppFiles("")
+	if err != nil {
+		return serviceNames, err
+	}
+
+	app, err := GetApp(appFiles, appName)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	composeFiles, err := GetComposeFiles(app.Recipe, app.Env)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	compose, err := GetAppComposeConfig(app.Recipe, opts, app.Env)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	for _, service := range compose.Services {
+		serviceNames = append(serviceNames, service.Name)
+	}
+
+	return serviceNames, nil
+}
+
+// GetAppNames retrieves a list of app names.
+func GetAppNames() ([]string, error) {
+	var appNames []string
+
+	appFiles, err := LoadAppFiles("")
+	if err != nil {
+		return appNames, err
+	}
+
+	apps, err := GetApps(appFiles, "")
+	if err != nil {
+		return appNames, err
+	}
+
+	for _, app := range apps {
+		appNames = append(appNames, app.Name)
+	}
+
+	return appNames, nil
+}
+
+// TemplateAppEnvSample copies the example env file for the app into the users
+// env files.
+func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
+	envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample")
+	envSample, err := ioutil.ReadFile(envSamplePath)
+	if err != nil {
+		return err
+	}
+
+	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
+		return fmt.Errorf("%s already exists?", appEnvPath)
+	}
+
+	err = ioutil.WriteFile(appEnvPath, envSample, 0664)
+	if err != nil {
+		return err
+	}
+
+	read, err := ioutil.ReadFile(appEnvPath)
+	if err != nil {
+		return err
+	}
+
+	newContents := strings.Replace(string(read), recipeName+".example.com", domain, -1)
+
+	err = ioutil.WriteFile(appEnvPath, []byte(newContents), 0)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("copied & templated %s to %s", envSamplePath, appEnvPath)
+
+	return nil
+}
+
+// SanitiseAppName makes a app name usable with Docker by replacing illegal
+// characters.
+func SanitiseAppName(name string) string {
+	return strings.ReplaceAll(name, ".", "_")
+}
+
+// GetAppStatuses queries servers to check the deployment status of given apps.
+func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) {
+	statuses := make(map[string]map[string]string)
+
+	servers := make(map[string]struct{})
+	for _, app := range apps {
+		if _, ok := servers[app.Server]; !ok {
+			servers[app.Server] = struct{}{}
+		}
+	}
+
+	var bar *progressbar.ProgressBar
+	if !MachineReadable {
+		bar = formatter.CreateProgressbar(len(servers), "querying remote servers...")
+	}
+
+	ch := make(chan stack.StackStatus, len(servers))
+	for server := range servers {
+		cl, err := client.New(server)
+		if err != nil {
+			return statuses, err
+		}
+
+		go func(s string) {
+			ch <- stack.GetAllDeployedServices(cl, s)
+			if !MachineReadable {
+				bar.Add(1)
+			}
+		}(server)
+	}
+
+	for range servers {
+		status := <-ch
+		if status.Err != nil {
+			return statuses, status.Err
+		}
+
+		for _, service := range status.Services {
+			result := make(map[string]string)
+			name := service.Spec.Labels[convert.LabelNamespace]
+
+			if _, ok := statuses[name]; !ok {
+				result["status"] = "deployed"
+			}
+
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name)
+			chaos, ok := service.Spec.Labels[labelKey]
+			if ok {
+				result["chaos"] = chaos
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name)
+			if chaosVersion, ok := service.Spec.Labels[labelKey]; ok {
+				result["chaosVersion"] = chaosVersion
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.autoupdate", name)
+			if autoUpdate, ok := service.Spec.Labels[labelKey]; ok {
+				result["autoUpdate"] = autoUpdate
+			} else {
+				result["autoUpdate"] = "false"
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.version", name)
+			if version, ok := service.Spec.Labels[labelKey]; ok {
+				result["version"] = version
+			} else {
+				continue
+			}
+
+			statuses[name] = result
+		}
+	}
+
+	logrus.Debugf("retrieved app statuses: %s", statuses)
+
+	return statuses, nil
+}
+
+// GetComposeFiles gets the list of compose files for an app (or recipe if you
+// don't already have an app) which should be merged into a composetypes.Config
+// while respecting the COMPOSE_FILE env var.
+func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
+	var composeFiles []string
+
+	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
+		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
+		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
+		composeFiles = append(composeFiles, path)
+		return composeFiles, nil
+	}
+
+	composeFileEnvVar := appEnv["COMPOSE_FILE"]
+	envVars := strings.Split(composeFileEnvVar, ":")
+	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
+	for _, file := range strings.Split(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
+		composeFiles = append(composeFiles, path)
+	}
+
+	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
+
+	return composeFiles, nil
+}
+
+// GetAppComposeConfig retrieves a compose specification for a recipe. This
+// specification is the result of a merge of all the compose.**.yml files in
+// the recipe repository.
+func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) {
+	compose, err := loader.LoadComposefile(opts, appEnv)
+	if err != nil {
+		return &composetypes.Config{}, err
+	}
+
+	logrus.Debugf("retrieved %s for %s", compose.Filename, recipe)
+
+	return compose, nil
+}
+
+// ExposeAllEnv exposes all env variables to the app container
+func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("Add the following environment to the app service config of %s:", stackName)
+			for k, v := range appEnv {
+				_, exists := service.Environment[k]
+				if !exists {
+					value := v
+					service.Environment[k] = &value
+					logrus.Debugf("Add Key: %s Value: %s to %s", k, value, stackName)
+				}
+			}
+		}
+	}
+}
+
+// SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container
+// to signal which recipe is connected to the deployed app
+func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName)
+			service.Deploy.Labels[labelKey] = recipe
+		}
+	}
+}
+
+// SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container
+// to signal if the app is deployed in chaos mode
+func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName)
+			service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos)
+		}
+	}
+}
+
+// SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container
+func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			logrus.Debugf("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName)
+			service.Deploy.Labels[labelKey] = chaosVersion
+		}
+	}
+}
+
+// SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the
+// auto update process for this app. The default if this variable is not set is to disable
+// the auto update process.
+func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"]
+			if !exists {
+				enable_auto_update = "false"
+			}
+			logrus.Debugf("set label 'coop-cloud.%s.autoupdate' to %s for %s", stackName, enable_auto_update, stackName)
+			labelKey := fmt.Sprintf("coop-cloud.%s.autoupdate", stackName)
+			service.Deploy.Labels[labelKey] = enable_auto_update
+		}
+	}
+}
+
+// GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files
+func GetLabel(compose *composetypes.Config, stackName string, label string) string {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+			logrus.Debugf("get label '%s'", labelKey)
+			if labelValue, ok := service.Deploy.Labels[labelKey]; ok {
+				return labelValue
+			}
+		}
+	}
+	logrus.Debugf("no %s label found for %s", label, stackName)
+	return ""
+}
+
+// GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value
+func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
+	var timeout = 50 // Default Timeout
+	var err error = nil
+	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
+		logrus.Debugf("timeout label: %s", timeoutLabel)
+		timeout, err = strconv.Atoi(timeoutLabel)
+	}
+	return timeout, err
+}

pkg/config/app_test.go

@@ -0,0 +1,36 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestNewApp(t *testing.T) {
+	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}
+
+func TestReadAppEnvFile(t *testing.T) {
+	app, err := readAppEnvFile(expectedAppFile, appName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}
+
+func TestGetApp(t *testing.T) {
+	app, err := GetApp(expectedAppFiles, appName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}

pkg/config/env.go

@@ -0,0 +1,170 @@
+package config
+
+import (
+	"bufio"
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/Autonomic-Cooperative/godotenv"
+	"github.com/sirupsen/logrus"
+)
+
+// getBaseDir retrieves the Abra base directory.
+func getBaseDir() string {
+	home := os.ExpandEnv("$HOME/.abra")
+	if customAbraDir, exists := os.LookupEnv("ABRA_DIR"); exists && customAbraDir != "" {
+		home = customAbraDir
+	}
+	return home
+}
+
+var ABRA_DIR = getBaseDir()
+var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
+var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
+var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
+var BACKUP_DIR = path.Join(ABRA_DIR, "backups")
+var CATALOGUE_DIR = path.Join(ABRA_DIR, "catalogue")
+var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json")
+var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
+var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
+var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
+
+// GetServers retrieves all servers.
+func GetServers() ([]string, error) {
+	var servers []string
+
+	servers, err := GetAllFoldersInDirectory(SERVERS_DIR)
+	if err != nil {
+		return servers, err
+	}
+
+	logrus.Debugf("retrieved %v servers: %s", len(servers), servers)
+
+	return servers, nil
+}
+
+// ReadEnv loads an app envivornment into a map.
+func ReadEnv(filePath string) (AppEnv, error) {
+	var envFile AppEnv
+
+	envFile, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("read %s from %s", envFile, filePath)
+
+	return envFile, nil
+}
+
+// ReadServerNames retrieves all server names.
+func ReadServerNames() ([]string, error) {
+	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
+
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("read %s from %s", strings.Join(serverNames, ","), SERVERS_DIR)
+
+	return serverNames, nil
+}
+
+// GetAllFilesInDirectory returns filenames of all files in directory
+func GetAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
+	var realFiles []fs.FileInfo
+
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		// Follow any symlinks
+		filePath := path.Join(directory, file.Name())
+
+		if filepath.Ext(strings.TrimSpace(filePath)) != ".env" {
+			continue
+		}
+
+		realPath, err := filepath.EvalSymlinks(filePath)
+		if err != nil {
+			logrus.Warningf("broken symlink in your abra config folders: %s", filePath)
+		} else {
+			realFile, err := os.Stat(realPath)
+			if err != nil {
+				return nil, err
+			}
+			if !realFile.IsDir() {
+				realFiles = append(realFiles, file)
+			}
+		}
+	}
+
+	return realFiles, nil
+}
+
+// GetAllFoldersInDirectory returns both folder and symlink paths
+func GetAllFoldersInDirectory(directory string) ([]string, error) {
+	var folders []string
+
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+	if len(files) == 0 {
+		return nil, fmt.Errorf("directory is empty: %s", directory)
+	}
+
+	for _, file := range files {
+		// Check if file is directory or symlink
+		if file.IsDir() || file.Mode()&fs.ModeSymlink != 0 {
+			filePath := path.Join(directory, file.Name())
+			realDir, err := filepath.EvalSymlinks(filePath)
+			if err != nil {
+				logrus.Warningf("broken symlink in your abra config folders: %s", filePath)
+			} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
+				// path is a directory
+				folders = append(folders, file.Name())
+			}
+		}
+	}
+
+	return folders, nil
+}
+
+// ReadAbraShEnvVars reads env vars from an abra.sh recipe file.
+func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
+	envVars := make(map[string]string)
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return envVars, nil
+		}
+		return envVars, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "export") {
+			splitVals := strings.Split(line, "export ")
+			envVarDef := splitVals[len(splitVals)-1]
+			keyVal := strings.Split(envVarDef, "=")
+			if len(keyVal) != 2 {
+				return envVars, fmt.Errorf("couldn't parse %s", line)
+			}
+			envVars[keyVal[0]] = keyVal[1]
+		}
+	}
+
+	logrus.Debugf("read %s from %s", envVars, abraSh)
+
+	return envVars, nil
+}

pkg/config/env_test.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"os"
+	"path"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+
+// make sure these are in alphabetical order
+var tFolders = []string{"folder1", "folder2"}
+var tFiles = []string{"bar.env", "foo.env"}
+
+var appName = "ecloud"
+var serverName = "evil.corp"
+
+var expectedAppEnv = AppEnv{
+	"DOMAIN": "ecloud.evil.corp",
+	"RECIPE": "ecloud",
+}
+
+var expectedApp = App{
+	Name:   appName,
+	Recipe: expectedAppEnv["RECIPE"],
+	Domain: expectedAppEnv["DOMAIN"],
+	Env:    expectedAppEnv,
+	Path:   expectedAppFile.Path,
+	Server: expectedAppFile.Server,
+}
+
+var expectedAppFile = AppFile{
+	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
+	Server: serverName,
+}
+
+var expectedAppFiles = map[string]AppFile{
+	appName: expectedAppFile,
+}
+
+// var expectedServerNames = []string{"evil.corp"}
+
+func TestGetAllFoldersInDirectory(t *testing.T) {
+	folders, err := GetAllFoldersInDirectory(testFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(folders, tFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	}
+}
+
+func TestGetAllFilesInDirectory(t *testing.T) {
+	files, err := GetAllFilesInDirectory(testFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fileNames []string
+	for _, file := range files {
+		fileNames = append(fileNames, file.Name())
+	}
+	if !reflect.DeepEqual(fileNames, tFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	}
+}
+
+func TestReadEnv(t *testing.T) {
+	env, err := ReadEnv(expectedAppFile.Path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(env, expectedAppEnv) {
+		t.Fatalf(
+			"did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s",
+			expectedAppEnv["DOMAIN"],
+			expectedAppEnv["RECIPE"],
+			env["DOMAIN"],
+			env["RECIPE"],
+		)
+	}
+}

pkg/container/container.go

@@ -0,0 +1,70 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// GetContainer retrieves a container. If noInput is false and the retrievd
+// count of containers does not match 1, then a prompt is presented to let the
+// user choose. A count of 0 is handled gracefully.
+func GetContainer(c context.Context, cl *client.Client, filters filters.Args, noInput bool) (types.Container, error) {
+	containerOpts := types.ContainerListOptions{Filters: filters}
+	containers, err := cl.ContainerList(c, containerOpts)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	if len(containers) == 0 {
+		filter := filters.Get("name")[0]
+		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
+	}
+
+	if len(containers) != 1 {
+		var containersRaw []string
+		for _, container := range containers {
+			containerName := strings.Join(container.Names, " ")
+			trimmed := strings.TrimPrefix(containerName, "/")
+			created := formatter.HumanDuration(container.Created)
+			containersRaw = append(containersRaw, fmt.Sprintf("%s (created %v)", trimmed, created))
+		}
+
+		if noInput {
+			err := fmt.Errorf("expected 1 container but found %v: %s", len(containers), strings.Join(containersRaw, " "))
+			return types.Container{}, err
+		}
+
+		logrus.Warnf("ambiguous container list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which container are you looking for?",
+			Options: containersRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return types.Container{}, err
+		}
+
+		chosenContainer := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, container := range containers {
+			containerName := strings.TrimSpace(strings.Join(container.Names, " "))
+			trimmed := strings.TrimPrefix(containerName, "/")
+			if trimmed == chosenContainer {
+				return container, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen container")
+	}
+
+	return containers[0], nil
+}

pkg/context/context.go

@@ -0,0 +1,41 @@
+package context
+
+import (
+	"errors"
+
+	"github.com/docker/cli/cli/command"
+	dConfig "github.com/docker/cli/cli/config"
+	"github.com/docker/cli/cli/context"
+	contextStore "github.com/docker/cli/cli/context/store"
+	cliflags "github.com/docker/cli/cli/flags"
+)
+
+func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
+	contextDir := dConfig.ContextStoreDir()
+	storeConfig := command.DefaultContextStoreConfig()
+	store := newContextStore(contextDir, storeConfig)
+
+	opts := &cliflags.ClientOptions{Context: "default"}
+
+	dockerContextStore := &command.ContextStoreWithDefault{
+		Store: store,
+		Resolver: func() (*command.DefaultContext, error) {
+			return command.ResolveDefaultContext(opts, storeConfig)
+		},
+	}
+
+	return dockerContextStore
+}
+
+func GetContextEndpoint(ctx contextStore.Metadata) (string, error) {
+	endpointmeta, ok := ctx.Endpoints["docker"].(context.EndpointMetaBase)
+	if !ok {
+		err := errors.New("context lacks Docker endpoint")
+		return "", err
+	}
+	return endpointmeta.Host, nil
+}
+
+func newContextStore(dir string, config contextStore.Config) contextStore.Store {
+	return contextStore.New(dir, config)
+}

pkg/dns/dns.go

@@ -0,0 +1,50 @@
+package dns
+
+import (
+	"fmt"
+	"net"
+)
+
+// EnsureIPv4 ensures that an ipv4 address is set for a domain name
+func EnsureIPv4(domainName string) (string, error) {
+	ipv4, err := net.ResolveIPAddr("ip", domainName)
+	if err != nil {
+		return "", err
+	}
+
+	return ipv4.String(), nil
+}
+
+// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address
+func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
+	if server == "default" || server == "local" {
+		return "", nil
+	}
+
+	var ipv4 string
+
+	domainIPv4, err := EnsureIPv4(domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if domainIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", domainName)
+	}
+
+	serverIPv4, err := EnsureIPv4(server)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if serverIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", server)
+	}
+
+	if domainIPv4 != serverIPv4 {
+		err := "app domain %s (%s) does not appear to resolve to app server %s (%s)?"
+		return ipv4, fmt.Errorf(err, domainName, domainIPv4, server, serverIPv4)
+	}
+
+	return ipv4, nil
+}

pkg/formatter/formatter.go

@@ -0,0 +1,91 @@
+package formatter
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/docker/go-units"
+	// "github.com/olekukonko/tablewriter"
+	"coopcloud.tech/abra/pkg/jsontable"
+	"github.com/schollz/progressbar/v3"
+	"github.com/sirupsen/logrus"
+)
+
+func ShortenID(str string) string {
+	return str[:12]
+}
+
+func SmallSHA(hash string) string {
+	return hash[:8]
+}
+
+// RemoveSha remove image sha from a string that are added in some docker outputs
+func RemoveSha(str string) string {
+	return strings.Split(str, "@")[0]
+}
+
+// HumanDuration from docker/cli RunningFor() to be accessible outside of the class
+func HumanDuration(timestamp int64) string {
+	date := time.Unix(timestamp, 0)
+	now := time.Now().UTC()
+	return units.HumanDuration(now.Sub(date)) + " ago"
+}
+
+// CreateTable prepares a table layout for output.
+func CreateTable(columns []string) *jsontable.JSONTable {
+	table := jsontable.NewJSONTable(os.Stdout)
+	table.SetAutoWrapText(false)
+	table.SetHeader(columns)
+	return table
+}
+
+// CreateProgressbar generates a progress bar
+func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
+	return progressbar.NewOptions(
+		length,
+		progressbar.OptionClearOnFinish(),
+		progressbar.OptionSetPredictTime(false),
+		progressbar.OptionShowCount(),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetDescription(title),
+	)
+}
+
+// StripTagMeta strips front-matter image tag data that we don't need for parsing.
+func StripTagMeta(image string) string {
+	originalImage := image
+
+	if strings.Contains(image, "docker.io") {
+		image = strings.Split(image, "/")[1]
+	}
+
+	if strings.Contains(image, "library") {
+		image = strings.Split(image, "/")[1]
+	}
+
+	if originalImage != image {
+		logrus.Debugf("stripped %s to %s for parsing", originalImage, image)
+	}
+
+	return image
+}
+
+// ByteCountSI presents a human friendly representation of a byte count. See
+// https://yourbasic.org/golang/formatting-byte-size-to-human-readable-format.
+func ByteCountSI(b uint64) string {
+	const unit = 1000
+
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+
+	div, exp := uint64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
+}

pkg/git/branch.go

@@ -0,0 +1,100 @@
+package git
+
+import (
+	"fmt"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// Check if a branch exists in a repo. Use this and not repository.Branch(),
+// because the latter does not actually check for existing branches. See
+// https://github.com/gogit/gogit/issues/518 for more.
+func HasBranch(repository *git.Repository, name string) bool {
+	var exist bool
+
+	if iter, err := repository.Branches(); err == nil {
+		iterFunc := func(reference *plumbing.Reference) error {
+			if name == reference.Name().Short() {
+				exist = true
+				return nil
+			}
+			return nil
+		}
+		_ = iter.ForEach(iterFunc)
+	}
+
+	return exist
+}
+
+// GetCurrentBranch retrieves the current branch of a repository.
+func GetCurrentBranch(repository *git.Repository) (string, error) {
+	branchRefs, err := repository.Branches()
+	if err != nil {
+		return "", err
+	}
+
+	headRef, err := repository.Head()
+	if err != nil {
+		return "", err
+	}
+
+	var currentBranchName string
+	err = branchRefs.ForEach(func(branchRef *plumbing.Reference) error {
+		if branchRef.Hash() == headRef.Hash() {
+			currentBranchName = branchRef.Name().String()
+
+			return nil
+		}
+
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return currentBranchName, nil
+}
+
+// GetDefaultBranch retrieves the default branch of a repository.
+func GetDefaultBranch(repo *git.Repository, repoPath string) (plumbing.ReferenceName, error) {
+	branch := "master"
+
+	if !HasBranch(repo, "master") {
+		if !HasBranch(repo, "main") {
+			return "", fmt.Errorf("failed to select default branch in %s", repoPath)
+		}
+		branch = "main"
+	}
+
+	return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
+}
+
+// CheckoutDefaultBranch checks out the default branch of a repository.
+func CheckoutDefaultBranch(repo *git.Repository, repoPath string) (plumbing.ReferenceName, error) {
+	branch, err := GetDefaultBranch(repo, repoPath)
+	if err != nil {
+		return plumbing.ReferenceName(""), err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return plumbing.ReferenceName(""), err
+	}
+
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: branch,
+	}
+
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out %s in %s", branch, repoPath)
+		return branch, err
+	}
+
+	logrus.Debugf("successfully checked out %v in %s", branch, repoPath)
+
+	return branch, nil
+}

pkg/git/clone.go

@@ -0,0 +1,50 @@
+package git
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// Clone runs a git clone which accounts for different default branches.
+func Clone(dir, url string) error {
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		logrus.Debugf("%s does not exist, attempting to git clone from %s", dir, url)
+
+		_, err := git.PlainClone(dir, false, &git.CloneOptions{
+			URL:           url,
+			Tags:          git.AllTags,
+			ReferenceName: plumbing.ReferenceName("refs/heads/master"),
+			SingleBranch:  true,
+		})
+		if err != nil {
+			logrus.Debugf("cloning %s default branch failed, attempting from main branch", url)
+
+			_, err := git.PlainClone(dir, false, &git.CloneOptions{
+				URL:           url,
+				Tags:          git.AllTags,
+				ReferenceName: plumbing.ReferenceName("refs/heads/main"),
+				SingleBranch:  true,
+			})
+			if err != nil {
+				if strings.Contains(err.Error(), "authentication required") {
+					name := filepath.Base(dir)
+					return fmt.Errorf("unable to clone %s, does %s exist?", name, url)
+				}
+
+				return err
+			}
+		}
+
+		logrus.Debugf("%s has been git cloned successfully", dir)
+	} else {
+		logrus.Debugf("%s already exists", dir)
+	}
+
+	return nil
+}

pkg/git/commit.go

@@ -0,0 +1,56 @@
+package git
+
+import (
+	"fmt"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// Commit runs a git commit
+func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
+	if commitMessage == "" {
+		return fmt.Errorf("no commit message specified?")
+	}
+
+	commitRepo, err := git.PlainOpen(repoPath)
+	if err != nil {
+		return err
+	}
+
+	commitWorktree, err := commitRepo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return err
+	}
+
+	if len(patterns) > 0 {
+		commitWorktree.Excludes = append(patterns, commitWorktree.Excludes...)
+	}
+
+	if !dryRun {
+		err = commitWorktree.AddGlob(glob)
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("staged %s for commit", glob)
+	} else {
+		logrus.Debugf("dry run: did not stage %s for commit", glob)
+	}
+
+	if !dryRun {
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
+		if err != nil {
+			return err
+		}
+		logrus.Debug("git changes commited")
+	} else {
+		logrus.Debug("dry run: no changes commited")
+	}
+
+	return nil
+}

pkg/git/common.go

@@ -0,0 +1,14 @@
+package git
+
+import (
+	"fmt"
+	"os"
+)
+
+// EnsureGitRepo ensures a git repo .git folder exists
+func EnsureGitRepo(repoPath string) error {
+	if _, err := os.Stat(repoPath); os.IsNotExist(err) {
+		return fmt.Errorf("no .git directory in %s?", repoPath)
+	}
+	return nil
+}

pkg/git/init.go

@@ -0,0 +1,38 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	gitPkg "github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// Init inits a new repo and commits all the stuff if you want
+func Init(repoPath string, commit bool) error {
+	if _, err := gitPkg.PlainInit(repoPath, false); err != nil {
+		logrus.Fatal(err)
+	}
+	logrus.Debugf("initialised new git repo in %s", repoPath)
+
+	if commit {
+		commitRepo, err := git.PlainOpen(repoPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		commitWorktree, err := commitRepo.Worktree()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := commitWorktree.AddWithOptions(&git.AddOptions{All: true}); err != nil {
+			return err
+		}
+
+		if _, err = commitWorktree.Commit("init", &git.CommitOptions{}); err != nil {
+			return err
+		}
+		logrus.Debugf("init committed all files for new git repo in %s", repoPath)
+	}
+
+	return nil
+}

pkg/git/push.go

@@ -0,0 +1,43 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/sirupsen/logrus"
+)
+
+// Push pushes the latest changes & optionally tags to the default remote
+func Push(repoDir string, remote string, tags bool, dryRun bool) error {
+	if dryRun {
+		logrus.Debugf("dry run: no git changes pushed in %s", repoDir)
+		return nil
+	}
+
+	commitRepo, err := git.PlainOpen(repoDir)
+	if err != nil {
+		return err
+	}
+
+	opts := &git.PushOptions{}
+	if remote != "" {
+		opts.RemoteName = remote
+	}
+
+	if err := commitRepo.Push(opts); err != nil {
+		return err
+	}
+
+	logrus.Debugf("git changes pushed")
+
+	if tags {
+		opts.RefSpecs = append(opts.RefSpecs, config.RefSpec("+refs/tags/*:refs/tags/*"))
+
+		if err := commitRepo.Push(opts); err != nil {
+			return err
+		}
+
+		logrus.Debugf("git tags pushed")
+	}
+
+	return nil
+}

pkg/git/read.go

@@ -0,0 +1,187 @@
+package git
+
+import (
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/go-git/go-git/v5"
+	gitConfigPkg "github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
+	"github.com/sirupsen/logrus"
+)
+
+// GetRecipeHead retrieves latest HEAD metadata.
+func GetRecipeHead(recipeName string) (*plumbing.Reference, error) {
+	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return nil, err
+	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return nil, err
+	}
+
+	return head, nil
+}
+
+// IsClean checks if a repo has unstaged changes
+func IsClean(repoPath string) (bool, error) {
+	repo, err := git.PlainOpen(repoPath)
+	if err != nil {
+		return false, err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return false, err
+	}
+
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return false, err
+	}
+
+	if len(patterns) > 0 {
+		worktree.Excludes = append(patterns, worktree.Excludes...)
+	}
+
+	status, err := worktree.Status()
+	if err != nil {
+		return false, err
+	}
+
+	if status.String() != "" {
+		logrus.Debugf("discovered git status in %s: %s", repoPath, status.String())
+	} else {
+		logrus.Debugf("discovered clean git status in %s", repoPath)
+	}
+
+	return status.IsClean(), nil
+}
+
+// GetExcludesFiles reads the exlude files from a global gitignore
+func GetExcludesFiles() ([]gitignore.Pattern, error) {
+	var err error
+	var patterns []gitignore.Pattern
+
+	cfg, err := parseGitConfig()
+	if err != nil {
+		return patterns, err
+	}
+
+	excludesfile := getExcludesFile(cfg)
+	patterns, err = parseExcludesFile(excludesfile)
+	if err != nil {
+		return patterns, err
+	}
+
+	return patterns, nil
+}
+
+func parseGitConfig() (*gitConfigPkg.Config, error) {
+	cfg := gitConfigPkg.NewConfig()
+
+	usr, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	globalGitConfig := filepath.Join(usr.HomeDir, ".gitconfig")
+	if _, err := os.Stat(globalGitConfig); err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no %s exists, not reading any global gitignore config", globalGitConfig)
+			return cfg, nil
+		}
+		return cfg, err
+	}
+
+	b, err := ioutil.ReadFile(globalGitConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cfg.Unmarshal(b); err != nil {
+		return nil, err
+	}
+
+	return cfg, err
+}
+
+func getExcludesFile(cfg *gitConfigPkg.Config) string {
+	for _, sec := range cfg.Raw.Sections {
+		if sec.Name == "core" {
+			for _, opt := range sec.Options {
+				if opt.Key == "excludesfile" {
+					return opt.Value
+				}
+			}
+		}
+	}
+
+	return "~/.gitignore"
+}
+
+func parseExcludesFile(excludesfile string) ([]gitignore.Pattern, error) {
+	var ps []gitignore.Pattern
+
+	excludesfile, err := expandTilde(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := os.Stat(excludesfile); err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no %s exists, skipping reading gitignore paths", excludesfile)
+			return ps, nil
+		}
+		return ps, err
+	}
+
+	data, err := ioutil.ReadFile(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	var pathsRaw []string
+	for _, s := range strings.Split(string(data), "\n") {
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			pathsRaw = append(pathsRaw, s)
+			ps = append(ps, gitignore.ParsePattern(s, nil))
+		}
+	}
+
+	logrus.Debugf("read global ignore paths: %s", strings.Join(pathsRaw, " "))
+
+	return ps, nil
+}
+
+func expandTilde(path string) (string, error) {
+	if !strings.HasPrefix(path, "~") {
+		return path, nil
+	}
+
+	var paths []string
+	u, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	for _, p := range strings.Split(path, string(filepath.Separator)) {
+		if p == "~" {
+			paths = append(paths, u.HomeDir)
+		} else {
+			paths = append(paths, p)
+		}
+	}
+
+	return filepath.Join(paths...), nil
+}

pkg/git/remote.go

@@ -0,0 +1,28 @@
+package git
+
+import (
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/sirupsen/logrus"
+)
+
+// CreateRemote creates a new git remote in a repository
+func CreateRemote(repo *git.Repository, name, url string, dryRun bool) error {
+	if dryRun {
+		logrus.Debugf("dry run: remote %s (%s) not created", name, url)
+		return nil
+	}
+
+	if _, err := repo.CreateRemote(&config.RemoteConfig{
+		Name: name,
+		URLs: []string{url},
+	}); err != nil {
+		if !strings.Contains(err.Error(), "remote already exists") {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/integration/integration.go

@@ -0,0 +1,12 @@
+package integration
+
+import (
+	"os"
+	"testing"
+)
+
+func skipIfNotIntegration(t *testing.T) {
+	if os.Getenv("ABRA_INTEGRATION") == "" {
+		t.Skip("missing 'ABRA_INTEGRATION', not running integration tests")
+	}
+}

pkg/jsontable/jsontable.go

@@ -0,0 +1,209 @@
+package jsontable
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+// A quick-and-dirty proxy/emulator of tablewriter to enable more easy machine readable output
+// - Does not strictly support types, just quoted or unquoted values
+// - Does not support nested values.
+// If a datalabel is set with SetDataLabel(true, "..."), that will be used as the key for teh data of the table,
+// otherwise if the caption is set with SetCaption(true, "..."), the data label will be set to the default of
+// "rows", otherwise the table will output as a JSON list.
+//
+// Proxys all actions through to the tablewriter except addrow and addbatch, which it does at render time
+//
+
+type JSONTable struct {
+	out             io.Writer
+	colsize         int
+	rows            [][]string
+	keys            []string
+	quoted          []bool // hack to do output typing, quoted vs. unquoted
+	hasDataLabel    bool
+	dataLabel       string
+	hasCaption      bool
+	caption         string // the actual caption
+	hasCaptionLabel bool
+	captionLabel    string // the key in the dictionary for the caption
+	tbl             *tablewriter.Table
+}
+
+func writeChar(w io.Writer, c byte) {
+	w.Write([]byte{c})
+
+}
+
+func NewJSONTable(writer io.Writer) *JSONTable {
+	t := &JSONTable{
+		out:             writer,
+		colsize:         0,
+		rows:            [][]string{},
+		keys:            []string{},
+		quoted:          []bool{},
+		hasDataLabel:    false,
+		dataLabel:       "rows",
+		hasCaption:      false,
+		caption:         "",
+		hasCaptionLabel: false,
+		captionLabel:    "caption",
+		tbl:             tablewriter.NewWriter(writer),
+	}
+	return t
+}
+
+func (t *JSONTable) NumLines() int {
+	// JSON only but reflects a shared state.
+	return len(t.rows)
+}
+
+func (t *JSONTable) SetHeader(keys []string) {
+	// Set the keys value which will assign each column to the keys.
+	// Note that we'll ignore values that are beyond the length of the keys list
+	t.colsize = len(keys)
+	t.keys = []string{}
+	for _, k := range keys {
+		t.keys = append(t.keys, k)
+		t.quoted = append(t.quoted, true)
+	}
+	t.tbl.SetHeader(keys)
+}
+
+func (t *JSONTable) SetColumnQuoting(quoting []bool) {
+	// Specify which columns are quoted or unquoted in output
+	// JSON only
+	for i := 0; i < t.colsize; i++ {
+		t.quoted[i] = quoting[i]
+	}
+}
+
+func (t *JSONTable) Append(row []string) {
+	// We'll just append whatever to the rows list. If they fix the keys after appending rows, it'll work as
+	// expected.
+	// We should detect if the row is narrower than the key list tho.
+	// JSON only (but we use the rows later when rendering a regular table)
+	t.rows = append(t.rows, row)
+}
+
+func (t *JSONTable) Render() {
+	// Load the table with rows and render.
+	// Proxy only
+	for _, row := range t.rows {
+		t.tbl.Append(row)
+	}
+
+	t.tbl.Render()
+}
+
+func (t *JSONTable) _JSONRenderInner() {
+	// JSON only
+	// Render the list of dictionaries to the writer.
+	//// inner render loop
+	writeChar(t.out, '[')
+	for rowidx, row := range t.rows {
+		if rowidx != 0 {
+			writeChar(t.out, ',')
+		}
+		writeChar(t.out, '{')
+		for keyidx, key := range t.keys {
+			value := "nil"
+			if keyidx < len(row) {
+				value = row[keyidx]
+			}
+			if keyidx != 0 {
+				writeChar(t.out, ',')
+			}
+			if t.quoted[keyidx] {
+				fmt.Fprintf(t.out, "\"%s\":\"%s\"", key, value)
+			} else {
+				fmt.Fprintf(t.out, "\"%s\":%s", key, value)
+			}
+		}
+		writeChar(t.out, '}')
+	}
+	writeChar(t.out, ']')
+
+}
+
+func (t *JSONTable) JSONRender() {
+	// write JSON table to output
+	// JSON only
+
+	if t.hasDataLabel || t.hasCaption {
+		// dict mode
+		writeChar(t.out, '{')
+
+		if t.hasCaption {
+			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
+
+		}
+		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
+
+	}
+
+	// write list
+	t._JSONRenderInner()
+
+	if t.hasDataLabel || t.hasCaption {
+		// dict mode
+		writeChar(t.out, '}')
+	}
+
+}
+
+func (t *JSONTable) SetCaption(caption bool, captionText ...string) {
+	t.hasCaption = caption
+	if len(captionText) == 1 {
+		t.caption = captionText[0]
+	}
+	t.tbl.SetCaption(caption, captionText...)
+}
+
+func (t *JSONTable) SetCaptionLabel(captionLabel bool, captionLabelText ...string) {
+	// JSON only
+	t.hasCaptionLabel = captionLabel
+	if len(captionLabelText) == 1 {
+		t.captionLabel = captionLabelText[0]
+	}
+}
+
+func (t *JSONTable) SetDataLabel(dataLabel bool, dataLabelText ...string) {
+	// JSON only
+	t.hasDataLabel = dataLabel
+	if len(dataLabelText) == 1 {
+		t.dataLabel = dataLabelText[0]
+	}
+}
+
+func (t *JSONTable) AppendBulk(rows [][]string) {
+	// JSON only but reflects shared state
+	for _, row := range rows {
+		t.Append(row)
+	}
+}
+
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAutoMergeCellsByColumnIndex(cols []int) {
+	// FIXME
+	t.tbl.SetAutoMergeCellsByColumnIndex(cols)
+}
+
+// Stuff we should implement but we just proxy for now.
+func (t *JSONTable) SetAlignment(align int) {
+	// FIXME
+	t.tbl.SetAlignment(align)
+}
+
+func (t *JSONTable) SetAutoMergeCells(auto bool) {
+	// FIXME
+	t.tbl.SetAutoMergeCells(auto)
+}
+
+// Stub functions
+func (t *JSONTable) SetAutoWrapText(auto bool) {
+	t.tbl.SetAutoWrapText(auto)
+	return
+}

pkg/jsontable/jsontable_test.go

@@ -0,0 +1,83 @@
+package jsontable
+
+import (
+	"testing"
+
+	"bytes"
+	"encoding/json"
+
+	"github.com/olekukonko/tablewriter"
+)
+
+var TestLine = []string{"1", "2"}
+var TestGroup = [][]string{{"1", "2", "3"}, {"a", "teohunteohu", "c", "d"}, {"☺", "☹"}}
+var TestKeys = []string{"key0", "key1", "key2"}
+
+// test creation
+func TestNewTable(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+	if tbl.NumLines() != 0 {
+		t.Fatalf("Something went weird when making table (should have 0 lines)")
+	}
+}
+
+// test adding things
+func TestTableAdd(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+
+	tbl.Append(TestLine)
+	if tbl.NumLines() != 1 {
+		t.Fatalf("Appending a line does not result in a length of 1.")
+	}
+
+	tbl.AppendBulk(TestGroup)
+	numlines := tbl.NumLines()
+	if numlines != (len(TestGroup) + 1) {
+		t.Fatalf("Appending two lines does not result in a length of 4 (length is %d).", numlines)
+	}
+}
+
+// test JSON output is parsable
+func TestJsonParsable(t *testing.T) {
+	var b bytes.Buffer
+	tbl := NewJSONTable(&b)
+
+	tbl.AppendBulk(TestGroup)
+	tbl.SetHeader(TestKeys)
+
+	tbl.JSONRender()
+
+	var son []map[string]interface{}
+
+	err := json.Unmarshal(b.Bytes(), &son)
+
+	if err != nil {
+		t.Fatalf("Did not produce parsable JSON: %s", err.Error())
+	}
+}
+
+// test identical commands to a tablewriter and jsontable produce the same rendered output
+func TestTableWriter(t *testing.T) {
+	var bjson bytes.Buffer
+	var btable bytes.Buffer
+
+	tbl := NewJSONTable(&bjson)
+
+	tbl.AppendBulk(TestGroup)
+	tbl.SetHeader(TestKeys)
+	tbl.Render()
+
+	wtbl := tablewriter.NewWriter(&btable)
+
+	wtbl.AppendBulk(TestGroup)
+	wtbl.SetHeader(TestKeys)
+	wtbl.Render()
+
+	if bytes.Compare(bjson.Bytes(), btable.Bytes()) != 0 {
+		t.Fatalf("JSON table and TableWriter produce non-identical outputs.\n%s\n%s", bjson.Bytes(), btable.Bytes())
+	}
+}
+
+/// FIXME test different output formats when captions etc. are added

pkg/limit/limit.go

@@ -0,0 +1,20 @@
+package limit // https://github.com/tidwall/limiter
+
+// Limiter is for limiting the number of concurrent operations. This
+type Limiter struct{ sem chan struct{} }
+
+// New returns a new Limiter. The limit param is the maximum number of
+// concurrent operations.
+func New(limit int) *Limiter {
+	return &Limiter{make(chan struct{}, limit)}
+}
+
+// Begin an operation.
+func (l *Limiter) Begin() {
+	l.sem <- struct{}{}
+}
+
+// End the operation.
+func (l *Limiter) End() {
+	<-l.sem
+}

pkg/lint/recipe.go

@@ -0,0 +1,433 @@
+package lint
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/docker/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+var Warn = "warn"
+var Critical = "critical"
+
+type LintFunction func(recipe.Recipe) (bool, error)
+
+// SkipFunction determines whether the LintFunction is run or not. It should
+// not take the lint rule level into account because some rules are always an
+// error but may depend on some additional context of the recipe configuration.
+// This function aims to cover those additional cases.
+type SkipFunction func(recipe.Recipe) (bool, error)
+
+// LintRule is a linting rule which helps a recipe maintainer avoid common
+// problems in their recipe configurations. We aim to highlight things that
+// might result in critical errors or hours lost in debugging obscure
+// Docker-isms. Humans make the final call on these rules, please raise an
+// issue if you disagree.
+type LintRule struct {
+	Ref           string       // Reference of the linting rule
+	Level         string       // Level of the warning
+	Description   string       // Description of the issue
+	HowToResolve  string       // Documentation for recipe maintainer
+	Function      LintFunction // Rule implementation
+	SkipCondition SkipFunction // Whether or not to execute the lint rule
+}
+
+// Skip implements the SkipFunction for the lint rule.
+func (l LintRule) Skip(recipe recipe.Recipe) bool {
+	if l.SkipCondition != nil {
+		ok, err := l.SkipCondition(recipe)
+		if err != nil {
+			logrus.Debugf("%s: skip condition: %s", l.Ref, err)
+		}
+		if ok {
+			logrus.Debugf("skipping %s based on skip condition", l.Ref)
+			return true
+		}
+	}
+
+	return false
+}
+
+var LintRules = map[string][]LintRule{
+	"warn": {
+		{
+			Ref:          "R001",
+			Level:        "warn",
+			Description:  "compose config has expected version",
+			HowToResolve: "ensure 'version: \"3.8\"' in compose configs",
+			Function:     LintComposeVersion,
+		},
+		{
+			Ref:          "R002",
+			Level:        "warn",
+			Description:  "healthcheck enabled for all services",
+			HowToResolve: "wire up healthchecks",
+			Function:     LintHealthchecks,
+		},
+		{
+			Ref:          "R003",
+			Level:        "warn",
+			Description:  "all images use a tag",
+			HowToResolve: "use a tag for all images",
+			Function:     LintAllImagesTagged,
+		},
+		{
+			Ref:          "R004",
+			Level:        "warn",
+			Description:  "no unstable tags",
+			HowToResolve: "tag all images with stable tags",
+			Function:     LintNoUnstableTags,
+		},
+		{
+			Ref:          "R005",
+			Level:        "warn",
+			Description:  "tags use semver-like format",
+			HowToResolve: "use semver-like tags",
+			Function:     LintSemverLikeTags,
+		},
+		{
+			Ref:          "R006",
+			Level:        "warn",
+			Description:  "has published catalogue version",
+			HowToResolve: "publish a recipe version to the catalogue",
+			Function:     LintHasPublishedVersion,
+		},
+		{
+			Ref:          "R007",
+			Level:        "warn",
+			Description:  "README.md metadata filled in",
+			HowToResolve: "fill out all the metadata",
+			Function:     LintMetadataFilledIn,
+		},
+		{
+			Ref:          "R013",
+			Level:        "warn",
+			Description:  "git.coopcloud.tech repo exists",
+			HowToResolve: "upload your recipe to git.coopcloud.tech/coop-cloud/...",
+			Function:     LintHasRecipeRepo,
+		},
+	},
+	"error": {
+		{
+			Ref:          "R008",
+			Level:        "error",
+			Description:  ".env.sample provided",
+			HowToResolve: "create an example .env.sample",
+			Function:     LintEnvConfigPresent,
+		},
+		{
+			Ref:          "R009",
+			Level:        "error",
+			Description:  "one service named 'app'",
+			HowToResolve: "name a servce 'app'",
+			Function:     LintAppService,
+		},
+		{
+			Ref:           "R010",
+			Level:         "error",
+			Description:   "traefik routing enabled",
+			HowToResolve:  "include \"traefik.enable=true\" deploy label",
+			Function:      LintTraefikEnabled,
+			SkipCondition: LintTraefikEnabledSkipCondition,
+		},
+		{
+			Ref:          "R011",
+			Level:        "error",
+			Description:  "all services have images",
+			HowToResolve: "ensure \"image: ...\" set on all services",
+			Function:     LintImagePresent,
+		},
+		{
+			Ref:          "R012",
+			Level:        "error",
+			Description:  "config version are vendored",
+			HowToResolve: "vendor config versions in an abra.sh",
+			Function:     LintAbraShVendors,
+		},
+		{
+			Ref:          "R014",
+			Level:        "error",
+			Description:  "only annotated tags used for recipe version",
+			HowToResolve: "replace lightweight tag with annotated tag",
+			Function:     LintValidTags,
+		},
+	},
+}
+
+// LintForErrors lints specifically for errors and not other levels. This is
+// used in code paths such as "app deploy" to avoid nasty surprises but not for
+// the typical linting commands, which do handle other levels.
+func LintForErrors(recipe recipe.Recipe) error {
+	logrus.Debugf("linting for critical errors in %s configs", recipe.Name)
+
+	for level := range LintRules {
+		if level != "error" {
+			continue
+		}
+
+		for _, rule := range LintRules[level] {
+			if rule.Skip(recipe) {
+				continue
+			}
+
+			ok, err := rule.Function(recipe)
+			if err != nil {
+				return err
+			}
+			if !ok {
+				return fmt.Errorf("lint error in %s configs: \"%s\" failed lint checks (%s)", recipe.Name, rule.Description, rule.Ref)
+			}
+		}
+	}
+
+	logrus.Debugf("linting successful, %s is well configured", recipe.Name)
+
+	return nil
+}
+
+func LintComposeVersion(recipe recipe.Recipe) (bool, error) {
+	if recipe.Config.Version == "3.8" {
+		return true, nil
+	}
+
+	return true, nil
+}
+
+func LintEnvConfigPresent(recipe recipe.Recipe) (bool, error) {
+	envSample := fmt.Sprintf("%s/%s/.env.sample", config.RECIPES_DIR, recipe.Name)
+	if _, err := os.Stat(envSample); !os.IsNotExist(err) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func LintAppService(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if service.Name == "app" {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// LintTraefikEnabledSkipCondition signals a skip for this linting rule if it
+// confirms that there is no "DOMAIN=..." in the .env.sample configuration of
+// the recipe. This typically means that no domain is required to deploy and
+// therefore no matching traefik deploy label will be present.
+func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	sampleEnv, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
+	}
+
+	if _, ok := sampleEnv["DOMAIN"]; !ok {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func LintTraefikEnabled(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		for label := range service.Deploy.Labels {
+			if label == "traefik.enable" {
+				if service.Deploy.Labels[label] == "true" {
+					return true, nil
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
+func LintHealthchecks(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if service.HealthCheck == nil {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintAllImagesTagged(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return false, err
+		}
+		if reference.IsNameOnly(img) {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintNoUnstableTags(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return false, err
+		}
+
+		var tag string
+		switch img.(type) {
+		case reference.NamedTagged:
+			tag = img.(reference.NamedTagged).Tag()
+		case reference.Named:
+			return false, nil
+		}
+
+		if tag == "latest" {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintSemverLikeTags(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return false, err
+		}
+
+		var tag string
+		switch img.(type) {
+		case reference.NamedTagged:
+			tag = img.(reference.NamedTagged).Tag()
+		case reference.Named:
+			return false, nil
+		}
+
+		if !tagcmp.IsParsable(tag) {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func LintImagePresent(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if service.Image == "" {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
+	catl, err := recipePkg.ReadRecipeCatalogue(false)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	versions, err := recipePkg.GetRecipeCatalogueVersions(recipe.Name, catl)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if len(versions) == 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func LintMetadataFilledIn(r recipe.Recipe) (bool, error) {
+	features, category, err := recipe.GetRecipeFeaturesAndCategory(r.Name)
+	if err != nil {
+		return false, err
+	}
+
+	if category == "" {
+		return false, nil
+	}
+
+	if features.Backups == "" ||
+		features.Email == "" ||
+		features.Healthcheck == "" ||
+		features.Image.Image == "" ||
+		features.SSO == "" {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func LintAbraShVendors(recipe recipe.Recipe) (bool, error) {
+	for _, service := range recipe.Config.Services {
+		if len(service.Configs) > 0 {
+			abraSh := path.Join(config.RECIPES_DIR, recipe.Name, "abra.sh")
+			if _, err := os.Stat(abraSh); err != nil {
+				if os.IsNotExist(err) {
+					return false, err
+				}
+				return false, err
+			}
+		}
+	}
+	return true, nil
+}
+
+func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
+	url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe.Name)
+
+	res, err := http.Get(url)
+	if err != nil {
+		return false, err
+	}
+
+	if res.StatusCode != 200 {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func LintValidTags(recipe recipe.Recipe) (bool, error) {
+	recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return false, fmt.Errorf("unable to open %s: %s", recipeDir, err)
+	}
+
+	iter, err := repo.Tags()
+	if err != nil {
+		logrus.Fatalf("unable to list local tags for %s", recipe.Name)
+	}
+
+	if err := iter.ForEach(func(ref *plumbing.Reference) error {
+		_, err := repo.TagObject(ref.Hash())
+		if err != nil {
+			switch err {
+			case plumbing.ErrObjectNotFound:
+				return fmt.Errorf("invalid lightweight tag detected")
+			default:
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}