fix: use x-platform code for pdeathsig

This might cause the macosx build not to fail, I hope.

See https://github.com/docker/cli/tree/v20.10.10/cli/connhelper/commandconn

.drone.yml

@@ -1,49 +1,74 @@
 ---
 kind: pipeline
-name: linters
+name: coopcloud.tech/abra
 steps:
-  - name: run shellcheck
-    image: koalaman/shellcheck-alpine:v0.7.1
+  - name: make check
+    image: golang:1.17
     commands:
-      - shellcheck abra
-      - shellcheck bin/*.sh
+      - make check
 
-  - name: run flake8
-    image: alpine/flake8:3.9.0
+  - name: make static
+    image: golang:1.17
+    ignore: true # until we decide we all want this check
+    environment:
+      STATIC_CHECK_URL: honnef.co/go/tools/cmd/staticcheck
+      STATIC_CHECK_VERSION: v0.2.0
     commands:
-      - flake8 --max-line-length 100 bin/app-json.py
+      - go install $STATIC_CHECK_URL@$STATIC_CHECK_VERSION
+      - make static
 
-  - name: run unit tests
-    image: decentral1se/docker-dind-bats-kcov
+  - name: make build
+    image: golang:1.17
     commands:
-      - bats tests
+      - make build
 
-  - name: collect code coverage
-    failure: ignore # until we fix this
-    image: decentral1se/docker-dind-bats-kcov
+  - name: make test
+    image: golang:1.17
     commands:
-      - kcov . bats tests || true
+      - make test
 
-  - name: send code coverage report to codecov
-    failure: ignore # until we fix this
-    image: plugins/codecov
+  - name: notify on failure
+    image: plugins/matrix
     settings:
-      token:
-        from_secret: codecov_token
-        required: true
-
-  - name: notify rocket chat
-    image: plugins/slack
-    settings:
-      webhook:
-        from_secret: rc_builds_url
-      username: comradebritney
-      channel: "internal.builds"
-      template: "{{repo.owner}}/{{repo.name}} build failed: {{build.link}}"
+      homeserver: https://matrix.autonomic.zone
+      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
+      userid: "@autono-bot:autonomic.zone"
+      accesstoken:
+        from_secret: autono_bot_access_token
+    depends_on:
+      - make check
+      - make build
+      - make test
     when:
       status:
         - failure
 
-trigger:
-  branch:
-    - main
+  - name: fetch
+    image: docker:git
+    commands:
+      - git fetch --tags
+    depends_on:
+      - make check
+      - make build
+      - make test
+    when:
+      event: tag
+
+  - name: release
+    image: golang:1.17
+    environment:
+      GITEA_TOKEN:
+        from_secret: goreleaser_gitea_token
+    volumes:
+      - name: deps
+        path: /go
+    commands:
+      - curl -sL https://git.io/goreleaser | bash
+    depends_on:
+      - fetch
+    when:
+      event: tag
+
+volumes:
+  - name: deps
+    temp: {}

.envrc.sample

@@ -0,0 +1,6 @@
+go env -w GOPRIVATE=coopcloud.tech
+
+# export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
+# export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
+# export CAPSUL_TOKEN=...
+# export GITEA_TOKEN=...

.gitea/ISSUE_TEMPLATE.md

@@ -0,0 +1,8 @@
+---
+name: "Do not use this issue tracker"
+about: "Do not use this issue tracker"
+title: "Do not use this issue tracker"
+labels: []
+---
+
+Please report your issue on [`coop-cloud/organising`](https://git.coopcloud.tech/coop-cloud/organising)

.gitignore

@@ -1,2 +1,6 @@
-/.venv
-coverage/
+abra
+.vscode/
+vendor/
+.envrc
+dist/
+*fmtcoverage.html

.goreleaser.yml

@@ -0,0 +1,37 @@
+---
+project_name: abra
+gitea_urls:
+  api: https://git.coopcloud.tech/api/v1
+  download: https://git.coopcloud.tech/
+  skip_tls_verify: false
+before:
+  hooks:
+    - go mod tidy
+    - go generate ./...
+builds:
+  - env:
+      - CGO_ENABLED=0
+    dir: cmd/abra
+    goos:
+      - linux
+      - darwin
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+archives:
+  - replacements:
+      386: i386
+      amd64: x86_64
+    format: binary
+checksum:
+  name_template: "checksums.txt"
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: desc
+  filters:
+    exclude:
+      - "^WIP:"
+      - "^style:"
+      - "^test:"
+      - "^tests:"

CHANGELOG.md

@@ -1,125 +0,0 @@
-> 🔥 🔥 🔥 Please note, while we are still in [public
-> alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
-> versioning scheme is not following [semver](https://semver.org/) conventions
-> because we are still in the exploratory phases of building this tool. Please
-> read the changes before upgrading your `abra` installation as there are
-> **most likely** breaking changes coming each release. Sorry for any
-> inconvenience caused, we're working hard to make this tool stable. Semver
-> will be respected when we reach public beta. 🔥 🔥 🔥
-
-# abra x.x.x (UNRELEASED)
-
-# abra 0.7.2 (2021-04-07)
-
-- Fix installation script development installs (again! Thanks Bash!) ([4747d9b7](https://git.autonomic.zone/coop-cloud/abra/commit/4747d9b7fb5fba914f210b6570bfe2db0b53da23))
-
-# abra 0.7.1 (2021-04-07)
-
-- Fix installation script development installs ([8f2fadb3c](https://git.autonomic.zone/coop-cloud/abra/commit/8f2fadb3c43c5915520f5ea531ea3815c2ba8531))
-
-# abra 0.7.0 (2021-04-07)
-
-- Add `--force` to the `deploy` command to allow overriding deployment logic ([#105](https://git.autonomic.zone/coop-cloud/abra/issues/105))
-- Handle undeployed apps in version summaries when deploying ([#104](https://git.autonomic.zone/coop-cloud/abra/issues/104))
-- Add `--force` to `undeploy` command ([e5e98d5](https://git.autonomic.zone/coop-cloud/abra/commit/e5e98d5))
-- Rename "app type" back to "stack" in the deployment overview ([54b6acc](https://git.autonomic.zone/coop-cloud/abra/commit/54b6acc))
-- Show context connection details on `abra server ls` ([#110](https://git.autonomic.zone/coop-cloud/abra/issues/110))
-- Allow to debug the SSH connection details on swarm init ([#109](https://git.autonomic.zone/coop-cloud/abra/issues/109))
-- Show correct status for apps deployed on servers with missing context ([#99](https://git.autonomic.zone/coop-cloud/abra/issues/99))
-- Search for subcommands in descending order of how many components there are ([#108](https://git.autonomic.zone/coop-cloud/abra/issues/108))
-- Add specific app version checking command (`abra app <app> version`) ([#108](https://git.autonomic.zone/coop-cloud/abra/issues/108))
-- Add docker version check (guestimating < v19 is a bad idea) ([#15](https://git.autonomic.zone/coop-cloud/abra/issues/15))
-- Fix git branch handling when not passing `-b <branch>` ([#122](https://git.autonomic.zone/coop-cloud/abra/issues/122))
-- Add work-around to correctly git clone non-master default branch app repositories ([#122](https://git.autonomic.zone/coop-cloud/abra/issues/122))
-- Replace `--force` (except for the `deploy` command) with a global `--no-prompt` for avoiding interactive questions ([#118](https://git.autonomic.zone/coop-cloud/abra/issues/118))
-- Use [docker-stack-wait-deploy](https://github.com/vitalets/docker-stack-wait-deploy) inspired logic to deploy apps ([#116](https://git.autonomic.zone/coop-cloud/abra/issues/116))
-- Add a domain polling check when deploying apps ([#113](https://git.autonomic.zone/coop-cloud/abra/issues/113))
-- Recognise when apps are already undeployed with `abra app <app> undeploy` ([#123](https://git.autonomic.zone/coop-cloud/abra/issues/123))
-- Add `abra doctor` command to help diagnose setup issues ([#119](https://git.autonomic.zone/coop-cloud/abra/issues/119))
-- Add apps version and feature catalogue generation script ([#121](https://git.autonomic.zone/coop-cloud/abra/issues/121))
-- New `--skip-version-check` option to `deploy` ([df4e504](https://git.autonomic.zone/coop-cloud/abra/commit/df4e504)
-- Look up local available version from compose files instead of `abra.sh` ([#131](https://git.autonomic.zone/coop-cloud/abra/issues/131))
-- Improve domain polling logging and allow to skip the check altogether with `--no-domain-poll` ([#140](https://git.autonomic.zone/coop-cloud/abra/issues/140), [#141](https://git.autonomic.zone/coop-cloud/abra/issues/141))
-- Support `ABRA_DIR` in the installer script ([4e94a424e94a42](https://git.autonomic.zone/coop-cloud/abra/commit/4e94a424e94a42))
-- Support [abra-hetzner](https://git.autonomic.zone/coop-cloud/abra-hetzner) plugin ([#88](https://git.autonomic.zone/coop-cloud/abra/issues/88))
-
-# abra 0.6.0 (2021-03-17)
-
-- Show version and digest of app if labelled ([98e674b8e8](https://git.autonomic.zone/coop-cloud/abra/commit/98e674b8e83458a83dcbf331e8e34c7188559c4a))
-- Implement basic version checking on deployment ([#82](https://git.autonomic.zone/coop-cloud/abra/issues/82))
-- New `app-catalogue.sh` script to auto-generate app list for documentation ([f163d4b](https://git.autonomic.zone/coop-cloud/abra/commit/f163d4b0fa920232e9d995a22d20fe78b174b3a9))
-- Support app service rollbacks with `abra <app> rollback <service>` ([#76](https://git.autonomic.zone/coop-cloud/abra/issues/76))
-- Detect when latest version is deployed and perform a no-op ([#87](https://git.autonomic.zone/coop-cloud/abra/issues/87))
-- Allow cloning of app repos with different main branches using `-b, --branch=<branch>` ([#80](https://git.autonomic.zone/coop-cloud/abra/issues/80))
-- Protect against lengthy app names which gives Docker trouble later on ([#83](https://git.autonomic.zone/coop-cloud/abra/issues/83))
-- Support removal of secrets and volumes when `rm`'ing apps ([#44](https://git.autonomic.zone/coop-cloud/abra/issues/44))
-- Always choose the default IPv4 address with `abra server <host> init` ([#91](https://git.autonomic.zone/coop-cloud/abra/issues/91))
-- Add `--type=<type>` filtering option to `abra <app> ls` ([0828189](https://git.autonomic.zone/coop-cloud/abra/commit/0828189))
-- Check for bash 4+ ([#96](https://git.autonomic.zone/coop-cloud/abra/commit/0828189))
-- Add `--dev` option to installer using `git clone` ([88d2a75](https://git.autonomic.zone/coop-cloud/abra/commit/88d2a75))
-- Support `--dev` on the `abra upgrade` command also ([bcc15ec](https://git.autonomic.zone/coop-cloud/abra/commit/bcc15ec))
-- Vendor [yq](https://github.com/mikefarah/yq/releases) automatically ([3b59adf](https://git.autonomic.zone/coop-cloud/abra/commit/3b59adf))
-- Extend version handling logic to support all underlying services ([#90](https://git.autonomic.zone/coop-cloud/abra/issues/90))
-- Fix development installation script symlink issue ([#98](https://git.autonomic.zone/coop-cloud/abra/issues/98))
-- Add `app-version.sh` script to help packagers version apps ([28618bd](https://git.autonomic.zone/coop-cloud/abra/commit/28618bd))
-- Add git digest to `abra version` output ([8b41416](https://git.autonomic.zone/coop-cloud/abra/commit/8b41416))
-
-# abra 0.5.0 (2021-03-01)
-
-- `secret auto` merged into `secret generate` and `app new --auto` is now `app new --secrets` ([#64](https://git.autonomic.zone/coop-cloud/abra/pulls/64))
-- Avoid outputting length during secret generation when not in use ([#67](https://git.autonomic.zone/coop-cloud/abra/issues/67))
-- Support graceful failure when missing secret generation commands ([44d3ac3](https://git.autonomic.zone/coop-cloud/abra/commit/44d3ac3a1cb86edc9b9e91eea1a00e70eae14965))
-- Fix secret detection when using new `.env` file format in apps ([5532452](https://git.autonomic.zone/coop-cloud/abra/commit/55324524ca77141666ffe6cc41b62cc71cf89ace))
-- Support choosing an `$EDITOR` when editing configs ([29cc392](https://git.autonomic.zone/coop-cloud/abra/commit/29cc392dff3e93e48e0e2edd3ce11b405c66a95a))
-- "server" shell completion fixed ([8839bd4](https://git.autonomic.zone/coop-cloud/abra/commit/8839bd45951d00dccf4ef81ece445bcc49e13ee6))
-- Drop `multilogs` command ([#56](https://git.autonomic.zone/coop-cloud/abra/pulls/56))
-- Remove `server use` command ([#51](https://git.autonomic.zone/coop-cloud/abra/issues/51))
-- `new <app>` becomes `new <type>` ([#48](https://git.autonomic.zone/coop-cloud/abra/issues/48))
-- `check` is run on `deploy` now and configurable ([77ba565](https://git.autonomic.zone/coop-cloud/abra/commit/77ba5652b2fe15820f5edfa0f642636f7b8eae7e))
-- App configurations are always updated now ([#42](https://git.autonomic.zone/coop-cloud/abra/issues/42))
-- We use docker format `.env` files (no "export" syntax) from now now ([#55](https://git.autonomic.zone/coop-cloud/abra/pulls/55))
-- Rename `<domain>` option to `<app>` and `APP` variable to `TYPE`, see ([#47](https://git.autonomic.zone/coop-cloud/abra/issues/47))
-- Use Docker-in-Docker (dind), and `dind-bats-kcov` Docker image, for `make test` ([1600b62](https://git.autonomic.zone/coop-cloud/abra/commit/1600b6277fbbffc4c6de1e4ba799c7bbe72ec6a0))
-- Add built-in documentation using `abra help <subcommand>...`, see ([#50](https://git.autonomic.zone/coop-cloud/abra/issues/50))
-- `version` subcommand ([e6b24fe](https://git.autonomic.zone/coop-cloud/abra/commit/e6b24fe))
-- Use `# length=x` comments to generate passwords with `pwgen` and drop `KEY`/`PASSWORD` logic ([#68](https://git.autonomic.zone/coop-cloud/abra/issues/68))
-- Global `--skip-update|-U` / `--skip-check|-C` options to make things quicker ([37e8b00](https://git.autonomic.zone/coop-cloud/abra/commit/37e8b00))
-- `app backup` and `app restore` commands; requires per-app definition ([#70](https://git.autonomic.zone/coop-cloud/abra/issues/70))
-- Rename per-type `abra-commands.sh` to `abra.sh`, and include config versions as type-level instead of app-level config ([#43](https://git.autonomic.zone/coop-cloud/abra/issues/43))
-- Show per-subcommand help by adding `-h/--help` to a command line ([#38](https://git.autonomic.zone/coop-cloud/abra/issues/78))
-
-# abra 0.4.1 (2020-12-24)
-
-- Bug-fixes on `app ls --status` & custom commands
-- Add `app ls --server=...` and alias
-
-# abra 0.4.0 (2020-12-24)
-
-- New command-line interface based on docopt
-- `~/.abra` directory instead of expecting local `.env` files
-- Integration tests & code coverage
-
-# abra 0.3.1 (2020-09-27)
-
-- Fix installer version
-
-# abra 0.3.0 (2020-09-27)
-
-- Add multilogs stack logs implementation ([#8](https://git.autonomic.zone/compose-stacks/abra/issues/8))
-- Add beginnings of "monorepo" functionality
-
-# abra 0.2.0 (2020-09-24)
-
-- Prepare for swarm install script using script.d ([#12](https://git.autonomic.zone/compose-stacks/planning/issues/12))
-
-# abra 0.1.2 (2020-09-22)
-
-- Add upgrade command ([#10](https://git.autonomic.zone/autonomic-cooperative/abra/issues/10))
-
-# abra 0.1.1 (2020-09-22)
-
-- Add installer script ([#9](https://git.autonomic.zone/autonomic-cooperative/abra/issues/9))
-
-# abra 0.1.0 (2020-09-22)
-
-- Initial pre-alpha release

Makefile

@@ -0,0 +1,45 @@
+ABRA    := ./cmd/abra
+COMMIT  := $(shell git rev-list -1 HEAD)
+GOPATH  := $(shell go env GOPATH)
+LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
+DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+export GOPRIVATE=coopcloud.tech
+
+all: run test install build clean format check static
+
+run:
+	@go run -ldflags=$(LDFLAGS) $(ABRA)
+
+install:
+	@go install -ldflags=$(LDFLAGS) $(ABRA)
+
+build-dev:
+	@go build -ldflags=$(LDFLAGS) $(ABRA)
+
+build:
+	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
+
+clean:
+	@rm '$(GOPATH)/bin/abra'
+
+format:
+	@gofmt -s -w .
+
+check:
+	@test -z $$(gofmt -l .) || (echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+
+static:
+	@staticcheck $(ABRA)
+
+test:
+	@go test ./... -cover
+
+loc:
+	@find . -name "*.go" | xargs wc -l
+
+loc-author:
+	@git ls-files -z | \
+		xargs -0rn 1 -P "$$(nproc)" -I{} sh -c 'git blame -w -M -C -C --line-porcelain -- {} | grep -I --line-buffered "^author "' | \
+   	sort -f | \
+   	uniq -ic | \
+   	sort -n

README.md

@@ -1,91 +1,61 @@
 # abra
 
-[![Build Status](https://drone.autonomic.zone/api/badges/coop-cloud/abra/status.svg)](https://drone.autonomic.zone/coop-cloud/abra)
-[![codecov](https://codecov.io/gh/Autonomic-Cooperative/abra/branch/main/graph/badge.svg?token=aX3I5NMRsj)](undefined)
+> https://coopcloud.tech
 
-> https://cloud.autonomic.zone
+[![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
 
-The cooperative cloud utility belt 🎩🐇
+The Co-op Cloud utility belt 🎩🐇
 
-`abra` is a command-line tool for managing your own [Co-op Cloud](https://cloud.autonomic.zone). It can provision new servers, create applications, deploy them, run backup and restore operations and a whole lot of other things. It is the go-to tool for day-to-day operations when managing a Co-op Cloud instance.
+`abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create apps, deploy them, run backup and restore operations and a whole lot of other things. Please see [docs.coopcloud.tech](https://docs.coopcloud.tech) for more extensive documentation.
 
-## Change log
+## Hacking
 
-> 🔥 🔥 🔥 Please note, while we are still in [public
-> alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
-> versioning scheme is not following [semver](https://semver.org/) conventions
-> because we are still in the exploratory phases of building this tool. Please
-> read the changes before upgrading your `abra` installation as there are
-> **most likely** breaking changes coming each release. Sorry for any
-> inconvenience caused, we're working hard to make this tool stable. Semver
-> will be respected when we reach public beta. 🔥 🔥 🔥
+### Getting started
 
-See [CHANGELOG.md](./CHANGELOG.md).
+Install [direnv](https://direnv.net), run `cp .envrc.sample .envrc`, then run `direnv allow` in this directory. This will set coopcloud repos as private due to [this bug.](https://git.coopcloud.tech/coop-cloud/coopcloud.tech/issues/20#issuecomment-8201). Or you can run `go env -w GOPRIVATE=coopcloud.tech` but I'm not sure how persistent this is.
 
-## Documentation
+Install [Go >= 1.16](https://golang.org/doc/install) and then:
 
-> [docs.cloud.autonomic.zone](https://docs.cloud.autonomic.zone/)
+- `make build` to build
+- `./abra` to run commands
+- `make test` will run tests
+- `make install` will install it to `$GOPATH/bin`
+- `go get <package>` and `go mod tidy` to add a new dependency
 
-## Install
+Our [Drone CI configuration](.drone.yml) runs a number of sanity on each pushed commit. See the [Makefile](./Makefile) for more handy targets.
 
-Install the latest stable release:
+Please use the [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/) for your commits so we can automate our change log.
 
-```sh
-curl https://install.abra.autonomic.zone | bash
-```
+### Versioning
 
-or the bleeding-edge development version:
+We use [goreleaser](https://goreleaser.com) to help us automate releases. We use [semver](https://semver.org) for versioning all releases of the tool. While we are still in the public alpha release phase, we will maintain a `0.y.z-alpha` format. Change logs are generated from our commit logs. We are still working this out and aim to refine our release praxis as we go.
 
-```sh
-curl https://install.abra.autonomic.zone | bash -s -- --dev
-```
+For developers, while using this `-alpha` format, the `y` part is the "major" version part. So, if you make breaking changes, you increment that and _not_ the `x` part. So, if you're on `0.1.0-alpha`, then you'd go to `0.1.1-alpha` for a backwards compatible change and `0.2.0-alpha` for a backwards incompatible change.
 
-The source for this script is [here](./deploy/install.abra.autonomic.zone/installer).
+### Making a new release
 
-## Update
+- Change `ABRA_VERSION` to match the new tag in [`scripts`](./scripts/installer/installer) (use [semver](https://semver.org))
+- Commit that change (e.g. `git commit -m 'chore: publish next tag x.y.z-alpha'`)
+- Make a new tag (e.g. `git tag -a x.y.z-alpha`)
+- Push the new tag (e.g. `git push && git push --tags`)
+- Wait until the build finishes on [build.coopcloud.tech](https://build.coopcloud.tech/coop-cloud/abra)
+- Deploy the new installer script (e.g. `cd ./scripts/installer && make`)
+- Check the release worked, (e.g. `abra upgrade; abra -v`)
 
-Run `abra upgrade` to automatically download and install the latest release
-version.
+### Fork maintenance
 
-To update the development version, run `abra upgrade --dev`.
+#### `godotenv`
 
-## Hack
+We maintain a fork of [godotenv](https://github.com/Autonomic-Cooperative/godotenv) for two features:
 
-It's written in Bash version 4 or greater!
+1. multi-line env var support
+2. inline comment parsing
 
-Install it via `curl https://install.abra.autonomic.zone | bash -s -- --dev`, then you can hack on the source in `~/.abra/src`.
+You can upgrade the version here by running `go get github.com/Autonomic-Cooperative/godotenv@<commit>` where `<commit>` is the
+latest commit you want to pin to. We are aiming to migrate to YAML format for the environment configuration, so this should only
+be a temporary thing.
 
-The command-line interface is generated via [docopt](http://docopt.org/). If you add arguments then you need to run `make docopt` ro regenerate the parser.
+#### `docker/client`
 
-Please remember to update the [CHANGELOG](./CHANGELOG.md) when you make a change.
-
-## Generating a new abra-apps.json
-
-You'll need to install the following requirements:
-
-- [requests](https://docs.python-requests.org/en/master/) (`apt install python3-requests` / `pip install requests`)
-- [skopeo](https://github.com/containers/skopeo) (check [the install docs](https://github.com/containers/skopeo/blob/master/install.md))
-- [jq](https://stedolan.github.io/jq/tutorial/) (`sudo apt-get install jq` or see [the install docs](https://stedolan.github.io/jq/download/))
-- [yq](https://mikefarah.gitbook.io/yq/) (see [the install docs](https://mikefarah.gitbook.io/yq/#install))
-
-Then run `./bin/app-json.py` ([source](./bin/app-json.py)) and it will spit out the JSON file into [deploy/abra-apps.cloud.autonomic.zone/abra-apps.json](./deploy/abra-apps.cloud.autonomic.zone/abra-apps.json).
-
-## Releasing
-
-### `abra`
-
-- Change the `x.x.x` header in [CHANGELOG.md](./CHANGELOG.md) to reflect new version and mark date
-- Update the version in [abra](./abra)
-- Update the version in [deploy/install.abra.autonomic.zone/installer](./deploy/install.abra.autonomic.zone/installer)
-- `git commit` the above changes and then tag it with `git tag <your-new-version>`
-- `git push` and `git push --tags`
-- Deploy a new installer script `make release-installer`
-- Tell the world (CoTech forum, Matrix public channel, Autonomic mastodon, etc.)
-
-### abra-apps.cloud.autonomic.zone
-
-> [abra-apps.cloud.autonomic.zone](https://abra-apps.cloud.autonomic.zone)
-
-```bash
-$ make release-abra-apps
-```
+A number of modules in [pkg/upstream](./pkg/upstream) are copy/pasta'd from the upstream [docker/docker/client](https://pkg.go.dev/github.com/docker/docker/client). We had to do this because upstream are not exposing their API as public.

bin/app-catalogue.sh

@@ -1,103 +0,0 @@
-#!/bin/bash
-
-# shellcheck disable=SC2119
-
-# Usage: ./app-catalogue.sh
-#
-# Gather metadata from Co-op Cloud apps in $ABRA_DIR/apps (default
-# ~/.abra/apps), and format it as a Markdown table for this page:
-# https://docs.cloud.autonomic.zone/apps/
-
-stack_dir="${ABRA_DIR:-$HOME/.abra}/apps/"
-
-cd "$stack_dir" || exit
-
-# load all README files into ENV_FILES array
-mapfile -t readmes < <(find -L . -name "README.md")
-# FIXME 3wc: requires bash 4, use for loop instead
-
-base_url="https://git.autonomic.zone/coop-cloud"
-
-cat_apps=()
-cat_development=()
-cat_utilities=()
-cat_graveyard=()
-
-get_var() {
-	echo "$1" | grep "$2" | sed 's/^[^:]*: //'
-}
-
-# shellcheck disable=SC2120
-trim() {
-	# accept input as argument or from STDIN, see here:
-	# https://zwbetz.com/passing-input-to-a-bash-function-via-arguments-or-stdin/
-	# shellcheck disable=SC2155
-	local input="$([[ -p /dev/stdin ]] && cat - || echo "$@")"
-	[[ -z "$input" ]] && return 1
-	echo "$input" | tr -d ' '
-}
-
-# shellcheck disable=SC2120
-prettify() {
-	# as above
-	# shellcheck disable=SC2155
-	local input="$([[ -p /dev/stdin ]] && cat - || echo "$@")"
-	[[ -z "$input" ]] && return 1
-
-	echo "$input" | sed -e 's/Yes/✅/' -e 's/No/❌/' -e 's/N\/A/⛔/'
-}
-
-for readme in "${readmes[@]}"; do
-	type="$(basename "${readme%README.md}")"
-	if [ "$type" = "example" ]; then
-		continue
-	fi
-	title="$(grep '^# ' "$type/README.md" | sed 's/^# //' )"
-	# find section between 'metadata' and 'endmetadata' comments
-	metadata="$(awk '/-- metadata --/,/-- endmetadata --/' "$type/README.md")"
-	status="$(get_var "$metadata" "Status")"
-	category="$(get_var "$metadata" "Category" | cut -d',' -f2 | trim)"
-
-	if [ -z "$category" ]; then
-		echo "ERROR: missing category for $type"
-		continue
-	fi
-
-	image="$(get_var "$metadata" "Image" | cut -d',' -f2 | trim)"
-	healthcheck="$(get_var "$metadata" "Healthcheck" | prettify)"
-	backups="$(get_var "$metadata" "Backups" | prettify)"
-	email="$(get_var "$metadata" "Email" | prettify)"
-	tests="$(get_var "$metadata" "Tests" | prettify)"
-	sso="$(get_var "$metadata" "SSO" | prettify)"
-
-	row="| [$title]($base_url/$type) | $status | $image | $healthcheck | $backups | $email | $tests | $sso |"
-
-	category_lower="$(echo "$category" | tr '[:upper:]' '[:lower:]')"
-	eval "cat_$category_lower+=( '$row' )"
-done
-
-headers="
-| **Name** | **Status** | **Image** | **Healtcheck** | **Backups** | **Email** | **CI** | **Single-Sign-On** |
-| --- | --- | --- | --- | --- | --- | --- | --- |"
-
-echo "## Applications"
-echo "$headers"
-printf '%s\n' "${cat_apps[@]}" | sort
-
-echo
-
-echo "## Developer tools"
-echo "$headers"
-printf '%s\n' "${cat_development[@]}" | sort
-
-echo
-
-echo "## Utilities"
-echo "$headers"
-printf '%s\n' "${cat_utilities[@]}" | sort
-
-echo
-
-echo "## Graveyard"
-echo "$headers"
-printf '%s\n' "${cat_graveyard[@]}" | sort

bin/app-json.py

@@ -1,253 +0,0 @@
-#!/usr/bin/env python3
-
-# Usage: ./app-json.py
-#
-# Gather metadata from Co-op Cloud apps in $ABRA_DIR/apps (default
-# ~/.abra/apps), and format it as JSON so that it can be hosted here:
-# https://abra-apps.cloud.autonomic.zone
-
-from json import dump
-from logging import DEBUG, basicConfig, getLogger
-from os import chdir, listdir, mkdir
-from os.path import basename, exists, expanduser
-from pathlib import Path
-from re import findall, search
-from shlex import split
-from subprocess import check_output, DEVNULL
-from sys import exit
-
-from requests import get
-
-HOME_PATH = expanduser("~/")
-CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
-YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
-SCRIPT_PATH = Path(__file__).absolute().parent
-REPOS_TO_SKIP = (
-    "abra",
-    "backup-bot",
-    "cloud.autonomic.zone",
-    "docs.cloud.autonomic.zone",
-    "example",
-    "organising",
-    "pyabra",
-    "stack-ssh-deploy",
-    "radicle-seed-node",
-    "coturn"
-)
-
-log = getLogger(__name__)
-basicConfig()
-log.setLevel(DEBUG)
-
-
-def _run_cmd(cmd, shell=False, **kwargs):
-    """Run a shell command."""
-    args = [split(cmd)]
-
-    if shell:
-        args = [cmd]
-        kwargs = {"shell": shell}
-
-    try:
-        return check_output(*args, **kwargs).decode("utf-8").strip()
-    except Exception as exception:
-        log.error(f"Failed to run {cmd}, saw {str(exception)}")
-        exit(1)
-
-
-def get_published_apps_json():
-    """Retrieve already published apps json."""
-    url = "https://abra-apps.cloud.autonomic.zone"
-
-    log.info(f"Retrieving {url}")
-
-    try:
-        return get(url, timeout=5).json()
-    except Exception as exception:
-        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
-        return {}
-
-
-def clone_all_apps():
-    """Clone all Co-op Cloud apps to ~/.abra/apps."""
-    if not exists(CLONES_PATH):
-        mkdir(CLONES_PATH)
-    url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
-
-    log.info(f"Retrieving {url}")
-
-    try:
-        response = get(url, timeout=10)
-    except Exception as exception:
-        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
-        exit(1)
-
-    repos = [[p["name"], p["ssh_url"]] for p in response.json()]
-
-    for name, url in repos:
-        if name in REPOS_TO_SKIP:
-            continue
-
-        if not exists(f"{CLONES_PATH}/{name}"):
-            log.info(f"Retrieving {url}")
-            _run_cmd(f"git clone {url} {CLONES_PATH}/{name}")
-
-            chdir(f"{CLONES_PATH}/{name}")
-            if not int(_run_cmd("git branch --list | wc -l", shell=True)):
-                log.info(f"Guessing main branch is HEAD for {name}")
-                _run_cmd("git checkout main")
-        else:
-            log.info(f"Updating {name}")
-            chdir(f"{CLONES_PATH}/{name}")
-            _run_cmd("git fetch -a")
-
-
-def generate_apps_json():
-    """Generate the abra-apps.json application versions file."""
-    apps_json = {}
-    cached_apps_json = get_published_apps_json()
-
-    for app in listdir(CLONES_PATH):
-        if app in REPOS_TO_SKIP:
-            log.info(f"Skipping {app}")
-            continue
-
-        app_path = f"{CLONES_PATH}/{app}"
-        chdir(app_path)
-
-        log.info(f"Processing {app}")
-        apps_json[app] = {
-            "category": "apps",
-            "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
-            # Note(decentral1se): please note that the app features do not
-            # correspond to version tags. We simply parse the latest features
-            # list from HEAD. This may lead to unexpected situations where
-            # users believe X feature is available under Y version but it is
-            # not.
-            "features": get_app_features(app_path),
-            "versions": get_app_versions(app_path, cached_apps_json),
-        }
-
-    return apps_json
-
-
-def get_app_features(app_path):
-    """Parse features from app repo README files."""
-    features = {}
-
-    chdir(app_path)
-
-    with open(f"{app_path}/README.md", "r") as handle:
-        log.info(f"{app_path}/README.md")
-        contents = handle.read()
-
-    try:
-        for match in findall(r"\*\*.*\s\*", contents):
-            title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
-
-            if title == "image":
-                value = {
-                    "image": search(r"(?<=`).*(?=`)", match).group(),
-                    "url": search(r"(?<=\().*(?=\))", match).group(),
-                    "rating": match.split(",")[1].strip(),
-                    "source": match.split(",")[-1].replace("*", "").strip(),
-                }
-            else:
-                value = match.split(":")[-1].replace("*", "").strip()
-
-            features[title] = value
-    except (IndexError, AttributeError):
-        log.info(f"Can't parse {app_path}/README.md")
-        return {}
-    finally:
-        _run_cmd("git checkout HEAD")
-
-    log.info(f"Parsed {features}")
-
-    return features
-
-
-def get_app_versions(app_path, cached_apps_json):
-    versions = {}
-
-    chdir(app_path)
-
-    tags = _run_cmd("git tag --list").split()
-
-    if not tags:
-        log.info("No tags discovered, moving on")
-        return {}
-
-    initial_branch = _run_cmd("git rev-parse --abbrev-ref HEAD")
-
-    app_name = basename(app_path)
-
-    try:
-        existing_tags = cached_apps_json[app_name]["versions"].keys()
-    except KeyError:
-        existing_tags = []
-
-    for tag in tags:
-        _run_cmd(f"git checkout {tag}",
-                 stderr=DEVNULL)
-
-        services_cmd = f"{YQ_PATH} e '.services | keys | .[]' compose*.yml"
-        services = _run_cmd(services_cmd, shell=True).split()
-
-        parsed_services = []
-        service_versions = {}
-        for service in services:
-            if service in ("null", "---"):
-                continue
-
-            if tag in existing_tags:
-                log.info(f"Skipping {tag} because we've already processed it")
-                existing_versions = cached_apps_json[app_name]["versions"][tag][service]
-                service_versions[service] = existing_versions
-                _run_cmd(f"git checkout {initial_branch}")
-                continue
-
-            if service in parsed_services:
-                log.info(f"Skipped {service}, we've already parsed it locally")
-                continue
-
-            services_cmd = f"{YQ_PATH} e '.services.{service}.image' compose*.yml"
-            images = _run_cmd(services_cmd, shell=True).split()
-
-            for image in images:
-                if image in ("null", "---"):
-                    continue
-
-                images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
-                output = _run_cmd(images_cmd, shell=True)
-
-                service_version_info = {
-                    "image": image.split(":")[0],
-                    "tag": image.split(":")[-1],
-                    "digest": output.split(":")[-1][:8],
-                }
-
-                log.info(f"Parsed {service_version_info}")
-                service_versions[service] = service_version_info
-
-            parsed_services.append(service)
-
-        versions[tag] = service_versions
-
-    _run_cmd(f"git checkout {initial_branch}")
-
-    return versions
-
-
-def main():
-    """Run the script."""
-    clone_all_apps()
-
-    target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
-    with open(target, "w", encoding="utf-8") as handle:
-        dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
-
-    log.info(f"Successfully generated {target}")
-
-
-main()

cli/app/app.go

@@ -0,0 +1,38 @@
+package app
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// AppCommand defines the `abra app` command and ets subcommands
+var AppCommand = &cli.Command{
+	Name:      "app",
+	Usage:     "Manage deployed apps",
+	Aliases:   []string{"a"},
+	ArgsUsage: "<app>",
+	Description: `
+This command provides all the functionality you need to manage the life cycle
+of your apps. From initial deployment, day-2 operations (e.g. backup/restore)
+to scaling apps up and spinning them down.
+`,
+	Subcommands: []*cli.Command{
+		appNewCommand,
+		appConfigCommand,
+		appDeployCommand,
+		appUpgradeCommand,
+		appUndeployCommand,
+		appBackupCommand,
+		appRestoreCommand,
+		appRemoveCommand,
+		appCheckCommand,
+		appListCommand,
+		appPsCommand,
+		appLogsCommand,
+		appCpCommand,
+		appRunCommand,
+		appRollbackCommand,
+		appSecretCommand,
+		appVolumeCommand,
+		appVersionCommand,
+	},
+}

cli/app/backup.go

@@ -0,0 +1,87 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var backupAllServices bool
+var backupAllServicesFlag = &cli.BoolFlag{
+	Name:        "all",
+	Value:       false,
+	Destination: &backupAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Backup all services",
+}
+
+var appBackupCommand = &cli.Command{
+	Name:      "backup",
+	Usage:     "Backup an app",
+	Aliases:   []string{"b"},
+	Flags:     []cli.Flag{backupAllServicesFlag},
+	ArgsUsage: "<service>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Get(1) != "" && backupAllServices {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<service>' and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
+			logrus.Fatal(err)
+		}
+
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+
+		execCmd := "abra_backup"
+		if !backupAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_backup_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/check.go

@@ -0,0 +1,64 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appCheckCommand = &cli.Command{
+	Name:      "check",
+	Usage:     "Check if app is configured correctly",
+	Aliases:   []string{"c"},
+	ArgsUsage: "<service>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", app.Type, ".env.sample")
+		if _, err := os.Stat(envSamplePath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", envSamplePath)
+			}
+			logrus.Fatal(err)
+		}
+
+		envSample, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var missing []string
+		for k := range envSample {
+			if _, ok := app.Env[k]; !ok {
+				missing = append(missing, k)
+			}
+		}
+
+		if len(missing) > 0 {
+			missingEnvVars := strings.Join(missing, ", ")
+			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
+		}
+
+		logrus.Infof("all necessary environment variables defined for '%s'", app.Name)
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/config.go

@@ -0,0 +1,70 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appConfigCommand = &cli.Command{
+	Name:    "config",
+	Aliases: []string{"c"},
+	Usage:   "Edit app config",
+	Action: func(c *cli.Context) error {
+		appName := c.Args().First()
+
+		if appName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+		}
+
+		files, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		appFile, exists := files[appName]
+		if !exists {
+			logrus.Fatalf("cannot find app with name '%s'", appName)
+		}
+
+		ed, ok := os.LookupEnv("EDITOR")
+		if !ok {
+			edPrompt := &survey.Select{
+				Message: "Which editor do you wish to use?",
+				Options: []string{"vi", "vim", "nvim", "nano", "pico", "emacs"},
+			}
+			if err := survey.AskOne(edPrompt, &ed); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		cmd := exec.Command(ed, appFile.Path)
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/cp.go

@@ -0,0 +1,124 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appCpCommand = &cli.Command{
+	Name:      "cp",
+	Aliases:   []string{"c"},
+	ArgsUsage: "<src> <dst>",
+	Usage:     "Copy files to/from a running app service",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		src := c.Args().Get(1)
+		dst := c.Args().Get(2)
+		if src == "" {
+			logrus.Fatal("missing <src> argument")
+		} else if dst == "" {
+			logrus.Fatal("missing <dest> argument")
+		}
+
+		parsedSrc := strings.SplitN(src, ":", 2)
+		parsedDst := strings.SplitN(dst, ":", 2)
+		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
+		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
+			logrus.Fatal(errorMsg)
+		} else if len(parsedSrc) != 2 {
+			if len(parsedDst) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		} else if len(parsedDst) != 2 {
+			if len(parsedSrc) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		}
+
+		var service string
+		var srcPath string
+		var dstPath string
+		isToContainer := false // <container:src> <dst>
+		if len(parsedSrc) == 2 {
+			service = parsedSrc[0]
+			srcPath = parsedSrc[1]
+			dstPath = dst
+			logrus.Debugf("assuming transfer is coming FROM the container")
+		} else if len(parsedDst) == 2 {
+			service = parsedDst[0]
+			dstPath = parsedDst[1]
+			srcPath = src
+			isToContainer = true // <src> <container:dst>
+			logrus.Debugf("assuming transfer is going TO the container")
+		}
+
+		appFiles, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		appEnv, err := config.GetApp(appFiles, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
+		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(containers) != 1 {
+			logrus.Fatalf("expected 1 container but got %v", len(containers))
+		}
+		container := containers[0]
+
+		logrus.Debugf("retrieved '%s' as target container on '%s'", formatter.ShortenID(container.ID), app.Server)
+
+		if isToContainer {
+			if _, err := os.Stat(srcPath); err != nil {
+				logrus.Fatalf("'%s' does not exist?", srcPath)
+			}
+
+			toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+			content, err := archive.TarWithOptions(srcPath, toTarOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+			if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer content.Close()
+			fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+			if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+		return nil
+	},
+}

cli/app/deploy.go

@@ -0,0 +1,45 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appDeployCommand = &cli.Command{
+	Name:    "deploy",
+	Aliases: []string{"d"},
+	Usage:   "Deploy an app",
+	Flags: []cli.Flag{
+		internal.ForceFlag,
+		internal.ChaosFlag,
+	},
+	Description: `
+This command deploys a new instance of an app. It does not support changing the
+version of an existing deployed app, for this you need to look at the "abra app
+upgrade <app>" command.
+
+You may pass "--force" to re-deploy the same version again. This can be useful
+if the container runtime has gotten into a weird state.
+
+Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+including unstaged changes and can be useful for live hacking and testing new
+recipes.
+`,
+	Action: internal.DeployAction,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/list.go

@@ -0,0 +1,173 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/tagcmp"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var status bool
+var statusFlag = &cli.BoolFlag{
+	Name:        "status",
+	Aliases:     []string{"S"},
+	Value:       false,
+	Usage:       "Show app deployment status",
+	Destination: &status,
+}
+
+var appType string
+var typeFlag = &cli.StringFlag{
+	Name:        "type",
+	Aliases:     []string{"t"},
+	Value:       "",
+	Usage:       "Show apps of a specific type",
+	Destination: &appType,
+}
+
+var listAppServer string
+var listAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &listAppServer,
+}
+
+var appListCommand = &cli.Command{
+	Name:  "list",
+	Usage: "List all managed apps",
+	Description: `
+This command looks at your local file system listing of apps and servers (e.g.
+in ~/.abra/) to generate a report of all your apps.
+
+By passing the "--status/-S" flag, you can query all your servers for the
+actual live deployment status. Depending on how many servers you manage, this
+can take some time.
+	`,
+	Aliases: []string{"ls"},
+	Flags: []cli.Flag{
+		statusFlag,
+		listAppServerFlag,
+		typeFlag,
+	},
+	Action: func(c *cli.Context) error {
+		appFiles, err := config.LoadAppFiles(listAppServer)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		apps, err := config.GetApps(appFiles)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		sort.Sort(config.ByServerAndType(apps))
+
+		statuses := make(map[string]map[string]string)
+		tableCol := []string{"Server", "Type", "Domain"}
+		if status {
+			tableCol = append(tableCol, "Status", "Version", "Updates")
+			statuses, err = config.GetAppStatuses(appFiles)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		table := abraFormatter.CreateTable(tableCol)
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+
+		var (
+			versionedAppsCount   int
+			unversionedAppsCount int
+			onLatestCount        int
+			canUpgradeCount      int
+		)
+
+		for _, app := range apps {
+			var tableRow []string
+			if app.Type == appType || appType == "" {
+				// If type flag is set, check for it, if not, Type == ""
+				tableRow = []string{app.Server, app.Type, app.Domain}
+				if status {
+					stackName := app.StackName()
+					status := "unknown"
+					version := "unknown"
+					if statusMeta, ok := statuses[stackName]; ok {
+						if currentVersion, exists := statusMeta["version"]; exists {
+							version = currentVersion
+						}
+						if statusMeta["status"] != "" {
+							status = statusMeta["status"]
+						}
+						tableRow = append(tableRow, status, version)
+						versionedAppsCount++
+					} else {
+						tableRow = append(tableRow, status, version)
+						unversionedAppsCount++
+					}
+
+					var newUpdates []string
+					if version != "unknown" {
+						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+
+						parsedVersion, err := tagcmp.Parse(version)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+
+						for _, update := range updates {
+							parsedUpdate, err := tagcmp.Parse(update)
+							if err != nil {
+								logrus.Fatal(err)
+							}
+
+							if update != version && parsedUpdate.IsGreaterThan(parsedVersion) {
+								newUpdates = append(newUpdates, update)
+							}
+						}
+					}
+
+					if len(newUpdates) == 0 {
+						if version == "unknown" {
+							tableRow = append(tableRow, "unknown")
+						} else {
+							tableRow = append(tableRow, "on latest")
+							onLatestCount++
+						}
+					} else {
+						// FIXME: jeezus golang why do you not have a list reverse function
+						for i, j := 0, len(newUpdates)-1; i < j; i, j = i+1, j-1 {
+							newUpdates[i], newUpdates[j] = newUpdates[j], newUpdates[i]
+						}
+						tableRow = append(tableRow, strings.Join(newUpdates, "\n"))
+						canUpgradeCount++
+					}
+				}
+			}
+			table.Append(tableRow)
+		}
+
+		stats := fmt.Sprintf(
+			"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
+			len(apps),
+			versionedAppsCount,
+			unversionedAppsCount,
+			onLatestCount,
+			canUpgradeCount,
+		)
+
+		table.SetCaption(true, stats)
+		table.Render()
+
+		return nil
+	},
+}

cli/app/logs.go

@@ -0,0 +1,124 @@
+package app
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// stackLogs lists logs for all stack services
+func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
+	filters := filters.NewArgs()
+	filters.Add("name", stackName)
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(c.Context, serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			logOpts := types.ContainerLogsOptions{
+				Details:    true,
+				Follow:     true,
+				ShowStderr: true,
+				ShowStdout: true,
+				Tail:       "20",
+				Timestamps: true,
+			}
+			logs, err := client.ServiceLogs(c.Context, s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			// defer after err check as any err returns a nil io.ReadCloser
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+	wg.Wait()
+	os.Exit(0)
+}
+
+var appLogsCommand = &cli.Command{
+	Name:      "logs",
+	Aliases:   []string{"l"},
+	ArgsUsage: "[<service>]",
+	Usage:     "Tail app logs",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			logrus.Debug("tailing logs for all app services")
+			stackLogs(c, app.StackName(), cl)
+		}
+		logrus.Debugf("tailing logs for '%s'", serviceName)
+
+		service := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+		filters := filters.NewArgs()
+		filters.Add("name", service)
+		serviceOpts := types.ServiceListOptions{Filters: filters}
+		services, err := cl.ServiceList(c.Context, serviceOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if len(services) != 1 {
+			logrus.Fatalf("expected 1 service but got %v", len(services))
+		}
+
+		logOpts := types.ContainerLogsOptions{
+			Details:    true,
+			Follow:     true,
+			ShowStderr: true,
+			ShowStdout: true,
+			Tail:       "20",
+			Timestamps: true,
+		}
+		logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		// defer after err check as any err returns a nil io.ReadCloser
+		defer logs.Close()
+
+		_, err = io.Copy(os.Stdout, logs)
+		if err != nil && err != io.EOF {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/new.go

@@ -0,0 +1,58 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appNewDescription = `
+This command takes a recipe and uses it to create a new app. This new app
+configuration is stored in your ~/.abra directory under the appropriate server.
+
+This command does not deploy your app for you. You will need to run "abra app
+deploy <app>" to do so.
+
+You can see what recipes are available (i.e. values for the <recipe> argument)
+by running "abra recipe ls".
+
+Passing the "--secrets/-S" flag will automatically generate secrets for your
+app and store them encrypted at rest on the chosen target server. These
+generated secrets are only visible at generation time, so please take care to
+store them somewhere safe.
+
+You can use the "--pass/-P" to store these generated passwords locally in a
+pass store (see passwordstore.org for more). The pass command must be available
+on your $PATH.
+`
+
+var appNewCommand = &cli.Command{
+	Name:        "new",
+	Usage:       "Create a new app",
+	Aliases:     []string{"n"},
+	Description: appNewDescription,
+	Flags: []cli.Flag{
+		internal.NewAppServerFlag,
+		internal.DomainFlag,
+		internal.NewAppNameFlag,
+		internal.PassFlag,
+		internal.SecretsFlag,
+	},
+	ArgsUsage: "<recipe>",
+	Action:    internal.NewAction,
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+}

cli/app/ps.go

@@ -0,0 +1,99 @@
+package app
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/cli/cli/command/formatter"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var watch bool
+var watchFlag = &cli.BoolFlag{
+	Name:        "watch",
+	Aliases:     []string{"w"},
+	Value:       false,
+	Usage:       "Watch status by polling repeatedly",
+	Destination: &watch,
+}
+
+var appPsCommand = &cli.Command{
+	Name:    "ps",
+	Usage:   "Check app status",
+	Aliases: []string{"p"},
+	Flags: []cli.Flag{
+		watchFlag,
+	},
+	Action: func(c *cli.Context) error {
+		if !watch {
+			showPSOutput(c)
+			return nil
+		}
+
+		// TODO: how do we make this update in-place in an x-platform way?
+		for {
+			showPSOutput(c)
+			time.Sleep(2 * time.Second)
+		}
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}
+
+// showPSOutput renders ps output.
+func showPSOutput(c *cli.Context) {
+	app := internal.ValidateApp(c)
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", app.StackName())
+
+	containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	tableCol := []string{"image", "created", "status", "ports", "names"}
+	table := abraFormatter.CreateTable(tableCol)
+
+	for _, container := range containers {
+		var containerNames []string
+		for _, containerName := range container.Names {
+			trimmed := strings.TrimPrefix(containerName, "/")
+			containerNames = append(containerNames, trimmed)
+		}
+
+		tableRow := []string{
+			abraFormatter.RemoveSha(container.Image),
+			abraFormatter.HumanDuration(container.Created),
+			container.Status,
+			formatter.DisplayablePorts(container.Ports),
+			strings.Join(containerNames, "\n"),
+		}
+		table.Append(tableRow)
+	}
+
+	table.Render()
+}

cli/app/remove.go

@@ -0,0 +1,168 @@
+package app
+
+import (
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// Volumes stores the variable from VolumesFlag
+var Volumes bool
+
+// VolumesFlag is used to specify if volumes should be deleted when deleting an app
+var VolumesFlag = &cli.BoolFlag{
+	Name:        "volumes",
+	Value:       false,
+	Destination: &Volumes,
+}
+
+var appRemoveCommand = &cli.Command{
+	Name:    "remove",
+	Usage:   "Remove an already undeployed app",
+	Aliases: []string{"rm"},
+	Flags: []cli.Flag{
+		VolumesFlag,
+		internal.ForceFlag,
+	},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if !internal.Force {
+			response := false
+			prompt := &survey.Confirm{
+				Message: fmt.Sprintf("about to delete %s, are you sure?", app.Name),
+			}
+			if err := survey.AskOne(prompt, &response); err != nil {
+				logrus.Fatal(err)
+			}
+			if !response {
+				logrus.Fatal("user aborted app removal")
+			}
+		}
+
+		appFiles, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !internal.Force {
+			// FIXME: only query for app we are interested in, not all of them!
+			statuses, err := config.GetAppStatuses(appFiles)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if statuses[app.Name]["status"] == "deployed" {
+				logrus.Fatalf("'%s' is still deployed. Run \"abra app %s undeploy\" or pass --force", app.Name, app.Name)
+			}
+		}
+
+		fs := filters.NewArgs()
+		fs.Add("name", app.Name)
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: fs})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secrets := make(map[string]string)
+		var secretNames []string
+
+		for _, cont := range secretList {
+			secrets[cont.Spec.Annotations.Name] = cont.ID // we have to map the names to ID's
+			secretNames = append(secretNames, cont.Spec.Annotations.Name)
+		}
+
+		if len(secrets) > 0 {
+			var secretNamesToRemove []string
+			if !internal.Force {
+				secretsPrompt := &survey.MultiSelect{
+					Message: "which secrets do you want to remove?",
+					Options: secretNames,
+					Default: secretNames,
+				}
+				if err := survey.AskOne(secretsPrompt, &secretNamesToRemove); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			for _, name := range secretNamesToRemove {
+				err := cl.SecretRemove(c.Context, secrets[name])
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("secret: %s removed", name))
+			}
+		} else {
+			logrus.Info("no secrets to remove")
+		}
+
+		volumeListOKBody, err := cl.VolumeList(c.Context, fs)
+		volumeList := volumeListOKBody.Volumes
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var vols []string
+		for _, vol := range volumeList {
+			vols = append(vols, vol.Name)
+		}
+
+		if len(vols) > 0 {
+			if Volumes {
+				var removeVols []string
+				if !internal.Force {
+					volumesPrompt := &survey.MultiSelect{
+						Message: "which volumes do you want to remove?",
+						Options: vols,
+						Default: vols,
+					}
+					if err := survey.AskOne(volumesPrompt, &removeVols); err != nil {
+						logrus.Fatal(err)
+					}
+				}
+				for _, vol := range removeVols {
+					err := cl.VolumeRemove(c.Context, vol, internal.Force) // last argument is for force removing
+					if err != nil {
+						logrus.Fatal(err)
+					}
+					logrus.Info(fmt.Sprintf("volume %s removed", vol))
+				}
+			} else {
+				logrus.Info("no volumes were removed")
+			}
+		} else {
+			logrus.Info("no volumes to remove")
+		}
+
+		err = os.Remove(app.Path)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Info(fmt.Sprintf("file: %s removed", app.Path))
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/restore.go

@@ -0,0 +1,79 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var restoreAllServices bool
+var restoreAllServicesFlag = &cli.BoolFlag{
+	Name:        "all",
+	Value:       false,
+	Destination: &restoreAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Restore all services",
+}
+
+var appRestoreCommand = &cli.Command{
+	Name:      "restore",
+	Usage:     "Restore an app from a backup",
+	Aliases:   []string{"r"},
+	Flags:     []cli.Flag{restoreAllServicesFlag},
+	ArgsUsage: "<service> [<backup file>]",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() > 1 && restoreAllServices {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use <service>/<backup file> and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
+			logrus.Fatal(err)
+		}
+
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+		execCmd := "abra_restore"
+		if !restoreAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_restore_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+		}
+
+		backupFile := c.Args().Get(2)
+		if backupFile != "" {
+			execCmd = fmt.Sprintf("%s %s", execCmd, backupFile)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/rollback.go

@@ -0,0 +1,177 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appRollbackCommand = &cli.Command{
+	Name:      "rollback",
+	Usage:     "Roll an app back to a previous version",
+	Aliases:   []string{"r", "downgrade"},
+	ArgsUsage: "<app>",
+	Flags: []cli.Flag{
+		internal.ForceFlag,
+		internal.ChaosFlag,
+	},
+	Description: `
+This command rolls an app back to a previous version if one exists.
+
+You may pass "--force/-f" to downgrade to the same version again. This can be
+useful if the container runtime has gotten into a weird state.
+
+This action could be destructive, please ensure you have a copy of your app
+data beforehand - see "abra app backup <app>" for more.
+
+Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+including unstaged changes and can be useful for live hacking and testing new
+recipes.
+`,
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
+		}
+
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var availableDowngrades []string
+		if deployedVersion == "" {
+			deployedVersion = "unknown"
+			availableDowngrades = versions
+			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos {
+			for _, version := range versions {
+				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				parsedVersion, err := tagcmp.Parse(version)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if parsedVersion != parsedDeployedVersion && parsedVersion.IsLessThan(parsedDeployedVersion) {
+					availableDowngrades = append(availableDowngrades, version)
+				}
+			}
+
+			if len(availableDowngrades) == 0 {
+				logrus.Fatal("no available downgrades, you're on latest")
+			}
+		}
+
+		// FIXME: jeezus golang why do you not have a list reverse function
+		for i, j := 0, len(availableDowngrades)-1; i < j; i, j = i+1, j-1 {
+			availableDowngrades[i], availableDowngrades[j] = availableDowngrades[j], availableDowngrades[i]
+		}
+
+		var chosenDowngrade string
+		if !internal.Chaos {
+			if internal.Force {
+				chosenDowngrade = availableDowngrades[0]
+				logrus.Debugf("choosing '%s' as version to downgrade to (--force)", chosenDowngrade)
+			} else {
+				prompt := &survey.Select{
+					Message: fmt.Sprintf("Please select a downgrade (current version: '%s'):", deployedVersion),
+					Options: availableDowngrades,
+				}
+				if err := survey.AskOne(prompt, &chosenDowngrade); err != nil {
+					return err
+				}
+			}
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureVersion(app.Type, chosenDowngrade); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warn("chaos mode engaged")
+			var err error
+			chosenDowngrade, err = recipe.ChaosVersion(app.Type)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Force {
+			if err := internal.NewVersionOverview(app, deployedVersion, chosenDowngrade); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/run.go

@@ -0,0 +1,128 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var user string
+var userFlag = &cli.StringFlag{
+	Name:        "user",
+	Value:       "",
+	Destination: &user,
+}
+
+var noTTY bool
+var noTTYFlag = &cli.BoolFlag{
+	Name:        "no-tty",
+	Value:       false,
+	Destination: &noTTY,
+}
+
+var appRunCommand = &cli.Command{
+	Name: "run",
+	Flags: []cli.Flag{
+		noTTYFlag,
+		userFlag,
+	},
+	Aliases:   []string{"r"},
+	ArgsUsage: "<service> <args>...",
+	Usage:     "Run a command in a service container",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() < 2 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no <service> provided?"))
+		}
+
+		if c.Args().Len() < 3 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no <args> provided?"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		stackAndServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+		filters := filters.NewArgs()
+		filters.Add("name", stackAndServiceName)
+
+		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(containers) == 0 {
+			logrus.Fatalf("no containers matching '%s' found?", stackAndServiceName)
+		}
+		if len(containers) > 1 {
+			logrus.Fatalf("expected 1 container matching '%s' but got %d", stackAndServiceName, len(containers))
+		}
+
+		cmd := c.Args().Slice()[2:]
+		execCreateOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          cmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if user != "" {
+			execCreateOpts.User = user
+		}
+		if noTTY {
+			execCreateOpts.Tty = false
+		}
+
+		// FIXME: an absolutely monumental hack to instantiate another command-line
+		// client withing our command-line client so that we pass something down
+		// the tubes that satisfies the necessary interface requirements. We should
+		// refactor our vendored container code to not require all this cruft.  For
+		// now, It Works.
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, containers[0].ID, &execCreateOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		switch c.NArg() {
+		case 0:
+			appNames, err := config.GetAppNames()
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, a := range appNames {
+				fmt.Println(a)
+			}
+		case 1:
+			appName := c.Args().First()
+			serviceNames, err := config.GetAppServiceNames(appName)
+			if err != nil {
+				logrus.Warn(err)
+			}
+			for _, s := range serviceNames {
+				fmt.Println(s)
+			}
+		}
+	},
+}

cli/app/secret.go

@@ -0,0 +1,280 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var allSecrets bool
+var allSecretsFlag = &cli.BoolFlag{
+	Name:        "all",
+	Aliases:     []string{"A"},
+	Value:       false,
+	Destination: &allSecrets,
+	Usage:       "Generate all secrets",
+}
+
+var appSecretGenerateCommand = &cli.Command{
+	Name:      "generate",
+	Aliases:   []string{"g"},
+	Usage:     "Generate secrets",
+	ArgsUsage: "<secret> <version>",
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() == 1 && !allSecrets {
+			err := errors.New("missing arguments <secret>/<version> or '--all'")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		if c.Args().Get(1) != "" && allSecrets {
+			err := errors.New("cannot use '<secret> <version>' and '--all' together")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		secretsToCreate := make(map[string]string)
+		secretEnvVars := secret.ReadSecretEnvVars(app.Env)
+		if allSecrets {
+			secretsToCreate = secretEnvVars
+		} else {
+			secretName := c.Args().Get(1)
+			secretVersion := c.Args().Get(2)
+			matches := false
+			for sec := range secretEnvVars {
+				parsed := secret.ParseSecretEnvVarName(sec)
+				if secretName == parsed {
+					secretsToCreate[sec] = secretVersion
+				}
+			}
+			if !matches {
+				logrus.Fatalf("'%s' doesn't exist in the env config?", secretName)
+			}
+		}
+
+		secretVals, err := secret.GenerateSecrets(secretsToCreate, app.StackName(), app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if internal.Pass {
+			for name, data := range secretVals {
+				if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		if len(secretVals) == 0 {
+			logrus.Warn("no secrets generated")
+			os.Exit(1)
+		}
+
+		tableCol := []string{"name", "value"}
+		table := abraFormatter.CreateTable(tableCol)
+		for name, val := range secretVals {
+			table.Append([]string{name, val})
+		}
+		table.Render()
+		logrus.Warn("generated secrets are not shown again, please take note of them *now*")
+
+		return nil
+	},
+}
+
+var appSecretInsertCommand = &cli.Command{
+	Name:      "insert",
+	Aliases:   []string{"i"},
+	Usage:     "Insert secret",
+	Flags:     []cli.Flag{internal.PassFlag},
+	ArgsUsage: "<app> <secret-name> <version> <data>",
+	Description: `
+This command inserts a secret into an app environment.
+
+This can be useful when you want to manually generate secrets for an app
+environment. Typically, you can let Abra generate them for you on app creation
+(see "abra app new --secrets" for more).
+
+Example:
+
+    abra app secret insert myapp db_pass v1 mySecretPassword
+
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() != 4 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
+		}
+
+		name := c.Args().Get(1)
+		version := c.Args().Get(2)
+		data := c.Args().Get(3)
+
+		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
+		if err := client.StoreSecret(secretName, data, app.Server); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if internal.Pass {
+			if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+var appSecretRmCommand = &cli.Command{
+	Name:      "remove",
+	Usage:     "Remove a secret",
+	Aliases:   []string{"rm"},
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
+	ArgsUsage: "<app> <secret-name>",
+	Description: `
+This command removes a secret from an app environment.
+
+Example:
+
+    abra app secret remove myapp db_pass
+
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Get(1) != "" && allSecrets {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret-name>' and '--all' together"))
+		}
+
+		if c.Args().Get(1) == "" && !allSecrets {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no secret(s) specified?"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", app.StackName())
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretToRm := c.Args().Get(1)
+		for _, cont := range secretList {
+			secretName := cont.Spec.Annotations.Name
+			parsed := secret.ParseGeneratedSecretName(secretName, app)
+			if allSecrets {
+				if err := cl.SecretRemove(c.Context, secretName); err != nil {
+					logrus.Fatal(err)
+				}
+				if internal.Pass {
+					if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+						logrus.Fatal(err)
+					}
+				}
+			} else {
+				if parsed == secretToRm {
+					if err := cl.SecretRemove(c.Context, secretName); err != nil {
+						logrus.Fatal(err)
+					}
+					if internal.Pass {
+						if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+							logrus.Fatal(err)
+						}
+					}
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+var appSecretLsCommand = &cli.Command{
+	Name:    "list",
+	Usage:   "List all secrets",
+	Aliases: []string{"ls"},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		secrets := secret.ReadSecretEnvVars(app.Env)
+
+		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", app.StackName())
+		secretList, err := cl.SecretList(c.Context, types.SecretListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		remoteSecretNames := make(map[string]bool)
+		for _, cont := range secretList {
+			remoteSecretNames[cont.Spec.Annotations.Name] = true
+		}
+
+		for sec := range secrets {
+			createdRemote := false
+			secretName := secret.ParseSecretEnvVarName(sec)
+			secVal, err := secret.ParseSecretEnvVarValue(secrets[sec])
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version)
+			if _, ok := remoteSecretNames[secretRemoteName]; ok {
+				createdRemote = true
+			}
+			tableRow := []string{secretName, secVal.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}
+
+var appSecretCommand = &cli.Command{
+	Name:      "secret",
+	Aliases:   []string{"s"},
+	Usage:     "Manage app secrets",
+	ArgsUsage: "<command>",
+	Subcommands: []*cli.Command{
+		appSecretGenerateCommand,
+		appSecretInsertCommand,
+		appSecretRmCommand,
+		appSecretLsCommand,
+	},
+}

cli/app/undeploy.go

@@ -0,0 +1,66 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appUndeployCommand = &cli.Command{
+	Name:    "undeploy",
+	Aliases: []string{"u"},
+	Usage:   "Undeploy an app",
+	Description: `
+This does not destroy any of the application data. However, you should remain
+vigilant, as your swarm installation will consider any previously attached
+volumes as eligiblef or pruning once undeployed.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("'%s' is not deployed?", stackName)
+		}
+
+		if err := internal.DeployOverview(app, deployedVersion, "continue with undeploy?"); err != nil {
+			logrus.Fatal(err)
+		}
+
+		rmOpts := stack.Remove{Namespaces: []string{app.StackName()}}
+		if err := stack.RunRemove(c.Context, cl, rmOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/upgrade.go

@@ -0,0 +1,179 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appUpgradeCommand = &cli.Command{
+	Name:      "upgrade",
+	Aliases:   []string{"u"},
+	Usage:     "Upgrade an app",
+	ArgsUsage: "<app>",
+	Flags: []cli.Flag{
+		internal.ForceFlag,
+		internal.ChaosFlag,
+	},
+	Description: `
+This command supports upgrading an app. You can use it to choose and roll out a
+new upgrade to an existing app.
+
+This command specifically supports changing the version of running apps, as
+opposed to "abra app deploy <app>" which will not change the version of a
+deployed app.
+
+You may pass "--force/-f" to upgrade to the same version again. This can be
+useful if the container runtime has gotten into a weird state.
+
+This action could be destructive, please ensure you have a copy of your app
+data beforehand - see "abra app backup <app>" for more.
+
+Chas mode ("--chaos") will deploy your local checkout of a recipe as-is,
+including unstaged changes and can be useful for live hacking and testing new
+recipes.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
+		}
+
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(versions) == 0 && !internal.Chaos {
+			logrus.Fatalf("no versions available '%s' in recipe catalogue?", app.Type)
+		}
+
+		var availableUpgrades []string
+		if deployedVersion == "" {
+			deployedVersion = "unknown"
+			availableUpgrades = versions
+			logrus.Warnf("failed to determine version of deployed '%s'", app.Name)
+		}
+
+		if deployedVersion != "unknown" && !internal.Chaos {
+			for _, version := range versions {
+				parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				parsedVersion, err := tagcmp.Parse(version)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if parsedVersion.IsGreaterThan(parsedDeployedVersion) {
+					availableUpgrades = append(availableUpgrades, version)
+				}
+			}
+
+			if len(availableUpgrades) == 0 && !internal.Force {
+				logrus.Fatal("no available upgrades, you're on latest")
+				availableUpgrades = versions
+			}
+		}
+
+		var chosenUpgrade string
+		if len(availableUpgrades) > 0 && !internal.Chaos {
+			if internal.Force {
+				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+				logrus.Debugf("choosing '%s' as version to upgrade to", chosenUpgrade)
+			} else {
+				prompt := &survey.Select{
+					Message: fmt.Sprintf("Please select an upgrade (current version: '%s'):", deployedVersion),
+					Options: availableUpgrades,
+				}
+				if err := survey.AskOne(prompt, &chosenUpgrade); err != nil {
+					return err
+				}
+			}
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureVersion(app.Type, chosenUpgrade); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if internal.Chaos {
+			logrus.Warn("chaos mode engaged")
+			var err error
+			chosenUpgrade, err = recipe.ChaosVersion(app.Type)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/version.go

@@ -0,0 +1,104 @@
+package app
+
+import (
+	"fmt"
+	"strings"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// getImagePath returns the image name
+func getImagePath(image string) (string, error) {
+	img, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return "", err
+	}
+	path := reference.Path(img)
+	if strings.Contains(path, "library") {
+		path = strings.Split(path, "/")[1]
+	}
+	logrus.Debugf("parsed '%s' from '%s'", path, image)
+	return path, nil
+}
+
+var appVersionCommand = &cli.Command{
+	Name:    "version",
+	Aliases: []string{"v"},
+	Usage:   "Show app versions",
+	Description: `
+This command shows all information about versioning related to a deployed app.
+This includes the individual image names, tags and digests. But also the Co-op
+Cloud recipe version.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		stackName := app.StackName()
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("checking whether '%s' is already deployed", stackName)
+
+		isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if deployedVersion == "" {
+			logrus.Fatalf("failed to determine version of deployed '%s'", app.Name)
+		}
+
+		if !isDeployed {
+			logrus.Fatalf("'%s' is not deployed?", app.Name)
+		}
+
+		recipeMeta, err := catalogue.GetRecipeMeta(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versionsMeta := make(map[string]catalogue.ServiceMeta)
+		for _, recipeVersion := range recipeMeta.Versions {
+			if currentVersion, exists := recipeVersion[deployedVersion]; exists {
+				versionsMeta = currentVersion
+			}
+		}
+
+		if len(versionsMeta) == 0 {
+			logrus.Fatalf("PANIC: could not retrieve deployed version ('%s') from recipe catalogue?", deployedVersion)
+		}
+
+		tableCol := []string{"name", "image", "version", "tag", "digest"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		for serviceName, versionMeta := range versionsMeta {
+			table.Append([]string{serviceName, versionMeta.Image, deployedVersion, versionMeta.Tag, versionMeta.Digest})
+		}
+
+		table.Render()
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}

cli/app/volume.go

@@ -0,0 +1,106 @@
+package app
+
+import (
+	"fmt"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appVolumeListCommand = &cli.Command{
+	Name:    "list",
+	Usage:   "List volumes associated with an app",
+	Aliases: []string{"ls"},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		volumeList, err := client.GetVolumes(c.Context, app.Server, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		table := abraFormatter.CreateTable([]string{"driver", "volume name"})
+		var volTable [][]string
+		for _, volume := range volumeList {
+			volRow := []string{
+				volume.Driver,
+				volume.Name,
+			}
+			volTable = append(volTable, volRow)
+		}
+
+		table.AppendBulk(volTable)
+		table.Render()
+
+		return nil
+	},
+}
+
+var appVolumeRemoveCommand = &cli.Command{
+	Name:    "remove",
+	Usage:   "Remove volume(s) associated with an app",
+	Aliases: []string{"rm"},
+	Flags: []cli.Flag{
+		internal.ForceFlag,
+	},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		volumeList, err := client.GetVolumes(c.Context, app.Server, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		volumeNames := client.GetVolumeNames(volumeList)
+
+		var volumesToRemove []string
+		if !internal.Force {
+			volumesPrompt := &survey.MultiSelect{
+				Message: "which volumes do you want to remove?",
+				Options: volumeNames,
+				Default: volumeNames,
+			}
+			if err := survey.AskOne(volumesPrompt, &volumesToRemove); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			volumesToRemove = volumeNames
+		}
+
+		err = client.RemoveVolumes(c.Context, app.Server, volumesToRemove, internal.Force)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Info("volumes removed successfully")
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
+	},
+}
+
+var appVolumeCommand = &cli.Command{
+	Name:      "volume",
+	Aliases:   []string{"vl"},
+	Usage:     "Manage app volumes",
+	ArgsUsage: "<command>",
+	Subcommands: []*cli.Command{
+		appVolumeListCommand,
+		appVolumeRemoveCommand,
+	},
+}

cli/autocomplete.go

@@ -0,0 +1,121 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// downloadFile downloads a file brah
+func downloadFile(filepath string, url string) (err error) {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AutoCompleteCommand helps people set up auto-complete in their shells
+var AutoCompleteCommand = &cli.Command{
+	Name:    "autocomplete",
+	Usage:   "Help set up shell autocompletion",
+	Aliases: []string{"ac"},
+	Description: `
+This command helps set up autocompletion in your shell by downloading the
+relevant autocompletion files and laying out what additional information must
+be loaded.
+
+Example:
+
+    abra autocomplete bash
+
+Supported shells are as follows:
+
+		fish
+		zsh
+    bash
+`,
+	ArgsUsage: "<shell>",
+	Action: func(c *cli.Context) error {
+		shellType := c.Args().First()
+
+		if shellType == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no shell provided"))
+		}
+
+		supportedShells := map[string]bool{
+			"bash": true,
+			"zsh":  true,
+			"fish": true,
+		}
+
+		if _, ok := supportedShells[shellType]; !ok {
+			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
+		}
+
+		if shellType == "fish" {
+			shellType = "zsh" // handled the same on the autocompletion side
+		}
+
+		autocompletionDir := path.Join(config.ABRA_DIR, "autocompletion")
+		if err := os.Mkdir(autocompletionDir, 0755); err != nil {
+			if !os.IsExist(err) {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("'%s' already created, moving on...", autocompletionDir)
+		}
+
+		autocompletionFile := path.Join(config.ABRA_DIR, "autocompletion", shellType)
+		if _, err := os.Stat(autocompletionFile); err != nil && os.IsNotExist(err) {
+			url := fmt.Sprintf("https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/autocomplete/%s", shellType)
+			logrus.Infof("fetching %s", url)
+			if err := downloadFile(autocompletionFile, url); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		switch shellType {
+		case "bash":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install autocompletion
+sudo mkdir /etc/bash/completion.d/
+sudo cp %s /etc/bash_completion.d/abra
+echo "source /etc/bash/completion.d/abra" >> ~/.bashrc
+`, autocompletionFile))
+		case "zsh":
+			fmt.Println(fmt.Sprintf(`
+# Run the following commands to install autocompletion
+sudo mkdir /etc/zsh/completion.d/
+sudo cp %s /etc/zsh/completion.d/abra
+echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
+`, autocompletionFile))
+		}
+
+		return nil
+	},
+}

cli/catalogue/catalogue.go

@@ -0,0 +1,17 @@
+package catalogue
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// CatalogueCommand defines the `abra catalogue` command and sub-commands.
+var CatalogueCommand = &cli.Command{
+	Name:        "catalogue",
+	Usage:       "Manage the recipe catalogue (for maintainers)",
+	Aliases:     []string{"c"},
+	ArgsUsage:   "<recipe>",
+	Description: "This command helps recipe packagers interact with the recipe catalogue",
+	Subcommands: []*cli.Command{
+		catalogueGenerateCommand,
+	},
+}

cli/catalogue/generate.go

@@ -0,0 +1,179 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/limit"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                  true,
+	"abra-bash":             true,
+	"abra-apps":             true,
+	"abra-aur":              true,
+	"abra-capsul":           true,
+	"abra-gandi":            true,
+	"abra-hetzner":          true,
+	"apps":                  true,
+	"aur-abra-git":          true,
+	"auto-apps-json":        true,
+	"auto-mirror":           true,
+	"backup-bot":            true,
+	"backup-bot-two":        true,
+	"coopcloud.tech":        true,
+	"coturn":                true,
+	"docker-cp-deploy":      true,
+	"docker-dind-bats-kcov": true,
+	"docs.coopcloud.tech":   true,
+	"example":               true,
+	"gardening":             true,
+	"go-abra":               true,
+	"organising":            true,
+	"pyabra":                true,
+	"radicle-seed-node":     true,
+	"stack-ssh-deploy":      true,
+	"swarm-cronjob":         true,
+	"tagcmp":                true,
+	"tyop":                  true,
+}
+
+var catalogueGenerateCommand = &cli.Command{
+	Name:      "generate",
+	Aliases:   []string{"g"},
+	Usage:     "Generate a new copy of the catalogue",
+	ArgsUsage: "[<recipe>]",
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		repos, err := catalogue.ReadReposMetadata()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos))
+
+		cloneLimiter := limit.New(10)
+		retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...")
+		ch := make(chan string, len(repos))
+		for _, repoMeta := range repos {
+			go func(rm catalogue.RepoMeta) {
+				cloneLimiter.Begin()
+				defer cloneLimiter.End()
+
+				if recipeName != "" && recipeName != rm.Name {
+					ch <- rm.Name
+					retrieveBar.Add(1)
+					return
+				}
+				if _, exists := CatalogueSkipList[rm.Name]; exists {
+					ch <- rm.Name
+					retrieveBar.Add(1)
+					return
+				}
+
+				recipeDir := path.Join(config.ABRA_DIR, "apps", rm.Name)
+
+				if err := git.Clone(recipeDir, rm.SSHURL); err != nil {
+					logrus.Fatal(err)
+				}
+
+				if err := git.EnsureUpToDate(recipeDir); err != nil {
+					logrus.Fatal(err)
+				}
+
+				ch <- rm.Name
+				retrieveBar.Add(1)
+			}(repoMeta)
+		}
+
+		for range repos {
+			<-ch // wait for everything
+		}
+
+		catl := make(catalogue.RecipeCatalogue)
+		catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...")
+		for _, recipeMeta := range repos {
+			if recipeName != "" && recipeName != recipeMeta.Name {
+				catlBar.Add(1)
+				continue
+			}
+
+			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
+				catlBar.Add(1)
+				continue
+			}
+
+			versions, err := catalogue.GetRecipeVersions(recipeMeta.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			catl[recipeMeta.Name] = catalogue.RecipeMeta{
+				Name:          recipeMeta.Name,
+				Repository:    recipeMeta.CloneURL,
+				Icon:          recipeMeta.AvatarURL,
+				DefaultBranch: recipeMeta.DefaultBranch,
+				Description:   recipeMeta.Description,
+				Website:       recipeMeta.Website,
+				Versions:      versions,
+				// Category:      ..., // FIXME: once we sort out the machine-readable catalogue interface
+				// Features:      ..., // FIXME: once we figure out the machine-readable catalogue interface
+			}
+			catlBar.Add(1)
+		}
+
+		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if _, err := os.Stat(config.APPS_JSON); err != nil && os.IsNotExist(err) {
+			if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if recipeName != "" {
+				catlFS, err := catalogue.ReadRecipeCatalogue()
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				catlFS[recipeName] = catl[recipeName]
+
+				updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				if err := ioutil.WriteFile(config.APPS_JSON, updatedRecipesJSON, 0644); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		logrus.Infof("generated new recipe catalogue in '%s'", config.APPS_JSON)
+
+		return nil
+	},
+}

cli/cli.go

@@ -0,0 +1,122 @@
+// Package cli provides the interface for the command-line.
+package cli
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/cli/app"
+	"coopcloud.tech/abra/cli/catalogue"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/recipe"
+	"coopcloud.tech/abra/cli/record"
+	"coopcloud.tech/abra/cli/server"
+	"coopcloud.tech/abra/pkg/config"
+	logrusStack "github.com/Gurpartap/logrus-stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// Verbose stores the variable from VerboseFlag.
+var Verbose bool
+
+// VerboseFlag turns on/off verbose logging down to the INFO level.
+var VerboseFlag = &cli.BoolFlag{
+	Name:        "verbose",
+	Aliases:     []string{"V"},
+	Value:       false,
+	Destination: &Verbose,
+	Usage:       "Show INFO messages",
+}
+
+// Debug stores the variable from DebugFlag.
+var Debug bool
+
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
+var DebugFlag = &cli.BoolFlag{
+	Name:        "debug",
+	Aliases:     []string{"d"},
+	Value:       false,
+	Destination: &Debug,
+	Usage:       "Show DEBUG messages",
+}
+
+func newAbraApp(version, commit string) *cli.App {
+	app := &cli.App{
+		Name: "abra",
+		Usage: `The Co-op Cloud command-line utility belt 🎩🐇
+
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+		`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []*cli.Command{
+			app.AppCommand,
+			server.ServerCommand,
+			recipe.RecipeCommand,
+			catalogue.CatalogueCommand,
+			record.RecordCommand,
+			UpgradeCommand,
+			AutoCompleteCommand,
+		},
+		Flags: []cli.Flag{
+			VerboseFlag,
+			DebugFlag,
+			internal.NoInputFlag,
+		},
+		Authors: []*cli.Author{
+			{Name: "3wordchant"},
+			{Name: "decentral1se"},
+			{Name: "knoflook"},
+			{Name: "roxxers"},
+		},
+	}
+
+	app.EnableBashCompletion = true
+
+	app.Before = func(c *cli.Context) error {
+		if Debug {
+			logrus.SetLevel(logrus.DebugLevel)
+			logrus.SetFormatter(&logrus.TextFormatter{})
+			logrus.SetOutput(os.Stderr)
+			logrus.AddHook(logrusStack.StandardHook())
+		}
+
+		paths := []string{
+			config.ABRA_DIR,
+			path.Join(config.ABRA_DIR, "servers"),
+			path.Join(config.ABRA_DIR, "apps"),
+			path.Join(config.ABRA_DIR, "vendor"),
+		}
+
+		for _, path := range paths {
+			if err := os.Mkdir(path, 0755); err != nil {
+				if !os.IsExist(err) {
+					logrus.Fatal(err)
+				}
+				logrus.Debugf("'%s' already created, moving on...", path)
+				continue
+			}
+			logrus.Debugf("'%s' is missing, creating...", path)
+		}
+
+		logrus.Debugf("abra version '%s', commit '%s'", version, commit)
+
+		return nil
+	}
+	return app
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := newAbraApp(version, commit)
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cli/formatter/formatter.go

@@ -0,0 +1,52 @@
+package formatter
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/docker/cli/cli/command/formatter"
+	"github.com/docker/go-units"
+	"github.com/olekukonko/tablewriter"
+	"github.com/schollz/progressbar/v3"
+)
+
+func ShortenID(str string) string {
+	return str[:12]
+}
+
+func Truncate(str string) string {
+	return fmt.Sprintf(`"%s"`, formatter.Ellipsis(str, 19))
+}
+
+// RemoveSha remove image sha from a string that are added in some docker outputs
+func RemoveSha(str string) string {
+	return strings.Split(str, "@")[0]
+}
+
+// HumanDuration from docker/cli RunningFor() to be accessible outside of the class
+func HumanDuration(timestamp int64) string {
+	date := time.Unix(timestamp, 0)
+	now := time.Now().UTC()
+	return units.HumanDuration(now.Sub(date)) + " ago"
+}
+
+// CreateTable prepares a table layout for output.
+func CreateTable(columns []string) *tablewriter.Table {
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader(columns)
+	return table
+}
+
+// CreateProgressbar generates a progress bar
+func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
+	return progressbar.NewOptions(
+		length,
+		progressbar.OptionClearOnFinish(),
+		progressbar.OptionSetPredictTime(false),
+		progressbar.OptionShowCount(),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetDescription(title),
+	)
+}

cli/internal/command.go

@@ -0,0 +1,39 @@
+package internal
+
+import (
+	"bufio"
+	"fmt"
+	"os/exec"
+)
+
+// RunCmd runs a shell command and streams stdout/stderr in real-time.
+func RunCmd(cmd *exec.Cmd) error {
+	r, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	cmd.Stderr = cmd.Stdout
+	done := make(chan struct{})
+	scanner := bufio.NewScanner(r)
+
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			fmt.Println(line)
+		}
+		done <- struct{}{}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	<-done
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/internal/common.go

@@ -0,0 +1,261 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Secrets stores the variable from SecretsFlag
+var Secrets bool
+
+// SecretsFlag turns on/off automatically generating secrets
+var SecretsFlag = &cli.BoolFlag{
+	Name:        "secrets",
+	Aliases:     []string{"S"},
+	Value:       false,
+	Usage:       "Automatically generate secrets",
+	Destination: &Secrets,
+}
+
+// Pass stores the variable from PassFlag
+var Pass bool
+
+// PassFlag turns on/off storing generated secrets in pass
+var PassFlag = &cli.BoolFlag{
+	Name:        "pass",
+	Aliases:     []string{"P"},
+	Value:       false,
+	Usage:       "Store the generated secrets in a local pass store",
+	Destination: &Pass,
+}
+
+// Context is temp
+var Context string
+
+// ContextFlag is temp
+var ContextFlag = &cli.StringFlag{
+	Name:        "context",
+	Value:       "",
+	Aliases:     []string{"c"},
+	Destination: &Context,
+}
+
+// Force force functionality without asking.
+var Force bool
+
+// ForceFlag turns on/off force functionality.
+var ForceFlag = &cli.BoolFlag{
+	Name:        "force",
+	Value:       false,
+	Aliases:     []string{"f"},
+	Destination: &Force,
+}
+
+// Chaos engages chaos mode.
+var Chaos bool
+
+// ChaosFlag turns on/off chaos functionality.
+var ChaosFlag = &cli.BoolFlag{
+	Name:        "chaos",
+	Value:       false,
+	Aliases:     []string{"ch"},
+	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Destination: &Chaos,
+}
+
+// DNSProvider specifies a DNS provider.
+var DNSProvider string
+
+// DNSProviderFlag selects a DNS provider.
+var DNSProviderFlag = &cli.StringFlag{
+	Name:        "provider",
+	Value:       "",
+	Aliases:     []string{"p"},
+	Usage:       "DNS provider",
+	Destination: &DNSProvider,
+}
+
+var NoInput bool
+var NoInputFlag = &cli.BoolFlag{
+	Name:        "no-input",
+	Value:       false,
+	Aliases:     []string{"n"},
+	Usage:       "Toggle non-interactive mode",
+	Destination: &NoInput,
+}
+
+var DNSType string
+
+var DNSTypeFlag = &cli.StringFlag{
+	Name:        "type",
+	Value:       "",
+	Aliases:     []string{"t"},
+	Usage:       "Domain name record type (e.g. A)",
+	Destination: &DNSType,
+}
+
+var DNSName string
+
+var DNSNameFlag = &cli.StringFlag{
+	Name:        "name",
+	Value:       "",
+	Aliases:     []string{"n"},
+	Usage:       "Domain name record name (e.g. mysubdomain)",
+	Destination: &DNSName,
+}
+
+var DNSValue string
+
+var DNSValueFlag = &cli.StringFlag{
+	Name:        "value",
+	Value:       "",
+	Aliases:     []string{"v"},
+	Usage:       "Domain name record value (e.g. 192.168.1.1)",
+	Destination: &DNSValue,
+}
+
+var DNSTTL int
+
+var DNSTTLFlag = &cli.IntFlag{
+	Name:        "ttl",
+	Value:       86400,
+	Aliases:     []string{"T"},
+	Usage:       "Domain name TTL value)",
+	Destination: &DNSTTL,
+}
+
+var DNSPriority int
+
+var DNSPriorityFlag = &cli.IntFlag{
+	Name:        "priority",
+	Value:       10,
+	Aliases:     []string{"P"},
+	Usage:       "Domain name priority value",
+	Destination: &DNSPriority,
+}
+
+var ServerProvider string
+
+var ServerProviderFlag = &cli.StringFlag{
+	Name:        "provider",
+	Aliases:     []string{"p"},
+	Usage:       "3rd party server provider",
+	Destination: &ServerProvider,
+}
+
+var CapsulInstanceURL string
+
+var CapsulInstanceURLFlag = &cli.StringFlag{
+	Name:        "capsul-url",
+	Value:       "yolo.servers.coop",
+	Aliases:     []string{"cu"},
+	Usage:       "capsul instance URL",
+	Destination: &CapsulInstanceURL,
+}
+
+var CapsulName string
+
+var CapsulNameFlag = &cli.StringFlag{
+	Name:        "capsul-name",
+	Value:       "",
+	Aliases:     []string{"cn"},
+	Usage:       "capsul name",
+	Destination: &CapsulName,
+}
+
+var CapsulType string
+
+var CapsulTypeFlag = &cli.StringFlag{
+	Name:        "capsul-type",
+	Value:       "f1-xs",
+	Aliases:     []string{"ct"},
+	Usage:       "capsul type",
+	Destination: &CapsulType,
+}
+
+var CapsulImage string
+
+var CapsulImageFlag = &cli.StringFlag{
+	Name:        "capsul-image",
+	Value:       "debian10",
+	Aliases:     []string{"ci"},
+	Usage:       "capsul image",
+	Destination: &CapsulImage,
+}
+
+var CapsulSSHKeys cli.StringSlice
+
+var CapsulSSHKeysFlag = &cli.StringSliceFlag{
+	Name:        "capsul-ssh-keys",
+	Aliases:     []string{"cs"},
+	Usage:       "capsul SSH key",
+	Destination: &CapsulSSHKeys,
+}
+
+var CapsulAPIToken string
+
+var CapsulAPITokenFlag = &cli.StringFlag{
+	Name:        "capsul-token",
+	Aliases:     []string{"ca"},
+	Usage:       "capsul API token",
+	EnvVars:     []string{"CAPSUL_TOKEN"},
+	Destination: &CapsulAPIToken,
+}
+
+var HetznerCloudName string
+
+var HetznerCloudNameFlag = &cli.StringFlag{
+	Name:        "hetzner-name",
+	Value:       "",
+	Aliases:     []string{"hn"},
+	Usage:       "hetzner cloud name",
+	Destination: &HetznerCloudName,
+}
+
+var HetznerCloudType string
+
+var HetznerCloudTypeFlag = &cli.StringFlag{
+	Name:        "hetzner-type",
+	Aliases:     []string{"ht"},
+	Usage:       "hetzner cloud type",
+	Destination: &HetznerCloudType,
+	Value:       "cx11",
+}
+
+var HetznerCloudImage string
+
+var HetznerCloudImageFlag = &cli.StringFlag{
+	Name:        "hetzner-image",
+	Aliases:     []string{"hi"},
+	Usage:       "hetzner cloud image",
+	Value:       "debian-10",
+	Destination: &HetznerCloudImage,
+}
+
+var HetznerCloudSSHKeys cli.StringSlice
+
+var HetznerCloudSSHKeysFlag = &cli.StringSliceFlag{
+	Name:        "hetzner-ssh-keys",
+	Aliases:     []string{"hs"},
+	Usage:       "hetzner cloud SSH keys (e.g. me@foo.com)",
+	Destination: &HetznerCloudSSHKeys,
+}
+
+var HetznerCloudLocation string
+
+var HetznerCloudLocationFlag = &cli.StringFlag{
+	Name:        "hetzner-location",
+	Aliases:     []string{"hl"},
+	Usage:       "hetzner cloud server location",
+	Value:       "hel1",
+	Destination: &HetznerCloudLocation,
+}
+
+var HetznerCloudAPIToken string
+
+var HetznerCloudAPITokenFlag = &cli.StringFlag{
+	Name:        "hetzner-token",
+	Aliases:     []string{"ha"},
+	Usage:       "hetzner cloud API token",
+	EnvVars:     []string{"HCLOUD_TOKEN"},
+	Destination: &HetznerCloudAPIToken,
+}

cli/internal/deploy.go

@@ -0,0 +1,191 @@
+package internal
+
+import (
+	"fmt"
+	"strings"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// DeployAction is the main command-line action for this package
+func DeployAction(c *cli.Context) error {
+	app := ValidateApp(c)
+	stackName := app.StackName()
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("checking whether '%s' is already deployed", stackName)
+
+	isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if isDeployed {
+		if Force {
+			logrus.Warnf("'%s' already deployed but continuing (--force)", stackName)
+		} else if Chaos {
+			logrus.Warnf("'%s' already deployed but continuing (--chaos)", stackName)
+		} else {
+			logrus.Fatalf("'%s' is already deployed", stackName)
+		}
+	}
+
+	version := deployedVersion
+	if version == "" && !Chaos {
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if len(versions) > 0 {
+			version = versions[len(versions)-1]
+			logrus.Debugf("choosing '%s' as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Type, version); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			version = "latest commit"
+			logrus.Warn("no versions detected, using latest commit")
+			if err := recipe.EnsureLatest(app.Type); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+	}
+
+	if version == "" && !Chaos {
+		logrus.Debugf("choosing '%s' as version to deploy", version)
+		if err := recipe.EnsureVersion(app.Type, version); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if Chaos {
+		logrus.Warnf("chaos mode engaged")
+		var err error
+		version, err = recipe.ChaosVersion(app.Type)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	for k, v := range abraShEnv {
+		app.Env[k] = v
+	}
+
+	composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	deployOpts := stack.Deploy{
+		Composefiles: composeFiles,
+		Namespace:    stackName,
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+	compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := DeployOverview(app, version, "continue with deployment?"); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}
+
+// DeployOverview shows a deployment overview
+func DeployOverview(app config.App, version, message string) error {
+	tableCol := []string{"server", "compose", "domain", "stack", "version"}
+	table := abraFormatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), version})
+	table.Render()
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: message,
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}
+
+// NewVersionOverview shows an upgrade or downgrade overview
+func NewVersionOverview(app config.App, currentVersion, newVersion string) error {
+	tableCol := []string{"server", "compose", "domain", "stack", "current version", "to be deployed"}
+	table := abraFormatter.CreateTable(tableCol)
+
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = strings.Join(strings.Split(composeFiles, ":"), "\n")
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	table.Append([]string{server, deployConfig, app.Domain, app.StackName(), currentVersion, newVersion})
+	table.Render()
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with deployment?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	return nil
+}

cli/internal/errors.go

@@ -0,0 +1,18 @@
+package internal
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// ShowSubcommandHelpAndError exits the program on error, logs the error to the
+// terminal, and shows the help command.
+func ShowSubcommandHelpAndError(c *cli.Context, err interface{}) {
+	if err2 := cli.ShowSubcommandHelp(c); err2 != nil {
+		logrus.Error(err2)
+	}
+	logrus.Error(err)
+	os.Exit(1)
+}

cli/internal/errors_test.go

@@ -0,0 +1,38 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Testing functions that call os.Exit
+// https://stackoverflow.com/questions/26225513/how-to-test-os-exit-scenarios-in-go
+// https://talks.golang.org/2014/testing.slide#23
+
+var testapp = &cli.App{
+	Name:  "abra",
+	Usage: `The Co-op Cloud command-line utility belt 🎩🐇`,
+}
+
+// not testing output as that changes. just if it exits with code 1
+// does not work because of some weird errors on cli's part. Its a hard lib to test effectively.
+// func TestShowSubcommandHelpAndError(t *testing.T) {
+// 	if os.Getenv("HelpAndError") == "1" {
+// 		ShowSubcommandHelpAndError(cli.NewContext(testapp, nil, nil), errors.New("Test error"))
+// 		return
+// 	}
+// 	cmd := exec.Command(os.Args[0], "-test.run=TestShowSubcommandHelpAndError")
+// 	cmd.Env = append(os.Environ(), "HelpAndError=1")
+// 	var out bytes.Buffer
+// 	cmd.Stderr = &out
+// 	err := cmd.Run()
+// 	println(out.String())
+// 	if !strings.Contains(out.String(), "Test error") {
+
+// 		t.Fatalf("expected command to show the error causing the exit, did not get correct stdout output")
+// 	}
+
+// 	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
+// 		return
+// 	}
+// 	t.Fatalf("process ran with err %v, want exit status 1", err)
+// }

cli/internal/new.go

@@ -0,0 +1,203 @@
+package internal
+
+import (
+	"fmt"
+	"path"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type AppSecrets map[string]string
+
+var Domain string
+var DomainFlag = &cli.StringFlag{
+	Name:        "domain",
+	Aliases:     []string{"d"},
+	Value:       "",
+	Usage:       "Choose a domain name",
+	Destination: &Domain,
+}
+
+var NewAppServer string
+var NewAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &NewAppServer,
+}
+
+var NewAppName string
+var NewAppNameFlag = &cli.StringFlag{
+	Name:        "app-name",
+	Aliases:     []string{"a"},
+	Value:       "",
+	Usage:       "Choose an app name",
+	Destination: &NewAppName,
+}
+
+// RecipeName is used for configuring recipe name programmatically
+var RecipeName string
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(sanitisedAppName string) (AppSecrets, error) {
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", sanitisedAppName))
+	appEnv, err := config.ReadEnv(appEnvPath)
+	if err != nil {
+		return nil, err
+	}
+
+	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
+	secrets, err := secret.GenerateSecrets(secretEnvVars, sanitisedAppName, NewAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if Pass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(secretValue, secretName, sanitisedAppName, NewAppServer); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return secrets, nil
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag(recipe recipe.Recipe, server string) error {
+	if Domain == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify app domain",
+			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
+		}
+		if err := survey.AskOne(prompt, &Domain); err != nil {
+			return err
+		}
+	}
+
+	if Domain == "" {
+		return fmt.Errorf("no domain provided")
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	servers, err := config.GetServers()
+	if err != nil {
+		return err
+	}
+
+	if NewAppServer == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select app server:",
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &NewAppServer); err != nil {
+			return err
+		}
+	}
+
+	if NewAppServer == "" {
+		return fmt.Errorf("no server provided")
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the AppName flag was used. if not, asks the user for it.
+func ensureAppNameFlag() error {
+	if NewAppName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify app name:",
+			Default: config.SanitiseAppName(Domain),
+		}
+		if err := survey.AskOne(prompt, &NewAppName); err != nil {
+			return err
+		}
+	}
+
+	if NewAppName == "" {
+		return fmt.Errorf("no app name provided")
+	}
+
+	return nil
+}
+
+// NewAction is the new app creation logic
+func NewAction(c *cli.Context) error {
+	recipe := ValidateRecipeWithPrompt(c)
+
+	if err := config.EnsureAbraDirExists(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureServerFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureDomainFlag(recipe, NewAppServer); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureAppNameFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	sanitisedAppName := config.SanitiseAppName(NewAppName)
+	if len(sanitisedAppName) > 45 {
+		logrus.Fatalf("'%s' cannot be longer than 45 characters", sanitisedAppName)
+	}
+	logrus.Debugf("'%s' sanitised as '%s' for new app", NewAppName, sanitisedAppName)
+
+	if err := config.TemplateAppEnvSample(recipe.Name, NewAppName, NewAppServer, Domain, recipe.Name); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if Secrets {
+		secrets, err := createSecrets(sanitisedAppName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretCols := []string{"Name", "Value"}
+		secretTable := abraFormatter.CreateTable(secretCols)
+		for secret := range secrets {
+			secretTable.Append([]string{secret, secrets[secret]})
+		}
+
+		if len(secrets) > 0 {
+			defer secretTable.Render()
+		}
+	}
+
+	if NewAppServer == "default" {
+		NewAppServer = "local"
+	}
+
+	tableCol := []string{"Name", "Domain", "Type", "Server"}
+	table := abraFormatter.CreateTable(tableCol)
+	table.Append([]string{sanitisedAppName, Domain, recipe.Name, NewAppServer})
+
+	fmt.Println("")
+	fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
+	fmt.Println("")
+	table.Render()
+	fmt.Println("")
+	fmt.Println("You can configure this app by running the following:")
+	fmt.Println(fmt.Sprintf("\n    abra app config %s", sanitisedAppName))
+	fmt.Println("")
+	fmt.Println("You can deploy this app by running the following:")
+	fmt.Println(fmt.Sprintf("\n    abra app deploy %s", sanitisedAppName))
+	fmt.Println("")
+
+	return nil
+}

cli/internal/record.go

@@ -0,0 +1,106 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/urfave/cli/v2"
+)
+
+// EnsureDNSProvider ensures a DNS provider is chosen.
+func EnsureDNSProvider() error {
+	if DNSProvider == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select DNS provider",
+			Options: []string{"gandi"},
+		}
+
+		if err := survey.AskOne(prompt, &DNSProvider); err != nil {
+			return err
+		}
+	}
+
+	if DNSProvider == "" {
+		return fmt.Errorf("missing DNS provider?")
+	}
+
+	return nil
+}
+
+// EnsureDNSTypeFlag ensures a DNS type flag is present.
+func EnsureDNSTypeFlag(c *cli.Context) error {
+	if DNSType == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record type",
+			Default: "A",
+		}
+		if err := survey.AskOne(prompt, &DNSType); err != nil {
+			return err
+		}
+	}
+
+	if DNSType == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record type provided"))
+	}
+
+	return nil
+}
+
+// EnsureDNSNameFlag ensures a DNS name flag is present.
+func EnsureDNSNameFlag(c *cli.Context) error {
+	if DNSName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record name",
+			Default: "mysubdomain",
+		}
+		if err := survey.AskOne(prompt, &DNSName); err != nil {
+			return err
+		}
+	}
+
+	if DNSName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record name provided"))
+	}
+
+	return nil
+}
+
+// EnsureDNSValueFlag ensures a DNS value flag is present.
+func EnsureDNSValueFlag(c *cli.Context) error {
+	if DNSValue == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify DNS record value",
+			Default: "192.168.1.2",
+		}
+		if err := survey.AskOne(prompt, &DNSValue); err != nil {
+			return err
+		}
+	}
+
+	if DNSName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no record value provided"))
+	}
+
+	return nil
+}
+
+// EnsureZoneArgument ensures a zone argument is present.
+func EnsureZoneArgument(c *cli.Context) (string, error) {
+	var zone string
+	if c.Args().First() == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify a domain name zone",
+			Default: "example.com",
+		}
+		if err := survey.AskOne(prompt, &zone); err != nil {
+			return zone, err
+		}
+	}
+
+	if zone == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no zone value provided"))
+	}
+
+	return zone, nil
+}

cli/internal/server.go

@@ -0,0 +1,208 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/urfave/cli/v2"
+)
+
+// EnsureServerProvider ensures a 3rd party server provider is chosen.
+func EnsureServerProvider() error {
+	if ServerProvider == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Select server provider",
+			Options: []string{"capsul", "hetzner-cloud"},
+		}
+
+		if err := survey.AskOne(prompt, &ServerProvider); err != nil {
+			return err
+		}
+	}
+
+	if ServerProvider == "" {
+		return fmt.Errorf("missing server provider?")
+	}
+
+	return nil
+}
+
+// EnsureNewCapsulVPSFlags ensure all flags are present.
+func EnsureNewCapsulVPSFlags(c *cli.Context) error {
+	if CapsulName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul name",
+		}
+		if err := survey.AskOne(prompt, &CapsulName); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul instance URL",
+			Default: CapsulInstanceURL,
+		}
+		if err := survey.AskOne(prompt, &CapsulInstanceURL); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul type",
+			Default: CapsulType,
+		}
+		if err := survey.AskOne(prompt, &CapsulType); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify capsul image",
+			Default: CapsulImage,
+		}
+		if err := survey.AskOne(prompt, &CapsulImage); err != nil {
+			return err
+		}
+	}
+
+	if len(CapsulSSHKeys.Value()) == 0 && !NoInput {
+		var sshKeys string
+		prompt := &survey.Input{
+			Message: "specify capsul SSH keys (e.g. me@foo.com)",
+			Default: "",
+		}
+		if err := survey.AskOne(prompt, &CapsulSSHKeys); err != nil {
+			return err
+		}
+		CapsulSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
+	}
+
+	if CapsulAPIToken == "" && !NoInput {
+		token, ok := os.LookupEnv("CAPSUL_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify capsul API token",
+			}
+			if err := survey.AskOne(prompt, &CapsulAPIToken); err != nil {
+				return err
+			}
+		} else {
+			CapsulAPIToken = token
+		}
+	}
+
+	if CapsulName == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul name?"))
+	}
+	if CapsulInstanceURL == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul instance url?"))
+	}
+	if CapsulType == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul type?"))
+	}
+	if CapsulImage == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul image?"))
+	}
+	if len(CapsulSSHKeys.Value()) == 0 {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul ssh keys?"))
+	}
+	if CapsulAPIToken == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing capsul API token?"))
+	}
+
+	return nil
+}
+
+// EnsureNewHetznerCloudVPSFlags ensure all flags are present.
+func EnsureNewHetznerCloudVPSFlags(c *cli.Context) error {
+	if HetznerCloudName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS name",
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudName); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS type",
+			Default: HetznerCloudType,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudType); err != nil {
+			return err
+		}
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS image",
+			Default: HetznerCloudImage,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudImage); err != nil {
+			return err
+		}
+	}
+
+	if len(HetznerCloudSSHKeys.Value()) == 0 && !NoInput {
+		var sshKeys string
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud SSH keys (e.g. me@foo.com)",
+			Default: "",
+		}
+		if err := survey.AskOne(prompt, &sshKeys); err != nil {
+			return err
+		}
+		HetznerCloudSSHKeys = *cli.NewStringSlice(strings.Split(sshKeys, ",")...)
+	}
+
+	if !NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS location",
+			Default: HetznerCloudLocation,
+		}
+		if err := survey.AskOne(prompt, &HetznerCloudLocation); err != nil {
+			return err
+		}
+	}
+
+	if HetznerCloudAPIToken == "" && !NoInput {
+		token, ok := os.LookupEnv("HCLOUD_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify hetzner cloud API token",
+			}
+			if err := survey.AskOne(prompt, &HetznerCloudAPIToken); err != nil {
+				return err
+			}
+		} else {
+			HetznerCloudAPIToken = token
+		}
+	}
+
+	if HetznerCloudName == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS name?"))
+	}
+	if HetznerCloudType == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS type?"))
+	}
+	if HetznerCloudImage == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud image?"))
+	}
+	if len(HetznerCloudSSHKeys.Value()) == 0 {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud ssh keys?"))
+	}
+	if HetznerCloudLocation == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud VPS location?"))
+	}
+	if HetznerCloudAPIToken == "" {
+		ShowSubcommandHelpAndError(c, fmt.Errorf("missing hetzner cloud API token?"))
+	}
+
+	return nil
+}

cli/internal/validate.go

@@ -0,0 +1,169 @@
+package internal
+
+import (
+	"errors"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// AppName is used for configuring app name programmatically
+var AppName string
+
+// ValidateRecipe ensures the recipe arg is valid.
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
+	recipeName := c.Args().First()
+
+	if recipeName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+	}
+
+	recipe, err := recipe.Get(recipeName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated '%s' as recipe argument", recipeName)
+
+	return recipe
+}
+
+// ValidateRecipeWithPrompt ensures a recipe argument is present before
+// validating, asking for input if required.
+func ValidateRecipeWithPrompt(c *cli.Context) recipe.Recipe {
+	recipeName := c.Args().First()
+
+	if recipeName == "" && !NoInput {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		var recipes []string
+		for name := range catl {
+			recipes = append(recipes, name)
+		}
+		prompt := &survey.Select{
+			Message: "Select recipe",
+			Options: recipes,
+		}
+		if err := survey.AskOne(prompt, &recipeName); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if RecipeName != "" {
+		recipeName = RecipeName
+		logrus.Debugf("programmatically setting recipe name to %s", recipeName)
+	}
+
+	if recipeName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+	}
+
+	recipe, err := recipe.Get(recipeName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated '%s' as recipe argument", recipeName)
+
+	return recipe
+}
+
+// ValidateApp ensures the app name arg is valid.
+func ValidateApp(c *cli.Context) config.App {
+	appName := c.Args().First()
+
+	if AppName != "" {
+		appName = AppName
+		logrus.Debugf("programmatically setting app name to %s", appName)
+	}
+
+	if appName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+	}
+
+	app, err := app.Get(appName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := recipe.EnsureExists(app.Type); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated '%s' as app argument", appName)
+
+	return app
+}
+
+// ValidateDomain ensures the domain name arg is valid.
+func ValidateDomain(c *cli.Context) (string, error) {
+	domainName := c.Args().First()
+
+	if domainName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: "Specify a domain name",
+			Default: "example.com",
+		}
+		if err := survey.AskOne(prompt, &domainName); err != nil {
+			return domainName, err
+		}
+	}
+
+	if domainName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no domain provided"))
+	}
+
+	logrus.Debugf("validated '%s' as domain argument", domainName)
+
+	return domainName, nil
+}
+
+// ValidateSubCmdFlags ensures flag order conforms to correct order
+func ValidateSubCmdFlags(c *cli.Context) bool {
+	for argIdx, arg := range c.Args().Slice() {
+		if !strings.HasPrefix(arg, "--") {
+			for _, flag := range c.Args().Slice()[argIdx:] {
+				if strings.HasPrefix(flag, "--") {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+// ValidateServer ensures the server name arg is valid.
+func ValidateServer(c *cli.Context) (string, error) {
+	serverName := c.Args().First()
+
+	serverNames, err := config.ReadServerNames()
+	if err != nil {
+		return serverName, err
+	}
+
+	if serverName == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: "Specify a server name",
+			Options: serverNames,
+		}
+		if err := survey.AskOne(prompt, &serverName); err != nil {
+			return serverName, err
+		}
+	}
+
+	if serverName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no server provided"))
+	}
+
+	logrus.Debugf("validated '%s' as server argument", serverName)
+
+	return serverName, nil
+}

cli/recipe/lint.go

@@ -0,0 +1,106 @@
+package recipe
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/tagcmp"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeLintCommand = &cli.Command{
+	Name:      "lint",
+	Usage:     "Lint a recipe",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		expectedVersion := false
+		if recipe.Config.Version == "3.8" {
+			expectedVersion = true
+		}
+
+		envSampleProvided := false
+		envSample := fmt.Sprintf("%s/%s/.env.sample", config.APPS_DIR, recipe.Name)
+		if _, err := os.Stat(envSample); !os.IsNotExist(err) {
+			envSampleProvided = true
+		} else if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceNamedApp := false
+		traefikEnabled := false
+		healthChecksForAllServices := true
+		allImagesTagged := true
+		noUnstableTags := true
+		semverLikeTags := true
+		for _, service := range recipe.Config.Services {
+			if service.Name == "app" {
+				serviceNamedApp = true
+			}
+
+			for label := range service.Deploy.Labels {
+				if label == "traefik.enable" {
+					if service.Deploy.Labels[label] == "true" {
+						traefikEnabled = true
+					}
+				}
+			}
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if reference.IsNameOnly(img) {
+				allImagesTagged = false
+			}
+
+			tag := img.(reference.NamedTagged).Tag()
+			if tag == "latest" {
+				noUnstableTags = false
+			}
+
+			if !tagcmp.IsParsable(tag) {
+				semverLikeTags = false
+			}
+
+			if service.HealthCheck == nil {
+				healthChecksForAllServices = false
+			}
+		}
+
+		tableCol := []string{"rule", "satisfied"}
+		table := formatter.CreateTable(tableCol)
+		table.Append([]string{"compose files have the expected version", strconv.FormatBool(expectedVersion)})
+		table.Append([]string{"environment configuration is provided", strconv.FormatBool(envSampleProvided)})
+		table.Append([]string{"recipe contains a service named 'app'", strconv.FormatBool(serviceNamedApp)})
+		table.Append([]string{"traefik routing enabled on at least one service", strconv.FormatBool(traefikEnabled)})
+		table.Append([]string{"all services have a healthcheck enabled", strconv.FormatBool(healthChecksForAllServices)})
+		table.Append([]string{"all images are using a tag", strconv.FormatBool(allImagesTagged)})
+		table.Append([]string{"no usage of unstable 'latest' tags", strconv.FormatBool(noUnstableTags)})
+		table.Append([]string{"all tags are using a semver-like format", strconv.FormatBool(semverLikeTags)})
+		table.Render()
+
+		return nil
+	},
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+}

cli/recipe/list.go

@@ -0,0 +1,39 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeListCommand = &cli.Command{
+	Name:    "list",
+	Usage:   "List available recipes",
+	Aliases: []string{"ls"},
+	Action: func(c *cli.Context) error {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err.Error())
+		}
+
+		recipes := catl.Flatten()
+		sort.Sort(catalogue.ByRecipeName(recipes))
+
+		tableCol := []string{"name", "category", "status"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, recipe := range recipes {
+			status := fmt.Sprintf("%v", recipe.Features.Status)
+			tableRow := []string{recipe.Name, recipe.Category, status}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+
+		return nil
+	},
+}

cli/recipe/new.go

@@ -0,0 +1,92 @@
+package recipe
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"text/template"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeNewCommand = &cli.Command{
+	Name:      "new",
+	Usage:     "Create a new recipe",
+	Aliases:   []string{"n"},
+	ArgsUsage: "<recipe>",
+	Description: `
+This command creates a new recipe.
+
+Abra uses our built-in example repository which is available here:
+
+    https://git.coopcloud.tech/coop-cloud/example
+
+Files within the example repository make use of the Golang templating system
+which Abra uses to inject values into the generated recipe folder (e.g. name of
+recipe and domain in the sample environment config).
+
+The new example repository is cloned to ~/.abra/apps/<recipe>.
+`,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+		}
+
+		directory := path.Join(config.APPS_DIR, recipeName)
+		if _, err := os.Stat(directory); !os.IsNotExist(err) {
+			logrus.Fatalf("'%s' recipe directory already exists?", directory)
+		}
+
+		url := fmt.Sprintf("%s/example.git", config.REPOS_BASE_URL)
+		if err := git.Clone(directory, url); err != nil {
+			logrus.Fatal(err)
+		}
+
+		gitRepo := path.Join(config.APPS_DIR, recipeName, ".git")
+		if err := os.RemoveAll(gitRepo); err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("removed git repo in '%s'", gitRepo)
+
+		toParse := []string{
+			path.Join(config.APPS_DIR, recipeName, "README.md"),
+			path.Join(config.APPS_DIR, recipeName, ".env.sample"),
+			path.Join(config.APPS_DIR, recipeName, ".drone.yml"),
+		}
+		for _, path := range toParse {
+			file, err := os.OpenFile(path, os.O_RDWR, 0755)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			tpl, err := template.ParseFiles(path)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			// TODO: ask for description and probably other things so that the
+			// template repository is more "ready" to go than the current best-guess
+			// mode of templating
+			if err := tpl.Execute(file, struct {
+				Name        string
+				Description string
+			}{recipeName, "TODO"}); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		logrus.Infof(
+			"new recipe '%s' created in %s, happy hacking!\n",
+			recipeName, path.Join(config.APPS_DIR, recipeName),
+		)
+
+		return nil
+	},
+}

cli/recipe/recipe.go

@@ -0,0 +1,51 @@
+package recipe
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var Major bool
+var MajorFlag = &cli.BoolFlag{
+	Name:        "major",
+	Value:       false,
+	Aliases:     []string{"ma", "x"},
+	Destination: &Major,
+}
+
+var Minor bool
+var MinorFlag = &cli.BoolFlag{
+	Name:        "minor",
+	Value:       false,
+	Aliases:     []string{"mi", "y"},
+	Destination: &Minor,
+}
+
+var Patch bool
+var PatchFlag = &cli.BoolFlag{
+	Name:        "patch",
+	Value:       false,
+	Aliases:     []string{"p", "z"},
+	Destination: &Patch,
+}
+
+// RecipeCommand defines all recipe related sub-commands.
+var RecipeCommand = &cli.Command{
+	Name:      "recipe",
+	Usage:     "Manage recipes (for maintainers)",
+	ArgsUsage: "<recipe>",
+	Aliases:   []string{"r"},
+	Description: `
+A recipe is a blueprint for an app. It is a bunch of configuration files which
+describe how to deploy and maintain an app. Recipes are maintained by the Co-op
+Cloud community and you can use Abra to read them and create apps for you.
+`,
+	Subcommands: []*cli.Command{
+		recipeListCommand,
+		recipeVersionCommand,
+		recipeReleaseCommand,
+		recipeNewCommand,
+		recipeUpgradeCommand,
+		recipeSyncCommand,
+		recipeLintCommand,
+	},
+}

cli/recipe/release.go

@@ -0,0 +1,319 @@
+package recipe
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Push bool
+var PushFlag = &cli.BoolFlag{
+	Name:        "push",
+	Value:       false,
+	Destination: &Push,
+}
+
+var Dry bool
+var DryFlag = &cli.BoolFlag{
+	Name:        "dry-run",
+	Value:       false,
+	Aliases:     []string{"d"},
+	Destination: &Dry,
+}
+
+var CommitMessage string
+var CommitMessageFlag = &cli.StringFlag{
+	Name:        "commit-message",
+	Usage:       "commit message. Implies --commit",
+	Aliases:     []string{"cm"},
+	Destination: &CommitMessage,
+}
+
+var Commit bool
+var CommitFlag = &cli.BoolFlag{
+	Name:        "commit",
+	Usage:       "add compose.yml to staging area and commit changes",
+	Value:       false,
+	Aliases:     []string{"c"},
+	Destination: &Commit,
+}
+
+var TagMessage string
+var TagMessageFlag = &cli.StringFlag{
+	Name:        "tag-comment",
+	Usage:       "tag comment. If not given, user will be asked for it",
+	Aliases:     []string{"t", "tm"},
+	Destination: &TagMessage,
+}
+
+var recipeReleaseCommand = &cli.Command{
+	Name:      "release",
+	Usage:     "tag a recipe",
+	Aliases:   []string{"rl"},
+	ArgsUsage: "<recipe> [<tag>]",
+	Description: `
+This command is used to specify a new tag for a recipe. These tags are used to
+identify different versions of the recipe and are published on the Co-op Cloud
+recipe catalogue.
+
+These tags take the following form:
+
+    a.b.c+x.y.z
+
+Where the "a.b.c" part is maintained as a semantic version of the recipe by the
+recipe maintainer. And the "x.y.z" part is the image tag of the recipe "app"
+service (the main container which contains the software to be used).
+
+We maintain a semantic versioning scheme ("a.b.c") alongside the libre app
+versioning scheme in order to maximise the chances that the nature of recipe
+updates are properly communicated.
+
+Abra does its best to read the "a.b.c" version scheme and communicate what
+action needs to be taken when performing different operations such as an update
+or a rollback of an app.
+`,
+	Flags: []cli.Flag{
+		DryFlag,
+		PatchFlag,
+		MinorFlag,
+		MajorFlag,
+		PushFlag,
+		CommitFlag,
+		CommitMessageFlag,
+		TagMessageFlag,
+	},
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+		directory := path.Join(config.APPS_DIR, recipe.Name)
+		tagstring := c.Args().Get(1)
+		imagesTmp := getImageVersions(recipe)
+		mainApp := getMainApp(recipe)
+		mainAppVersion := imagesTmp[mainApp]
+
+		if err := recipePkg.EnsureExists(recipe.Name); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if mainAppVersion == "" {
+			logrus.Fatal("main app version is empty?")
+		}
+
+		if tagstring != "" {
+			if _, err := tagcmp.Parse(tagstring); err != nil {
+				logrus.Fatal("invalid tag specified")
+			}
+		}
+
+		if TagMessage == "" {
+			prompt := &survey.Input{
+				Message: "tag message",
+			}
+			if err := survey.AskOne(prompt, &TagMessage); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		var createTagOptions git.CreateTagOptions
+		createTagOptions.Message = TagMessage
+
+		if Commit || (CommitMessage != "") {
+			commitRepo, err := git.PlainOpen(directory)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			commitWorktree, err := commitRepo.Worktree()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if CommitMessage == "" {
+				prompt := &survey.Input{
+					Message: "commit message",
+				}
+				if err := survey.AskOne(prompt, &CommitMessage); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			err = commitWorktree.AddGlob("compose.**yml")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debug("staged compose.**yml for commit")
+
+			_, err = commitWorktree.Commit(CommitMessage, &git.CommitOptions{})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info("changes commited")
+		}
+
+		repo, err := git.PlainOpen(directory)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		head, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		// bumpType is used to decide what part of the tag should be incremented
+		bumpType := btoi(Major)*4 + btoi(Minor)*2 + btoi(Patch)
+		if bumpType != 0 {
+			// a bitwise check if the number is a power of 2
+			if (bumpType & (bumpType - 1)) != 0 {
+				logrus.Fatal("you can only use one of: --major, --minor, --patch.")
+			}
+		}
+
+		if tagstring != "" {
+			if bumpType > 0 {
+				logrus.Warn("user specified a version number and --major/--minor/--patch at the same time! using version number...")
+			}
+			tag, err := tagcmp.Parse(tagstring)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if tag.MissingMinor {
+				tag.Minor = "0"
+				tag.MissingMinor = false
+			}
+			if tag.MissingPatch {
+				tag.Patch = "0"
+				tag.MissingPatch = false
+			}
+			tagstring = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
+			if Dry {
+				logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", tagstring, head.Hash()))
+				return nil
+			}
+
+			repo.CreateTag(tagstring, head.Hash(), &createTagOptions)
+			logrus.Info(fmt.Sprintf("created tag %s at %s", tagstring, head.Hash()))
+			if Push {
+				if err := repo.Push(&git.PushOptions{}); err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("pushed tag %s to remote", tagstring))
+			}
+
+			return nil
+		}
+
+		// get the latest tag with its hash, name etc
+		var lastGitTag tagcmp.Tag
+		iter, err := repo.Tags()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if err := iter.ForEach(func(ref *plumbing.Reference) error {
+			obj, err := repo.TagObject(ref.Hash())
+			if err != nil {
+				return err
+			}
+			tagcmpTag, err := tagcmp.Parse(obj.Name)
+			if err != nil {
+				return err
+			}
+			if (lastGitTag == tagcmp.Tag{}) {
+				lastGitTag = tagcmpTag
+			} else if tagcmpTag.IsGreaterThan(lastGitTag) {
+				lastGitTag = tagcmpTag
+			}
+			return nil
+		}); err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Println(lastGitTag)
+
+		newTag := lastGitTag
+		var newTagString string
+		if bumpType > 0 {
+			if Patch {
+				now, err := strconv.Atoi(newTag.Patch)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = strconv.Itoa(now + 1)
+			} else if Minor {
+				now, err := strconv.Atoi(newTag.Minor)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = "0"
+				newTag.Minor = strconv.Itoa(now + 1)
+			} else if Major {
+				now, err := strconv.Atoi(newTag.Major)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				newTag.Patch = "0"
+				newTag.Minor = "0"
+				newTag.Major = strconv.Itoa(now + 1)
+			}
+		} else {
+			logrus.Fatal("we don't support automatic tag generation yet - specify a version or use one of: --major --minor --patch")
+		}
+		newTag.Metadata = mainAppVersion
+		newTagString = newTag.String()
+		if Dry {
+			logrus.Info(fmt.Sprintf("dry run only: NOT creating tag %s at %s", newTagString, head.Hash()))
+			return nil
+		}
+
+		repo.CreateTag(newTagString, head.Hash(), &createTagOptions)
+		logrus.Info(fmt.Sprintf("created tag %s at %s", newTagString, head.Hash()))
+		if Push {
+			if err := repo.Push(&git.PushOptions{}); err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info(fmt.Sprintf("pushed tag %s to remote", newTagString))
+		}
+
+		return nil
+
+	},
+}
+
+func getImageVersions(recipe recipe.Recipe) map[string]string {
+
+	var services = make(map[string]string)
+	for _, service := range recipe.Config.Services {
+		if service.Image == "" {
+			continue
+		}
+		srv := strings.Split(service.Image, ":")
+		services[srv[0]] = srv[1]
+	}
+
+	return services
+}
+
+func getMainApp(recipe recipe.Recipe) string {
+	for _, service := range recipe.Config.Services {
+		name := service.Name
+		if name == "app" {
+			return strings.Split(service.Image, ":")[0]
+		}
+	}
+	return ""
+}
+
+func btoi(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}

cli/recipe/sync.go

@@ -0,0 +1,88 @@
+package recipe
+
+import (
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeSyncCommand = &cli.Command{
+	Name:      "sync",
+	Usage:     "Ensure recipe version labels are up-to-date",
+	Aliases:   []string{"s"},
+	ArgsUsage: "<recipe> <version>",
+	Description: `
+This command will generate labels for the main recipe service (i.e. by
+convention, typically the service named "app") which corresponds to the
+following format:
+
+    coop-cloud.${STACK_NAME}.version=<version>
+
+The <version> is determined by the recipe maintainer and is specified on the
+command-line. The <recipe> configuration will be updated on the local file
+system.
+`,
+	BashComplete: func(c *cli.Context) {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for name := range catl {
+			fmt.Println(name)
+		}
+	},
+	Action: func(c *cli.Context) error {
+		if c.Args().Len() != 2 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <recipe>/<version> arguments?"))
+		}
+
+		recipe := internal.ValidateRecipe(c)
+
+		// TODO: validate with tagcmp when new commits come in
+		// See https://git.coopcloud.tech/coop-cloud/abra/pulls/109
+		nextTag := c.Args().Get(1)
+
+		mainService := "app"
+		var services []string
+		hasAppService := false
+		for _, service := range recipe.Config.Services {
+			services = append(services, service.Name)
+			if service.Name == "app" {
+				hasAppService = true
+				logrus.Debugf("detected app service in '%s'", recipe.Name)
+			}
+		}
+
+		if !hasAppService {
+			logrus.Warnf("no 'app' service defined in '%s'", recipe.Name)
+			var chosenService string
+			prompt := &survey.Select{
+				Message: fmt.Sprintf("what is the main service name for '%s'?", recipe.Name),
+				Options: services,
+			}
+			if err := survey.AskOne(prompt, &chosenService); err != nil {
+				logrus.Fatal(err)
+			}
+			mainService = chosenService
+		}
+
+		logrus.Debugf("selecting '%s' as the service to sync version labels", mainService)
+
+		label := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", nextTag)
+		if err := recipe.UpdateLabel(mainService, label); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("synced label '%s' to service '%s'", label, mainService)
+
+		return nil
+	},
+}

cli/recipe/upgrade.go

@@ -0,0 +1,235 @@
+package recipe
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type imgPin struct {
+	image   string
+	version tagcmp.Tag
+}
+
+var recipeUpgradeCommand = &cli.Command{
+	Name:    "upgrade",
+	Usage:   "Upgrade recipe image tags",
+	Aliases: []string{"u"},
+	Description: `
+This command reads and attempts to parse all image tags within the given
+<recipe> configuration and prompt with more recent tags to upgrade to. It will
+update the relevant compose file tags on the local file system.
+
+Some image tags cannot be parsed because they do not follow some sort of
+semver-like convention. In this case, all possible tags will be listed and it
+is up to the end-user to decide.
+`,
+	ArgsUsage: "<recipe>",
+	Flags: []cli.Flag{
+		PatchFlag,
+		MinorFlag,
+		MajorFlag,
+	},
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		bumpType := btoi(Major)*4 + btoi(Minor)*2 + btoi(Patch)
+		if bumpType != 0 {
+			// a bitwise check if the number is a power of 2
+			if (bumpType & (bumpType - 1)) != 0 {
+				logrus.Fatal("you can only use one of: --major, --minor, --patch.")
+			}
+		}
+
+		// check for versions file and load pinned versions
+		versionsPresent := false
+		recipeDir := path.Join(config.ABRA_DIR, "apps", recipe.Name)
+		versionsPath := path.Join(recipeDir, "versions")
+		var servicePins = make(map[string]imgPin)
+		if _, err := os.Stat(versionsPath); err == nil {
+			logrus.Debugf("found versions file for %s", recipe.Name)
+			file, err := os.Open(versionsPath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := scanner.Text()
+				splitLine := strings.Split(line, " ")
+				if splitLine[0] != "pin" || len(splitLine) != 3 {
+					logrus.Fatalf("malformed version pin specification: %s", line)
+				}
+				pinSlice := strings.Split(splitLine[2], ":")
+				pinTag, err := tagcmp.Parse(pinSlice[1])
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				pin := imgPin{
+					image:   pinSlice[0],
+					version: pinTag,
+				}
+				servicePins[splitLine[1]] = pin
+			}
+			if err := scanner.Err(); err != nil {
+				logrus.Error(err)
+			}
+			versionsPresent = true
+		} else {
+			logrus.Debugf("did not find versions file for %s", recipe.Name)
+		}
+
+		for _, service := range recipe.Config.Services {
+			catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			image := reference.Path(img)
+			regVersions, err := client.GetRegistryTags(image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("retrieved '%s' from remote registry for '%s'", regVersions, image)
+
+			if strings.Contains(image, "library") {
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
+				// postgres:<tag>, i.e. images which do not have a username in the
+				// first position of the string
+				image = strings.Split(image, "/")[1]
+			}
+			semverLikeTag := true
+			if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+				logrus.Debugf("'%s' not considered semver-like", img.(reference.NamedTagged).Tag())
+				semverLikeTag = false
+			}
+
+			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
+			if err != nil && semverLikeTag {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("parsed '%s' for '%s'", tag, service.Name)
+			var compatible []tagcmp.Tag
+			for _, regVersion := range regVersions {
+				other, err := tagcmp.Parse(regVersion.Name)
+				if err != nil {
+					continue // skip tags that cannot be parsed
+				}
+
+				if tag.IsCompatible(other) && tag.IsLessThan(other) && !tag.Equals(other) {
+					compatible = append(compatible, other)
+				}
+			}
+
+			logrus.Debugf("detected potential upgradable tags '%s' for '%s'", compatible, service.Name)
+
+			sort.Sort(tagcmp.ByTagDesc(compatible))
+
+			if len(compatible) == 0 && semverLikeTag {
+				logrus.Info(fmt.Sprintf("no new versions available for '%s', '%s' is the latest", image, tag))
+				continue // skip on to the next tag and don't update any compose files
+			}
+
+			var compatibleStrings []string
+			for _, compat := range compatible {
+				skip := false
+				for _, catlVersion := range catlVersions {
+					if compat.String() == catlVersion {
+						skip = true
+					}
+				}
+				if !skip {
+					compatibleStrings = append(compatibleStrings, compat.String())
+				}
+			}
+
+			logrus.Debugf("detected compatible upgradable tags '%s' for '%s'", compatibleStrings, service.Name)
+
+			var upgradeTag string
+			_, ok := servicePins[service.Name]
+			if versionsPresent && ok {
+				pinnedTag := servicePins[service.Name].version
+				if tag.IsLessThan(pinnedTag) {
+					pinnedTagString := pinnedTag.String()
+					contains := false
+					for _, v := range compatible {
+						if pinnedTag.IsUpgradeCompatible(v) {
+							contains = true
+							upgradeTag = v.String()
+							break
+						}
+					}
+					if contains {
+						logrus.Infof("Upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString)
+					} else {
+						logrus.Infof("service %s, image %s pinned to %s. No compatible upgrade found", service.Name, servicePins[service.Name].image, pinnedTagString)
+						continue
+					}
+				} else {
+					logrus.Fatalf("Service %s is at version %s, but pinned to %s. Please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String())
+					continue
+				}
+			} else {
+				if bumpType != 0 {
+					for _, upTag := range compatible {
+						upElement, err := tag.UpgradeDelta(upTag)
+						if err != nil {
+							return err
+						}
+						delta := upElement.UpgradeType()
+						if delta <= bumpType {
+							upgradeTag = upTag.String()
+							break
+						}
+					}
+					if upgradeTag == "" {
+						logrus.Warnf("not upgrading from '%s' to '%s' for '%s', because the upgrade type is more serious than what user wants.", tag.String(), compatible[0].String(), image)
+						continue
+					}
+				} else {
+					msg := fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
+					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+						tag := img.(reference.NamedTagged).Tag()
+						logrus.Warning(fmt.Sprintf("unable to determine versioning semantics of '%s', listing all tags", tag))
+						msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
+						compatibleStrings = []string{}
+						for _, regVersion := range regVersions {
+							compatibleStrings = append(compatibleStrings, regVersion.Name)
+						}
+					}
+
+					prompt := &survey.Select{
+						Message: msg,
+						Options: compatibleStrings,
+					}
+					if err := survey.AskOne(prompt, &upgradeTag); err != nil {
+						logrus.Fatal(err)
+					}
+				}
+			}
+			if err := recipe.UpdateTag(image, upgradeTag); err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Infof("tag upgraded from '%s' to '%s' for '%s'", tag.String(), upgradeTag, image)
+		}
+
+		return nil
+	},
+}

cli/recipe/version.go

@@ -0,0 +1,45 @@
+package recipe
+
+import (
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeVersionCommand = &cli.Command{
+	Name:      "versions",
+	Usage:     "List recipe versions",
+	Aliases:   []string{"v"},
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		catalogue, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		recipeMeta, ok := catalogue[recipe.Name]
+		if !ok {
+			logrus.Fatalf("'%s' recipe doesn't exist?", recipe.Name)
+		}
+
+		tableCol := []string{"Version", "Service", "Image", "Tag", "Digest"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, serviceVersion := range recipeMeta.Versions {
+			for tag, meta := range serviceVersion {
+				for service, serviceMeta := range meta {
+					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Tag, serviceMeta.Digest})
+				}
+			}
+		}
+
+		table.SetAutoMergeCells(true)
+		table.Render()
+
+		return nil
+	},
+}

cli/record/list.go

@@ -0,0 +1,79 @@
+package record
+
+import (
+	"fmt"
+	"strconv"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"github.com/libdns/gandi"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// RecordListCommand lists domains.
+var RecordListCommand = &cli.Command{
+	Name:      "list",
+	Usage:     "List domain name records",
+	Aliases:   []string{"ls"},
+	ArgsUsage: "<zone>",
+	Flags: []cli.Flag{
+		internal.DNSProviderFlag,
+	},
+	Description: `
+This command lists all domain name records managed by a 3rd party provider for
+a specific zone.
+
+You must specify a zone (e.g. example.com) under which your domain name records
+are listed. This zone must already be created on your provider account.
+`,
+	Action: func(c *cli.Context) error {
+		if err := internal.EnsureDNSProvider(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		zone, err := internal.EnsureZoneArgument(c)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var provider gandi.Provider
+		switch internal.DNSProvider {
+		case "gandi":
+			provider, err = gandiPkg.New()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		default:
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+		}
+
+		records, err := provider.GetRecords(c.Context, zone)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableCol := []string{"type", "name", "value", "TTL", "priority"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		for _, record := range records {
+			value := record.Value
+			if len(record.Value) > 30 {
+				value = fmt.Sprintf("%s...", record.Value[:30])
+			}
+
+			table.Append([]string{
+				record.Type,
+				record.Name,
+				value,
+				record.TTL.String(),
+				strconv.Itoa(record.Priority),
+			})
+		}
+
+		table.Render()
+
+		return nil
+	},
+}

cli/record/new.go

@@ -0,0 +1,136 @@
+package record
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"github.com/libdns/gandi"
+	"github.com/libdns/libdns"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// RecordNewCommand creates a new domain name record.
+var RecordNewCommand = &cli.Command{
+	Name:      "new",
+	Usage:     "Create a new domain record",
+	Aliases:   []string{"n"},
+	ArgsUsage: "<zone>",
+	Flags: []cli.Flag{
+		internal.DNSProviderFlag,
+		internal.DNSTypeFlag,
+		internal.DNSNameFlag,
+		internal.DNSValueFlag,
+		internal.DNSTTLFlag,
+		internal.DNSPriorityFlag,
+	},
+	Description: `
+This command creates a new domain name record for a specific zone.
+
+You must specify a zone (e.g. example.com) under which your domain name records
+are listed. This zone must already be created on your provider account.
+
+Example:
+
+    abra record new foo.com -p gandi -t A -n myapp -v 192.168.178.44
+
+You may also invoke this command in "wizard" mode and be prompted for input
+
+    abra record new
+`,
+	Action: func(c *cli.Context) error {
+		zone, err := internal.EnsureZoneArgument(c)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := internal.EnsureDNSProvider(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		var provider gandi.Provider
+		switch internal.DNSProvider {
+		case "gandi":
+			provider, err = gandiPkg.New()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		default:
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+		}
+
+		if err := internal.EnsureDNSTypeFlag(c); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := internal.EnsureDNSNameFlag(c); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := internal.EnsureDNSValueFlag(c); err != nil {
+			logrus.Fatal(err)
+		}
+
+		record := libdns.Record{
+			Type:  internal.DNSType,
+			Name:  internal.DNSName,
+			Value: internal.DNSValue,
+			TTL:   time.Duration(internal.DNSTTL),
+		}
+
+		if internal.DNSType == "MX" || internal.DNSType == "SRV" || internal.DNSType == "URI" {
+			record.Priority = internal.DNSPriority
+		}
+
+		records, err := provider.GetRecords(c.Context, zone)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, existingRecord := range records {
+			if existingRecord.Type == record.Type &&
+				existingRecord.Name == record.Name &&
+				existingRecord.Value == record.Value {
+				logrus.Fatal("provider library reports that this record already exists?")
+			}
+		}
+
+		createdRecords, err := provider.SetRecords(
+			c.Context,
+			zone,
+			[]libdns.Record{record},
+		)
+
+		if len(createdRecords) == 0 {
+			logrus.Fatal("provider library reports that no record was created?")
+		}
+
+		createdRecord := createdRecords[0]
+
+		tableCol := []string{"type", "name", "value", "TTL", "priority"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		value := createdRecord.Value
+		if len(createdRecord.Value) > 30 {
+			value = fmt.Sprintf("%s...", createdRecord.Value[:30])
+		}
+
+		table.Append([]string{
+			createdRecord.Type,
+			createdRecord.Name,
+			value,
+			createdRecord.TTL.String(),
+			strconv.Itoa(createdRecord.Priority),
+		})
+
+		table.Render()
+
+		logrus.Info("record created")
+
+		return nil
+	},
+}

cli/record/record.go

@@ -0,0 +1,38 @@
+package record
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// RecordCommand supports managing DNS entries.
+var RecordCommand = &cli.Command{
+	Name:      "record",
+	Usage:     "Manage domain name records via 3rd party providers",
+	Aliases:   []string{"rc"},
+	ArgsUsage: "<record>",
+	Description: `
+This command supports managing domain name records via 3rd party providers such
+as Gandi DNS. It supports listing, creating and removing all types of records
+that you might need for managing Co-op Cloud apps.
+
+The following providers are supported:
+
+    Gandi DNS https://www.gandi.net
+
+You need an account with such a provider already. Typically, you need to
+provide an API token on the Abra command-line when using these commands so that
+you can authenticate with your provider account.
+
+New providers can be integrated, we welcome change sets. See the underlying DNS
+library documentation for more. It supports many existing providers and allows
+to implement new provider support easily.
+
+    https://pkg.go.dev/github.com/libdns/libdns
+
+`,
+	Subcommands: []*cli.Command{
+		RecordListCommand,
+		RecordNewCommand,
+		RecordRemoveCommand,
+	},
+}

cli/record/remove.go

@@ -0,0 +1,130 @@
+package record
+
+import (
+	"fmt"
+	"strconv"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	gandiPkg "coopcloud.tech/abra/pkg/dns/gandi"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/libdns/gandi"
+	"github.com/libdns/libdns"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// RecordRemoveCommand lists domains.
+var RecordRemoveCommand = &cli.Command{
+	Name:      "remove",
+	Usage:     "Remove a domain name record",
+	Aliases:   []string{"rm"},
+	ArgsUsage: "<zone>",
+	Flags: []cli.Flag{
+		internal.DNSProviderFlag,
+		internal.DNSTypeFlag,
+		internal.DNSNameFlag,
+	},
+	Description: `
+This command removes a domain name record for a specific zone.
+
+It uses the type of record and name to match existing records and choose one
+for deletion. You must specify a zone (e.g. example.com) under which your
+domain name records are listed. This zone must already be created on your
+provider account.
+
+Example:
+
+    abra record remove foo.com -p gandi -t A -n myapp
+
+You may also invoke this command in "wizard" mode and be prompted for input
+
+    abra record rm
+`,
+	Action: func(c *cli.Context) error {
+		zone, err := internal.EnsureZoneArgument(c)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := internal.EnsureDNSProvider(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		var provider gandi.Provider
+		switch internal.DNSProvider {
+		case "gandi":
+			provider, err = gandiPkg.New()
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		default:
+			logrus.Fatalf("'%s' is not a supported DNS provider", internal.DNSProvider)
+		}
+
+		if err := internal.EnsureDNSTypeFlag(c); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := internal.EnsureDNSNameFlag(c); err != nil {
+			logrus.Fatal(err)
+		}
+
+		records, err := provider.GetRecords(c.Context, zone)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var toDelete libdns.Record
+		for _, record := range records {
+			if record.Type == internal.DNSType && record.Name == internal.DNSName {
+				toDelete = record
+				break
+			}
+		}
+
+		if (libdns.Record{}) == toDelete {
+			logrus.Fatal("provider library reports no matching record?")
+		}
+
+		tableCol := []string{"type", "name", "value", "TTL", "priority"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		value := toDelete.Value
+		if len(toDelete.Value) > 30 {
+			value = fmt.Sprintf("%s...", toDelete.Value[:30])
+		}
+
+		table.Append([]string{
+			toDelete.Type,
+			toDelete.Name,
+			value,
+			toDelete.TTL.String(),
+			strconv.Itoa(toDelete.Priority),
+		})
+
+		table.Render()
+
+		response := false
+		prompt := &survey.Confirm{
+			Message: "continue with record deletion?",
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return err
+		}
+
+		if !response {
+			logrus.Fatal("exiting as requested")
+		}
+
+		_, err = provider.DeleteRecords(c.Context, zone, []libdns.Record{toDelete})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Info("record successfully deleted")
+
+		return nil
+	},
+}

cli/server/add.go

@@ -0,0 +1,510 @@
+package server
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"os/user"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/server"
+	"coopcloud.tech/abra/pkg/ssh"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sfreiberg/simplessh"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	dockerInstallMsg = `
+A docker installation cannot be found on %s. This is a required system
+dependency for running Co-op Cloud on your server. If you would like, Abra can
+attempt to install Docker for you using the upstream non-interactive
+installation script.
+
+See the following documentation for more:
+
+    https://docs.docker.com/engine/install/debian/#install-using-the-convenience-script
+
+N.B Docker doesn't recommend it for production environments but many use it for
+such purposes. Docker stable is now installed by default by this script. The
+source for this script can be seen here:
+
+    https://github.com/docker/docker-install
+
+`
+)
+
+var local bool
+var localFlag = &cli.BoolFlag{
+	Name:        "local",
+	Aliases:     []string{"l"},
+	Value:       false,
+	Usage:       "Use local server",
+	Destination: &local,
+}
+
+var provision bool
+var provisionFlag = &cli.BoolFlag{
+	Name:        "provision",
+	Aliases:     []string{"p"},
+	Value:       false,
+	Usage:       "Provision server so it can deploy apps",
+	Destination: &provision,
+}
+
+var sshAuth string
+var sshAuthFlag = &cli.StringFlag{
+	Name:        "ssh-auth",
+	Aliases:     []string{"sh"},
+	Value:       "identity-file",
+	Usage:       "Select SSH authentication method (identity-file, password)",
+	Destination: &sshAuth,
+}
+
+var askSudoPass bool
+var askSudoPassFlag = &cli.BoolFlag{
+	Name:        "ask-sudo-pass",
+	Aliases:     []string{"as"},
+	Value:       false,
+	Usage:       "Ask for sudo password",
+	Destination: &askSudoPass,
+}
+
+var traefik bool
+var traefikFlag = &cli.BoolFlag{
+	Name:        "traefik",
+	Aliases:     []string{"t"},
+	Value:       false,
+	Usage:       "Deploy traefik",
+	Destination: &traefik,
+}
+
+func cleanUp(domainName string) {
+	logrus.Warnf("cleaning up context for %s", domainName)
+	if err := client.DeleteContext(domainName); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Warnf("cleaning up server directory for %s", domainName)
+	if err := os.RemoveAll(filepath.Join(config.ABRA_SERVER_FOLDER, domainName)); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func installDockerLocal(c *cli.Context) error {
+	fmt.Println(fmt.Sprintf(dockerInstallMsg, "this local server"))
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: fmt.Sprintf("attempt install docker on local server?"),
+	}
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	cmd := exec.Command("bash", "-c", "curl -s https://get.docker.com | bash")
+	if err := internal.RunCmd(cmd); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newLocalServer(c *cli.Context, domainName string) error {
+	if err := createServerDir(domainName); err != nil {
+		return err
+	}
+
+	cl, err := newClient(c, domainName)
+	if err != nil {
+		return err
+	}
+
+	if provision {
+		out, err := exec.Command("which", "docker").Output()
+		if err != nil {
+			return err
+		}
+		if string(out) == "" {
+			if err := installDockerLocal(c); err != nil {
+				return err
+			}
+		}
+		if err := initSwarmLocal(c, cl, domainName); err != nil {
+			if !strings.Contains(err.Error(), "proxy already exists") {
+				logrus.Fatal(err)
+			}
+		}
+	}
+
+	if traefik {
+		if err := deployTraefik(c, cl, domainName); err != nil {
+			return err
+		}
+	}
+
+	logrus.Info("local server has been added")
+
+	return nil
+}
+
+func newContext(c *cli.Context, domainName, username, port string) error {
+	store := client.NewDefaultDockerContextStore()
+	contexts, err := store.Store.List()
+	if err != nil {
+		return err
+	}
+
+	for _, context := range contexts {
+		if context.Name == domainName {
+			logrus.Debugf("context for %s already exists", domainName)
+			return nil
+		}
+	}
+
+	logrus.Debugf("creating context with domain %s, username %s and port %s", domainName, username, port)
+
+	if err := client.CreateContext(domainName, username, port); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newClient(c *cli.Context, domainName string) (*dockerClient.Client, error) {
+	cl, err := client.New(domainName)
+	if err != nil {
+		return &dockerClient.Client{}, err
+	}
+	return cl, nil
+}
+
+func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *simplessh.Client, domainName string) error {
+	result, err := sshCl.Exec("which docker")
+	if err != nil && string(result) != "" {
+		return err
+	}
+
+	if string(result) == "" {
+		fmt.Println(fmt.Sprintf(dockerInstallMsg, domainName))
+
+		response := false
+		prompt := &survey.Confirm{
+			Message: fmt.Sprintf("attempt install docker on %s?", domainName),
+		}
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return err
+		}
+		if !response {
+			logrus.Fatal("exiting as requested")
+		}
+
+		cmd := "curl -s https://get.docker.com | bash"
+
+		var sudoPass string
+		if askSudoPass {
+			prompt := &survey.Password{
+				Message: "sudo password?",
+			}
+			if err := survey.AskOne(prompt, &sudoPass); err != nil {
+				return err
+			}
+			logrus.Debugf("running '%s' on %s now with sudo password", cmd, domainName)
+			if err := ssh.RunSudoCmd(cmd, sudoPass, sshCl); err != nil {
+				return err
+			}
+		} else {
+			logrus.Debugf("running '%s' on %s now without sudo password", cmd, domainName)
+			if err := ssh.Exec(cmd, sshCl); err != nil {
+				return err
+			}
+		}
+	}
+
+	logrus.Infof("docker is installed on %s", domainName)
+
+	return nil
+}
+
+func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string) error {
+	initReq := swarm.InitRequest{ListenAddr: "0.0.0.0:2377"}
+	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
+		if !strings.Contains(err.Error(), "is already part of a swarm") {
+			return err
+		}
+		logrus.Info("swarm mode already initialised on local server")
+	} else {
+		logrus.Infof("initialised swarm mode on local server")
+	}
+
+	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
+	if _, err := cl.NetworkCreate(c.Context, "proxy", netOpts); err != nil {
+		if !strings.Contains(err.Error(), "proxy already exists") {
+			return err
+		}
+		logrus.Info("swarm overlay network already created on local server")
+	} else {
+		logrus.Infof("swarm overlay network created on local server")
+	}
+
+	return nil
+}
+
+func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error {
+	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
+	freifunkDNS := "5.1.66.255:53"
+
+	resolver := &net.Resolver{
+		PreferGo: false,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: time.Millisecond * time.Duration(10000),
+			}
+			return d.DialContext(ctx, "udp", freifunkDNS)
+		},
+	}
+
+	logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
+
+	ips, err := resolver.LookupIPAddr(c.Context, domainName)
+	if err != nil {
+		return err
+	}
+
+	if len(ips) == 0 {
+		return fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
+	}
+
+	ipv4 := ips[0].IP.To4().String()
+	logrus.Debugf("discovered the following ipv4 addr: %s", ipv4)
+
+	initReq := swarm.InitRequest{
+		ListenAddr:    "0.0.0.0:2377",
+		AdvertiseAddr: ipv4,
+	}
+	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
+		if !strings.Contains(err.Error(), "is already part of a swarm") {
+			return err
+		}
+		logrus.Infof("swarm mode already initialised on %s", domainName)
+	} else {
+		logrus.Infof("initialised swarm mode on %s", domainName)
+	}
+
+	netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
+	if _, err := cl.NetworkCreate(c.Context, "proxy", netOpts); err != nil {
+		if !strings.Contains(err.Error(), "proxy already exists") {
+			return err
+		}
+		logrus.Infof("swarm overlay network already created on %s", domainName)
+	} else {
+		logrus.Infof("swarm overlay network created on %s", domainName)
+	}
+
+	return nil
+}
+
+func createServerDir(domainName string) error {
+	if err := server.CreateServerDir(domainName); err != nil {
+		if !os.IsExist(err) {
+			return err
+		}
+		logrus.Debugf("server dir for %s already created", domainName)
+	}
+	return nil
+}
+
+func deployTraefik(c *cli.Context, cl *dockerClient.Client, domainName string) error {
+	internal.NoInput = true
+
+	internal.RecipeName = "traefik"
+	internal.NewAppServer = domainName
+	internal.Domain = fmt.Sprintf("%s.%s", "traefik", domainName)
+	internal.NewAppName = fmt.Sprintf("%s_%s", "traefik", config.SanitiseAppName(domainName))
+
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", internal.Domain, fmt.Sprintf("%s.env", internal.NewAppName))
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
+		fmt.Println(fmt.Sprintf(`
+	You specified "--traefik/-t" and that means that Abra will now try to
+	automatically create a new Traefik app on %s.
+	`, internal.NewAppServer))
+
+		tableCol := []string{"recipe", "domain", "server", "name"}
+		table := abraFormatter.CreateTable(tableCol)
+		table.Append([]string{internal.RecipeName, internal.Domain, internal.NewAppServer, internal.NewAppName})
+
+		if err := internal.NewAction(c); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		logrus.Infof("%s already exists, not creating again", appEnvPath)
+	}
+
+	internal.AppName = internal.NewAppName
+	if err := internal.DeployAction(c); err != nil {
+		logrus.Fatal(err)
+	}
+
+	return nil
+}
+
+var serverAddCommand = &cli.Command{
+	Name:  "add",
+	Usage: "Add a server to your configuration",
+	Description: `
+This command adds a new server to your configuration so that it can be managed
+by Abra. This can be useful when you already have a server provisioned and want
+to start running Abra commands against it. 
+
+This command can also provision your server ("--provision/-p") so that it is
+capable of hosting Co-op Cloud apps. Abra will default to expecting that you
+have a running ssh-agent and are using SSH keys to connect to your new server.
+Abra will also read your SSH config (matching "Host" as <domain>). SSH
+connection details precedence follows as such: command-line > SSH config >
+guessed defaults.
+
+If you have no SSH key configured for this host and are instead using password
+authentication, you may pass "--ssh-auth password" to have Abra ask you for the
+password. "--ask-sudo-pass" may be passed if you run your provisioning commands
+via sudo privilege escalation.
+
+If "--local" is passed, then Abra assumes that the current local server is
+intended as the target server. This is useful when you want to have your entire
+Co-op Cloud config located on the server itself, and not on your local
+developer machine.
+
+Example:
+
+    abra server add --local
+
+Otherwise, you may specify a remote server. The <domain> argument must be a
+publicy accessible domain name which points to your server. You should have SSH
+access to this server, Abra will assume port 22 and will use your current
+system username to make an initial connection. You can use the <user> and
+<port> arguments to adjust this.
+
+Example:
+
+    abra server add --provision --traefik varia.zone glodemodem 12345
+
+Abra will construct the following SSH connection and Docker context:
+
+    ssh://globemodem@varia.zone:12345
+
+All communication between Abra and the server will use this SSH connection.
+
+In this example, Abra will run the following operations:
+
+    1. Install Docker
+		2. Initialise Swarm mode
+		3. Deploy Traefik (core web proxy)
+
+You may omit flags to avoid performing this provisioning logic.
+`,
+	Aliases: []string{"a"},
+	Flags: []cli.Flag{
+		localFlag,
+		provisionFlag,
+		sshAuthFlag,
+		askSudoPassFlag,
+		traefikFlag,
+	},
+	ArgsUsage: "<domain> [<user>] [<port>]",
+	Action: func(c *cli.Context) error {
+		if c.Args().Len() > 0 && local || !internal.ValidateSubCmdFlags(c) {
+			err := errors.New("cannot use '<domain>' and '--local' together")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		if sshAuth != "password" && sshAuth != "identity-file" {
+			err := errors.New("--ssh-auth only accepts 'identity-file' or 'password'")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		if local {
+			if err := newLocalServer(c, "default"); err != nil {
+				logrus.Fatal(err)
+			}
+			return nil
+		}
+
+		domainName, err := internal.ValidateDomain(c)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		username := c.Args().Get(1)
+		if username == "" {
+			systemUser, err := user.Current()
+			if err != nil {
+				return err
+			}
+			username = systemUser.Username
+		}
+
+		port := c.Args().Get(2)
+		if port == "" {
+			port = "22"
+		}
+
+		if err := createServerDir(domainName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := newContext(c, domainName, username, port); err != nil {
+			logrus.Fatal(err)
+		}
+
+		cl, err := newClient(c, domainName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if provision {
+			logrus.Debugf("attempting to construct SSH client for %s", domainName)
+			sshCl, err := ssh.New(domainName, sshAuth, username, port)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer sshCl.Close()
+			logrus.Debugf("successfully created SSH client for %s", domainName)
+
+			if err := installDocker(c, cl, sshCl, domainName); err != nil {
+				logrus.Fatal(err)
+			}
+			if err := initSwarm(c, cl, domainName); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		if _, err := cl.Info(c.Context); err != nil {
+			cleanUp(domainName)
+			logrus.Fatalf("couldn't make a remote docker connection to %s? use --provision/-p to attempt to install", domainName)
+		}
+
+		if traefik {
+			if err := deployTraefik(c, cl, domainName); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}

cli/server/list.go

@@ -0,0 +1,64 @@
+package server
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/cli/cli/connhelper/ssh"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var serverListCommand = &cli.Command{
+	Name:      "list",
+	Aliases:   []string{"ls"},
+	Usage:     "List managed servers",
+	ArgsUsage: " ",
+	HideHelp:  true,
+	Action: func(c *cli.Context) error {
+		dockerContextStore := client.NewDefaultDockerContextStore()
+		contexts, err := dockerContextStore.Store.List()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableColumns := []string{"name", "host", "user", "port"}
+		table := formatter.CreateTable(tableColumns)
+		defer table.Render()
+
+		serverNames, err := config.ReadServerNames()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, serverName := range serverNames {
+			var row []string
+			for _, ctx := range contexts {
+				endpoint, err := client.GetContextEndpoint(ctx)
+				if err != nil && strings.Contains(err.Error(), "does not exist") {
+					// No local context found, we can continue safely
+					continue
+				}
+				if ctx.Name == serverName {
+					sp, err := ssh.ParseURL(endpoint)
+					if err != nil {
+						logrus.Fatal(err)
+					}
+					row = []string{serverName, sp.Host, sp.User, sp.Port}
+				}
+			}
+			if len(row) == 0 {
+				if serverName == "default" {
+					row = []string{serverName, "local", "n/a", "n/a"}
+				} else {
+					row = []string{serverName, "unknown", "unknown", "unknown"}
+				}
+			}
+			table.Append(row)
+		}
+
+		return nil
+	},
+}

cli/server/new.go

@@ -0,0 +1,236 @@
+package server
+
+import (
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/libcapsul"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+func newHetznerCloudVPS(c *cli.Context) error {
+	if err := internal.EnsureNewHetznerCloudVPSFlags(c); err != nil {
+		return err
+	}
+
+	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
+
+	var sshKeysRaw []string
+	var sshKeys []*hcloud.SSHKey
+	for _, sshKey := range c.StringSlice("hetzner-ssh-keys") {
+		if sshKey == "" {
+			continue
+		}
+
+		sshKey, _, err := client.SSHKey.GetByName(c.Context, sshKey)
+		if err != nil {
+			return err
+		}
+		sshKeys = append(sshKeys, sshKey)
+		sshKeysRaw = append(sshKeysRaw, sshKey.Name)
+	}
+
+	serverOpts := hcloud.ServerCreateOpts{
+		Name:       internal.HetznerCloudName,
+		ServerType: &hcloud.ServerType{Name: internal.HetznerCloudType},
+		Image:      &hcloud.Image{Name: internal.HetznerCloudImage},
+		SSHKeys:    sshKeys,
+		Location:   &hcloud.Location{Name: internal.HetznerCloudLocation},
+	}
+
+	tableColumns := []string{"name", "type", "image", "ssh-keys", "location"}
+	table := formatter.CreateTable(tableColumns)
+	table.Append([]string{
+		internal.HetznerCloudName,
+		internal.HetznerCloudType,
+		internal.HetznerCloudImage,
+		strings.Join(sshKeysRaw, "\n"),
+		internal.HetznerCloudLocation,
+	})
+	table.Render()
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with hetzner cloud VPS creation?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	res, _, err := client.Server.Create(c.Context, serverOpts)
+	if err != nil {
+		return err
+	}
+
+	var rootPassword string
+	if len(sshKeys) > 0 {
+		rootPassword = "N/A (using SSH keys)"
+	} else {
+		rootPassword = res.RootPassword
+	}
+
+	ip := res.Server.PublicNet.IPv4.IP.String()
+
+	fmt.Println(fmt.Sprintf(`
+Your new Hetzner Cloud VPS has successfully been created! Here are the details:
+
+    name: %s
+    IP address: %s
+    root password: %s
+
+You can access this new VPS via SSH using the following command:
+
+    ssh root@%s
+
+Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
+not list this server)! You will need to assign a domain name record ("abra
+record new") and add the server to your Abra configuration ("abra server add")
+to have a working server that you can deploy Co-op Cloud apps to.
+	`,
+		internal.HetznerCloudName, ip, rootPassword,
+		ip,
+	))
+
+	return nil
+}
+
+func newCapsulVPS(c *cli.Context) error {
+	if err := internal.EnsureNewCapsulVPSFlags(c); err != nil {
+		return err
+	}
+
+	capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", internal.CapsulInstanceURL)
+
+	var sshKeys []string
+	for _, sshKey := range c.StringSlice("capsul-ssh-keys") {
+		if sshKey == "" {
+			continue
+		}
+		sshKeys = append(sshKeys, sshKey)
+	}
+
+	tableColumns := []string{"instance", "name", "type", "image", "ssh-keys"}
+	table := formatter.CreateTable(tableColumns)
+	table.Append([]string{
+		internal.CapsulInstanceURL,
+		internal.CapsulName,
+		internal.CapsulType,
+		internal.CapsulImage,
+		strings.Join(sshKeys, "\n"),
+	})
+	table.Render()
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with capsul creation?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	capsulClient := libcapsul.New(capsulCreateURL, internal.CapsulAPIToken)
+	resp, err := capsulClient.Create(
+		internal.CapsulName,
+		internal.CapsulType,
+		internal.CapsulImage,
+		sshKeys,
+	)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(fmt.Sprintf(`
+Your new Capsul has successfully been created! Here are the details:
+
+    Capsul name: %s
+    Capsul ID:   %v
+
+You will need to log into your Capsul instance web interface to retrieve the IP
+address. You can learn all about how to get SSH access to your new Capsul on:
+
+    %s/about-ssh
+
+Please note, this server is not managed by Abra yet (i.e. "abra server ls" will
+not list this server)! You will need to assign a domain name record ("abra
+record new") and add the server to your Abra configuration ("abra server add")
+to have a working server that you can deploy Co-op Cloud apps to.
+	`, internal.CapsulName, resp.ID, internal.CapsulInstanceURL))
+
+	return nil
+}
+
+var serverNewCommand = &cli.Command{
+	Name:    "new",
+	Aliases: []string{"n"},
+	Usage:   "Create a new server using a 3rd party provider",
+	Description: `
+This command creates a new server via a 3rd party provider.
+
+The following providers are supported:
+
+		Capsul https://git.cyberia.club/Cyberia/capsul-flask
+    Hetzner Cloud https://docs.hetzner.com/cloud
+
+You may invoke this command in "wizard" mode and be prompted for input:
+
+    abra record new
+
+API tokens are read from the environment if specified, e.g.
+
+    export HCLOUD_TOKEN=...
+
+Where "$provider_TOKEN" is the expected env var format.
+`,
+	ArgsUsage: "<provider>",
+	Flags: []cli.Flag{
+		internal.ServerProviderFlag,
+
+		// Capsul
+		internal.CapsulInstanceURLFlag,
+		internal.CapsulTypeFlag,
+		internal.CapsulImageFlag,
+		internal.CapsulSSHKeysFlag,
+		internal.CapsulAPITokenFlag,
+
+		// Hetzner
+		internal.HetznerCloudNameFlag,
+		internal.HetznerCloudTypeFlag,
+		internal.HetznerCloudImageFlag,
+		internal.HetznerCloudSSHKeysFlag,
+		internal.HetznerCloudLocationFlag,
+		internal.HetznerCloudAPITokenFlag,
+	},
+	Action: func(c *cli.Context) error {
+		if err := internal.EnsureServerProvider(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		switch internal.ServerProvider {
+		case "capsul":
+			if err := newCapsulVPS(c); err != nil {
+				logrus.Fatal(err)
+			}
+		case "hetzner-cloud":
+			if err := newHetznerCloudVPS(c); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}

cli/server/remove.go

@@ -0,0 +1,159 @@
+package server
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var rmServer bool
+var rmServerFlag = &cli.BoolFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       false,
+	Usage:       "remove the actual server also",
+	Destination: &rmServer,
+}
+
+func rmHetznerCloudVPS(c *cli.Context) error {
+	if internal.HetznerCloudName == "" && !internal.NoInput {
+		prompt := &survey.Input{
+			Message: "specify hetzner cloud VPS name",
+		}
+		if err := survey.AskOne(prompt, &internal.HetznerCloudName); err != nil {
+			return err
+		}
+	}
+
+	if internal.HetznerCloudAPIToken == "" && !internal.NoInput {
+		token, ok := os.LookupEnv("HCLOUD_TOKEN")
+		if !ok {
+			prompt := &survey.Input{
+				Message: "specify hetzner cloud API token",
+			}
+			if err := survey.AskOne(prompt, &internal.HetznerCloudAPIToken); err != nil {
+				return err
+			}
+		} else {
+			internal.HetznerCloudAPIToken = token
+		}
+	}
+
+	client := hcloud.NewClient(hcloud.WithToken(internal.HetznerCloudAPIToken))
+
+	server, _, err := client.Server.Get(c.Context, internal.HetznerCloudName)
+	if err != nil {
+		return err
+	}
+
+	if server == nil {
+		logrus.Fatalf("library provider reports that %s doesn't exist?", internal.HetznerCloudName)
+	}
+
+	fmt.Println(fmt.Sprintf(`
+You have requested that Abra delete the following server (%s). Please be
+absolutely sure that this is indeed the server that you would like to have
+removed. There will be no going back once you confirm, the server will be
+destroyed.
+`, server.Name))
+
+	tableColumns := []string{"name", "type", "image", "location"}
+	table := formatter.CreateTable(tableColumns)
+	table.Append([]string{
+		server.Name,
+		server.ServerType.Name,
+		server.Image.Name,
+		server.Datacenter.Name,
+	})
+	table.Render()
+
+	response := false
+	prompt := &survey.Confirm{
+		Message: "continue with hetzner cloud VPS removal?",
+	}
+
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		logrus.Fatal("exiting as requested")
+	}
+
+	_, err = client.Server.Delete(c.Context, server)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("%s has been deleted from your hetzner cloud account", internal.HetznerCloudName)
+
+	return nil
+}
+
+var serverRemoveCommand = &cli.Command{
+	Name:      "remove",
+	Aliases:   []string{"rm"},
+	ArgsUsage: "<server>",
+	Usage:     "Remove a managed server",
+	Description: `
+This command removes a server from Abra management.
+
+Depending on whether you used a 3rd party provider to create this server ("abra
+server new"), you can also destroy the virtual server as well. Pass
+"--server/-s" to tell Abra to try to delete this VPS.
+
+Otherwise, Abra will remove the internal bookkeeping (~/.abra/servers/...) and
+underlying client connection context. This server will then be lost in time,
+like tears in rain.
+`,
+	Flags: []cli.Flag{
+		rmServerFlag,
+
+		// Hetzner
+		internal.HetznerCloudNameFlag,
+		internal.HetznerCloudAPITokenFlag,
+	},
+	Action: func(c *cli.Context) error {
+		serverName, err := internal.ValidateServer(c)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if rmServer {
+			if err := internal.EnsureServerProvider(); err != nil {
+				logrus.Fatal(err)
+			}
+
+			switch internal.ServerProvider {
+			case "capsul":
+				logrus.Warn("capsul provider does not support automatic removal yet, sorry!")
+			case "hetzner-cloud":
+				if err := rmHetznerCloudVPS(c); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+		}
+
+		if err := client.DeleteContext(serverName); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := os.RemoveAll(filepath.Join(config.ABRA_SERVER_FOLDER, serverName)); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Infof("server at '%s' has been lost in time, like tears in rain", serverName)
+
+		return nil
+	},
+}

cli/server/server.go

@@ -0,0 +1,27 @@
+package server
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// ServerCommand defines the `abra server` command and its subcommands
+var ServerCommand = &cli.Command{
+	Name:    "server",
+	Aliases: []string{"s"},
+	Usage:   "Manage servers via 3rd party providers",
+	Description: `
+These commands support creating, managing and removing servers using 3rd party
+integrations. 
+
+Servers can be created from scratch using the "abra server new" command. If you
+already have a server, you can add it to your configuration using "abra server
+add". Abra can provision servers so that they are ready to deploy Co-op Cloud
+apps, see available flags on "server add" for more.
+`,
+	Subcommands: []*cli.Command{
+		serverNewCommand,
+		serverAddCommand,
+		serverListCommand,
+		serverRemoveCommand,
+	},
+}

cli/upgrade.go

@@ -0,0 +1,23 @@
+package cli
+
+import (
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = &cli.Command{
+	Name:  "upgrade",
+	Usage: "Upgrade abra",
+	Action: func(c *cli.Context) error {
+		cmd := exec.Command("bash", "-c", "curl -s https://install.abra.coopcloud.tech | bash")
+		logrus.Debugf("attempting to run '%s'", cmd)
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+		return nil
+	},
+}

cmd/abra/main.go

@@ -0,0 +1,24 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli"
+)
+
+// Version is the current version of abra.
+var Version string
+
+// Commit is the current commit of abra.
+var Commit string
+
+func main() {
+	// If not set in the ld-flags
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	cli.RunApp(Version, Commit)
+}

completion/_abra

@@ -1,52 +0,0 @@
-#compdef abra
-
-_abra () {
-	local context state line curcontext="$curcontext" ret=1
-	_arguments -n : \
-		{-h,--help}'[Help message]' \
-		'1:commands:(app server)' \
-		'*::arguments:->arguments' \
-		&& ret=0
-
-	case $state in
-		(arguments)
-			curcontext="${curcontext%:*:*}:abra-arguments-$words[1]:"
-			case $words[1] in
-				(app)
-					_arguments \
-						'1: :_abra_apps' \
-					&& ret=0
-				;;
-				(server)
-					_arguments \
-						'1:servers:_abra_servers' \
-					&& ret=0
-				;;
-			esac
-		;;
-	esac
-
-	return ret
-}
-
-_abra_servers() {
-	_path_files -/W $HOME/.abra/servers
-}
-
-_abra_apps()
-{
-	local newapps apps=($HOME/.abra/servers/*/*.env)
-	typeset -a apps
-	newapps=()
-	for app in $apps; do
-		newapps+=($(_abra_basename "${app}"))
-	done
-	_describe -t apps 'app' newapps
-}
-
-_abra_basename()
-{
-    printf -- "${1##*/}"
-}
-
-_abra "$@"

completion/abra.bash

@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-_abra_basename()
-{
-    echo "${1##*/}"
-}
-
-_abra_servers()
-{
-    # FIXME 3wc: copied from abra/get_servers()
-    shopt -s nullglob dotglob
-    local SERVERS=(~/.abra/servers/*)
-    shopt -u nullglob dotglob
-
-    for SERVER in "${SERVERS[@]}"; do
-        _abra_basename "${SERVER}"
-    done
-}
-
-_abra_complete_servers()
-{
-    mapfile -t COMPREPLY < <(compgen -W "$(_abra_servers)" -- "$1")
-}
-
-_abra_apps()
-{
-    shopt -s nullglob dotglob
-    local APPS=(~/.abra/servers/*/*.env)
-    shopt -u nullglob dotglob
-
-    for APP in "${APPS[@]}"; do
-        _abra_basename "${APP%.env}"
-    done
-}
-
-_abra_complete_apps()
-{
-    mapfile -t COMPREPLY < <(compgen -W "$(_abra_apps)" -- "$1")
-}
-
-_abra_complete()
-{
-    compopt +o default +o nospace
-    COMPREPLY=()
-
-    local -r cmds='
-        app
-        server
-    '
-    local -r short_opts='-e      -h     -s      -v'
-    local -r long_opts='--env --help --stack --version'
-
-    # Scan through the command line and find the abra command
-    # (if present), as well as its expected position.
-    local cmd
-    local cmd_index=1 # Expected index of the command token.
-    local i
-    for (( i = 1; i < ${#COMP_WORDS[@]}; i++ )); do
-        local word="${COMP_WORDS[i]}"
-        case "$word" in
-            -*)
-                ((cmd_index++))
-                ;;
-            *)
-                cmd="$word"
-                break
-                ;;
-        esac
-    done
-
-    local cur="${COMP_WORDS[COMP_CWORD]}"
-
-    if (( COMP_CWORD < cmd_index )); then
-        # Offer option completions.
-        case "$cur" in
-            --*)
-                mapfile -t COMPREPLY < <(compgen -W "$long_opts" -- "$cur")
-                ;;
-            -*)
-                mapfile -t COMPREPLY < <(compgen -W "$short_opts" -- "$cur")
-                ;;
-            *)
-                # Skip completion; we should never get here.
-                ;;
-        esac
-    elif (( COMP_CWORD == cmd_index )); then
-        # Offer command name completions.
-        mapfile -t COMPREPLY < <(compgen -W "$cmds" -- "$cur")
-    else
-        # Offer command argument completions.
-        case "$cmd" in
-            server)
-                # Offer exactly one server name completion.
-                if (( COMP_CWORD == cmd_index + 1 )); then
-                    _abra_complete_servers "$cur"
-                fi
-                ;;
-            app)
-                # Offer exactly one app completion.
-                if (( COMP_CWORD == cmd_index + 1 )); then
-                    _abra_complete_apps "$cur"
-                fi
-                ;;
-            #help)
-            #    # Offer exactly one command name completion.
-            #    if (( COMP_CWORD == cmd_index + 1 )); then
-            #        COMPREPLY=($(compgen -W "$cmds" -- "$cur"))
-            #    fi
-            #    ;;
-            *)
-                # Unknown command or unknowable argument.
-                ;;
-        esac
-    fi
-}
-
-complete -o default -F _abra_complete abra

deploy/abra-apps.cloud.autonomic.zone/compose.yml

@@ -1,38 +0,0 @@
----
-version: "3.8"
-
-services:
-  app:
-    image: "nginx:stable"
-    configs:
-      - source: abra_conf
-        target: /etc/nginx/conf.d/abra.conf
-      - source: abra_apps_json
-        target: /var/www/abra-apps/abra-apps.json
-    volumes:
-      - "public:/var/www/abra-apps"
-    networks:
-      - proxy
-    deploy:
-      update_config:
-        failure_action: rollback
-        order: start-first
-      labels:
-        - "traefik.enable=true"
-        - "traefik.http.services.abra-apps.loadbalancer.server.port=80"
-        - "traefik.http.routers.abra-apps.rule=Host(`abra-apps.cloud.autonomic.zone`)"
-        - "traefik.http.routers.abra-apps.entrypoints=web-secure"
-        - "traefik.http.routers.abra-apps.tls.certresolver=production"
-
-configs:
-  abra_apps_json:
-    file: abra-apps.json
-  abra_conf:
-    file: nginx.conf
-
-networks:
-  proxy:
-    external: true
-
-volumes:
-  public:

deploy/abra-apps.cloud.autonomic.zone/nginx.conf

@@ -1,10 +0,0 @@
-server {
-  listen         80 default_server;
-  server_name    abra-apps.cloud.autonomic.zone;
-
-  location / {
-    root /var/www/abra-apps;
-    add_header Content-Type application/json;
-    index abra-apps.json;
-  }
-}

deploy/install.abra.autonomic.zone/installer

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-ABRA_VERSION="0.7.2"
-GIT_URL="https://git.autonomic.zone/coop-cloud/abra"
-ABRA_SRC="$GIT_URL/raw/tag/$ABRA_VERSION/abra"
-ABRA_DIR="${ABRA_DIR:-$HOME/.abra/}"
-
-function install_abra_release {
-  mkdir -p "$HOME/.local/bin"
-  curl "$ABRA_SRC" > "$HOME/.local/bin/abra"
-  chmod +x "$HOME/.local/bin/abra"
-  echo "abra installed to $HOME/.local/bin/abra"
-}
-
-function install_abra_dev {
-  mkdir -p "$ABRA_DIR/"
-  if [[ ! -d "$ABRA_DIR/src" ]]; then
-    git clone "$GIT_URL" "$ABRA_DIR/src"
-  fi
-  (cd "$ABRA_DIR/src" && git pull origin main && cd - || exit)
-  mkdir -p "$HOME/.local/bin"
-  ln -sf "$ABRA_DIR/src/abra" "$HOME/.local/bin/abra"
-  echo "abra installed to $HOME/.local/bin/abra (development bleeding edge)"
-}
-
-function run_installation {
-  if [ "$1" = "--dev" ]; then
-    install_abra_dev
-  else
-    install_abra_release
-  fi
-}
-
-run_installation "$@"
-exit 0

go.mod

@@ -0,0 +1,48 @@
+module coopcloud.tech/abra
+
+go 1.16
+
+require (
+	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	github.com/AlecAivazis/survey/v2 v2.3.1
+	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
+	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
+	github.com/docker/cli v20.10.8+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/go-units v0.4.0
+	github.com/go-git/go-git/v5 v5.4.2
+	github.com/hetznercloud/hcloud-go v1.32.0
+	github.com/moby/sys/signal v0.5.0
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/pkg/errors v0.9.1
+	github.com/schollz/progressbar/v3 v3.8.3
+	github.com/schultz-is/passgen v1.0.1
+	github.com/sirupsen/logrus v1.8.1
+	github.com/urfave/cli/v2 v2.3.0
+	gotest.tools/v3 v3.0.3
+)
+
+require (
+	coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
+	github.com/Microsoft/hcsshim v0.8.21 // indirect
+	github.com/containerd/containerd v1.5.5 // indirect
+	github.com/davidmz/go-pageant v1.0.2 // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351
+	github.com/libdns/gandi v1.0.2
+	github.com/libdns/libdns v0.2.1
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
+	github.com/pkg/sftp v1.13.4 // indirect
+	github.com/sfreiberg/simplessh v0.0.0-20180301191542-495cbb862a9c
+	github.com/theupdateframework/notary v0.7.0 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0
+)

makefile

@@ -1,58 +0,0 @@
-.PHONY: test shellcheck docopt kcov codecov release-installer
-
-test:
-	@sudo DOCKER_CONTEXT=default docker run \
-		-v $$(pwd):/workdir \
-		--privileged \
-		-d \
-		--name=abra-test-dind \
-		-e DOCKER_TLS_CERTDIR="" \
-		decentral1se/docker-dind-bats-kcov
-	@DOCKER_CONTEXT=default docker exec \
-		-it \
-		abra-test-dind \
-		sh -c "cd /workdir && bats /workdir/tests"
-	@DOCKER_CONTEXT=default docker stop abra-test-dind
-	@DOCKER_CONTEXT=default docker rm abra-test-dind
-
-shellcheck:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		koalaman/shellcheck-alpine \
-		shellcheck /workdir/abra && \
-		shellcheck /workdir/bin/*.sh
-
-docopt:
-	@if [ ! -d ".venv" ]; then \
-		python3 -m venv .venv && \
-		.venv/bin/pip install -U pip setuptools wheel && \
-		.venv/bin/pip install docopt-sh; \
-	fi
-	.venv/bin/docopt.sh abra
-
-kcov:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		kcov/kcov:latest \
-		sh -c "kcov /workdir/coverage /workdir/abra || true"
-
-codecov: SHELL:=/bin/bash
-codecov:
-	@bash <(curl -s https://codecov.io/bash) \
-		-s coverage -t $$(pass show hosts/swarm.autonomic.zone/drone/codecov/token)
-
-release-installer:
-	@DOCKER_CONTEXT=swarm.autonomic.zone \
-		docker stack rm abra-installer-script && \
-		cd deploy/install.abra.autonomic.zone && \
-		DOCKER_CONTEXT=swarm.autonomic.zone docker stack deploy -c compose.yml abra-installer-script
-
-release-abra-apps:
-	@DOCKER_CONTEXT=swarm.autonomic.zone \
-		docker stack rm abra-apps-json && \
-		cd deploy/abra-apps.cloud.autonomic.zone && \
-		DOCKER_CONTEXT=swarm.autonomic.zone docker stack deploy -c compose.yml abra-apps-json

pkg/app/app.go

@@ -0,0 +1,85 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	apiclient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// Get retrieves an app
+func Get(appName string) (config.App, error) {
+	files, err := config.LoadAppFiles("")
+	if err != nil {
+		return config.App{}, err
+	}
+
+	app, err := config.GetApp(files, appName)
+	if err != nil {
+		return config.App{}, err
+	}
+
+	logrus.Debugf("retrieved '%s' for '%s'", app, appName)
+
+	return app, nil
+}
+
+// deployedServiceSpec represents a deployed service of an app.
+type deployedServiceSpec struct {
+	Name    string
+	Version string
+}
+
+// VersionSpec represents a deployed app and associated metadata.
+type VersionSpec map[string]deployedServiceSpec
+
+// DeployedVersions lists metadata (e.g. versions) for deployed
+func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App) (VersionSpec, bool, error) {
+	services, err := stack.GetStackServices(ctx, cl, app.StackName())
+	if err != nil {
+		return VersionSpec{}, false, err
+	}
+
+	appSpec := make(VersionSpec)
+	for _, service := range services {
+		serviceName := ParseServiceName(service.Spec.Name)
+		label := fmt.Sprintf("coop-cloud.%s.%s.version", app.StackName(), serviceName)
+		if deployLabel, ok := service.Spec.Labels[label]; ok {
+			version, _ := ParseVersionLabel(deployLabel)
+			appSpec[serviceName] = deployedServiceSpec{Name: serviceName, Version: version}
+		}
+	}
+
+	deployed := len(services) > 0
+
+	if deployed {
+		logrus.Debugf("detected '%s' as deployed versions of '%s'", appSpec, app.Name)
+	} else {
+		logrus.Debugf("detected '%s' as not deployed", app.Name)
+	}
+
+	return appSpec, len(services) > 0, nil
+}
+
+// ParseVersionLabel parses a $VERSION-$DIGEST app service label.
+func ParseVersionLabel(label string) (string, string) {
+	// versions may look like v4.2-abcd or v4.2-alpine-abcd
+	idx := strings.LastIndex(label, "-")
+	version := label[:idx]
+	digest := label[idx+1:]
+	logrus.Debugf("parsed '%s' as version from '%s'", version, label)
+	logrus.Debugf("parsed '%s' as digest from '%s'", digest, label)
+	return version, digest
+}
+
+// ParseVersionName parses a $STACK_NAME_$SERVICE_NAME service label.
+func ParseServiceName(label string) string {
+	idx := strings.LastIndex(label, "_")
+	serviceName := label[idx+1:]
+	logrus.Debugf("parsed '%s' as service name from '%s'", serviceName, label)
+	return serviceName
+}

pkg/catalogue/catalogue.go

@@ -0,0 +1,500 @@
+// Package catalogue provides ways of interacting with recipe catalogues which
+// are JSON data structures which contain meta information about recipes (e.g.
+// what versions of the Nextcloud recipe are available?).
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/web"
+	"github.com/docker/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// RecipeCatalogueURL is the only current recipe catalogue available.
+const RecipeCatalogueURL = "https://apps.coopcloud.tech"
+
+// ReposMetadataURL is the recipe repository metadata
+const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
+
+// image represents a recipe container image.
+type image struct {
+	Image  string `json:"image"`
+	Rating string `json:"rating"`
+	Source string `json:"source"`
+	URL    string `json:"url"`
+}
+
+// features represent what top-level features a recipe supports (e.g. does this
+// recipe support backups?).
+type features struct {
+	Backups     string `json:"backups"`
+	Email       string `json:"email"`
+	Healthcheck string `json:"healthcheck"`
+	Image       image  `json:"image"`
+	Status      int    `json:"status"`
+	Tests       string `json:"tests"`
+}
+
+// tag represents a git tag.
+type tag = string
+
+// service represents a service within a recipe.
+type service = string
+
+// ServiceMeta represents meta info associated with a service.
+type ServiceMeta struct {
+	Digest string `json:"digest"`
+	Image  string `json:"image"`
+	Tag    string `json:"tag"`
+}
+
+// RecipeVersions are the versions associated with a recipe.
+type RecipeVersions []map[tag]map[service]ServiceMeta
+
+// RecipeMeta represents metadata for a recipe in the abra catalogue.
+type RecipeMeta struct {
+	Category      string         `json:"category"`
+	DefaultBranch string         `json:"default_branch"`
+	Description   string         `json:"description"`
+	Features      features       `json:"features"`
+	Icon          string         `json:"icon"`
+	Name          string         `json:"name"`
+	Repository    string         `json:"repository"`
+	Versions      RecipeVersions `json:"versions"`
+	Website       string         `json:"website"`
+}
+
+// LatestVersion returns the latest version of a recipe.
+func (r RecipeMeta) LatestVersion() string {
+	var version string
+
+	// apps.json versions are sorted so the last key is latest
+	latest := r.Versions[len(r.Versions)-1]
+
+	for tag := range latest {
+		version = tag
+	}
+
+	logrus.Debugf("choosing '%s' as latest version of '%s'", version, r.Name)
+
+	return version
+}
+
+// Name represents a recipe name.
+type Name = string
+
+// RecipeCatalogue represents the entire recipe catalogue.
+type RecipeCatalogue map[Name]RecipeMeta
+
+// Flatten converts AppCatalogue to slice
+func (r RecipeCatalogue) Flatten() []RecipeMeta {
+	recipes := make([]RecipeMeta, 0, len(r))
+
+	for name := range r {
+		recipes = append(recipes, r[name])
+	}
+
+	return recipes
+}
+
+// ByRecipeName sorts recipes by name.
+type ByRecipeName []RecipeMeta
+
+func (r ByRecipeName) Len() int      { return len(r) }
+func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+func (r ByRecipeName) Less(i, j int) bool {
+	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
+}
+
+// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
+// is up to date.
+func recipeCatalogueFSIsLatest() (bool, error) {
+	httpClient := &http.Client{Timeout: web.Timeout}
+	res, err := httpClient.Head(RecipeCatalogueURL)
+	if err != nil {
+		return false, err
+	}
+
+	lastModified := res.Header["Last-Modified"][0]
+	parsed, err := time.Parse(time.RFC1123, lastModified)
+	if err != nil {
+		return false, err
+	}
+
+	info, err := os.Stat(config.APPS_JSON)
+	if err != nil {
+		if os.IsNotExist(err) {
+			logrus.Debugf("no recipe catalogue found in file system cache")
+			return false, nil
+		}
+		return false, err
+	}
+
+	localModifiedTime := info.ModTime().Unix()
+	remoteModifiedTime := parsed.Unix()
+
+	if localModifiedTime < remoteModifiedTime {
+		logrus.Debug("file system cached recipe catalogue is out-of-date")
+		return false, nil
+	}
+
+	logrus.Debug("file system cached recipe catalogue is up-to-date")
+
+	return true, nil
+}
+
+// ReadRecipeCatalogue reads the recipe catalogue.
+func ReadRecipeCatalogue() (RecipeCatalogue, error) {
+	recipes := make(RecipeCatalogue)
+
+	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
+	if err != nil {
+		return nil, err
+	}
+
+	if !recipeFSIsLatest {
+		logrus.Debugf("reading recipe catalogue from web to get latest")
+		if err := readRecipeCatalogueWeb(&recipes); err != nil {
+			return nil, err
+		}
+		return recipes, nil
+	}
+
+	logrus.Debugf("reading recipe catalogue from file system cache to get latest")
+	if err := readRecipeCatalogueFS(&recipes); err != nil {
+		return nil, err
+	}
+
+	return recipes, nil
+}
+
+// readRecipeCatalogueFS reads the catalogue from the file system.
+func readRecipeCatalogueFS(target interface{}) error {
+	recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
+	if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
+		return err
+	}
+
+	logrus.Debugf("read recipe catalogue from file system cache in '%s'", config.APPS_JSON)
+
+	return nil
+}
+
+// readRecipeCatalogueWeb reads the catalogue from the web.
+func readRecipeCatalogueWeb(target interface{}) error {
+	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
+		return err
+	}
+
+	recipesJSON, err := json.MarshalIndent(target, "", "    ")
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
+		return err
+	}
+
+	logrus.Debugf("read recipe catalogue from web at '%s'", RecipeCatalogueURL)
+
+	return nil
+}
+
+// VersionsOfService lists the version of a service.
+func VersionsOfService(recipe, serviceName string) ([]string, error) {
+	catalogue, err := ReadRecipeCatalogue()
+	if err != nil {
+		return nil, err
+	}
+
+	rec, ok := catalogue[recipe]
+	if !ok {
+		return nil, fmt.Errorf("recipe '%s' does not exist?", recipe)
+	}
+
+	versions := []string{}
+	alreadySeen := make(map[string]bool)
+	for _, serviceVersion := range rec.Versions {
+		for tag := range serviceVersion {
+			if _, ok := alreadySeen[tag]; !ok {
+				alreadySeen[tag] = true
+				versions = append(versions, tag)
+			}
+		}
+	}
+
+	logrus.Debugf("detected versions '%s' for '%s'", strings.Join(versions, ", "), recipe)
+
+	return versions, nil
+}
+
+// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
+func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
+	catl, err := ReadRecipeCatalogue()
+	if err != nil {
+		return RecipeMeta{}, err
+	}
+
+	recipeMeta, ok := catl[recipeName]
+	if !ok {
+		err := fmt.Errorf("recipe '%s' does not exist?", recipeName)
+		return RecipeMeta{}, err
+	}
+
+	if err := recipe.EnsureExists(recipeName); err != nil {
+		return RecipeMeta{}, err
+	}
+
+	logrus.Debugf("recipe metadata retrieved for '%s'", recipeName)
+
+	return recipeMeta, nil
+}
+
+// RepoMeta is a single recipe repo metadata.
+type RepoMeta struct {
+	ID                        int `json:"id"`
+	Owner                     Owner
+	Name                      string      `json:"name"`
+	FullName                  string      `json:"full_name"`
+	Description               string      `json:"description"`
+	Empty                     bool        `json:"empty"`
+	Private                   bool        `json:"private"`
+	Fork                      bool        `json:"fork"`
+	Template                  bool        `json:"template"`
+	Parent                    interface{} `json:"parent"`
+	Mirror                    bool        `json:"mirror"`
+	Size                      int         `json:"size"`
+	HTMLURL                   string      `json:"html_url"`
+	SSHURL                    string      `json:"ssh_url"`
+	CloneURL                  string      `json:"clone_url"`
+	OriginalURL               string      `json:"original_url"`
+	Website                   string      `json:"website"`
+	StarsCount                int         `json:"stars_count"`
+	ForksCount                int         `json:"forks_count"`
+	WatchersCount             int         `json:"watchers_count"`
+	OpenIssuesCount           int         `json:"open_issues_count"`
+	OpenPRCount               int         `json:"open_pr_counter"`
+	ReleaseCounter            int         `json:"release_counter"`
+	DefaultBranch             string      `json:"default_branch"`
+	Archived                  bool        `json:"archived"`
+	CreatedAt                 string      `json:"created_at"`
+	UpdatedAt                 string      `json:"updated_at"`
+	Permissions               Permissions
+	HasIssues                 bool `json:"has_issues"`
+	InternalTracker           InternalTracker
+	HasWiki                   bool   `json:"has_wiki"`
+	HasPullRequests           bool   `json:"has_pull_requests"`
+	HasProjects               bool   `json:"has_projects"`
+	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
+	AllowMergeCommits         bool   `json:"allow_merge_commits"`
+	AllowRebase               bool   `json:"allow_rebase"`
+	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
+	AllowSquashMerge          bool   `json:"allow_squash_merge"`
+	AvatarURL                 string `json:"avatar_url"`
+	Internal                  bool   `json:"internal"`
+	MirrorInterval            string `json:"mirror_interval"`
+}
+
+// Owner is the repo organisation owner metadata.
+type Owner struct {
+	ID         int    `json:"id"`
+	Login      string `json:"login"`
+	FullName   string `json:"full_name"`
+	Email      string `json:"email"`
+	AvatarURL  string `json:"avatar_url"`
+	Language   string `json:"language"`
+	IsAdmin    bool   `json:"is_admin"`
+	LastLogin  string `json:"last_login"`
+	Created    string `json:"created"`
+	Restricted bool   `json:"restricted"`
+	Username   string `json:"username"`
+}
+
+// Permissions is perms metadata for a repo.
+type Permissions struct {
+	Admin bool `json:"admin"`
+	Push  bool `json:"push"`
+	Pull  bool `json:"pull"`
+}
+
+// InternalTracker is issue tracker metadata for a repo.
+type InternalTracker struct {
+	EnableTimeTracker                bool `json:"enable_time_tracker"`
+	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
+	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
+}
+
+// RepoCatalogue represents all the recipe repo metadata.
+type RepoCatalogue map[string]RepoMeta
+
+// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
+func ReadReposMetadata() (RepoCatalogue, error) {
+	reposMeta := make(RepoCatalogue)
+
+	pageIdx := 1
+	for {
+		var reposList []RepoMeta
+
+		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
+
+		logrus.Debugf("fetching repo metadata from '%s'", pagedURL)
+
+		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
+			return reposMeta, err
+		}
+
+		if len(reposList) == 0 {
+			break
+		}
+
+		for idx, repo := range reposList {
+			reposMeta[repo.Name] = reposList[idx]
+		}
+
+		pageIdx++
+	}
+
+	return reposMeta, nil
+}
+
+// GetRecipeVersions retrieves all recipe versions.
+func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
+	versions := RecipeVersions{}
+
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return versions, err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	gitTags, err := repo.Tags()
+	if err != nil {
+		return versions, err
+	}
+
+	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
+		tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
+
+		logrus.Debugf("processing '%s' for '%s'", tag, recipeName)
+
+		checkOutOpts := &git.CheckoutOptions{
+			Create: false,
+			Force:  true,
+			Branch: plumbing.ReferenceName(ref.Name()),
+		}
+		if err := worktree.Checkout(checkOutOpts); err != nil {
+			logrus.Debugf("failed to check out '%s' in '%s'", tag, recipeDir)
+			return err
+		}
+
+		logrus.Debugf("successfully checked out '%s' in '%s'", ref.Name(), recipeDir)
+
+		recipe, err := recipe.Get(recipeName)
+		if err != nil {
+			return err
+		}
+
+		versionMeta := make(map[string]ServiceMeta)
+		for _, service := range recipe.Config.Services {
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return err
+			}
+
+			path := reference.Path(img)
+			if strings.Contains(path, "library") {
+				path = strings.Split(path, "/")[1]
+			}
+
+			digest, err := client.GetTagDigest(img)
+			if err != nil {
+				return err
+			}
+
+			versionMeta[service.Name] = ServiceMeta{
+				Digest: digest,
+				Image:  path,
+				Tag:    img.(reference.NamedTagged).Tag(),
+			}
+
+			logrus.Debugf("collecting digest: '%s', image: '%s', tag: '%s'", digest, path, tag)
+		}
+
+		versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
+
+		return nil
+	}); err != nil {
+		return versions, err
+	}
+
+	branch := "master"
+	if _, err := repo.Branch("master"); err != nil {
+		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", recipeDir)
+			logrus.Fatal(err)
+		}
+		branch = "main"
+	}
+
+	refName := fmt.Sprintf("refs/heads/%s", branch)
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: plumbing.ReferenceName(refName),
+	}
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", branch, recipeDir)
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("switched back to '%s' in '%s'", branch, recipeDir)
+	logrus.Debugf("collected '%s' for '%s'", versions, recipeName)
+
+	return versions, nil
+}
+
+// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
+func GetRecipeCatalogueVersions(recipeName string) ([]string, error) {
+	var versions []string
+
+	catl, err := ReadRecipeCatalogue()
+	if err != nil {
+		return versions, err
+	}
+
+	if recipeMeta, exists := catl[recipeName]; exists {
+		for _, versionMeta := range recipeMeta.Versions {
+			for tag := range versionMeta {
+				versions = append(versions, tag)
+			}
+		}
+	}
+
+	return versions, nil
+}

pkg/client/client.go

@@ -0,0 +1,60 @@
+// Package client provides Docker client initiatialisation functions.
+package client
+
+import (
+	"net/http"
+	"os"
+	"time"
+
+	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// New initiates a new Docker client.
+func New(contextName string) (*client.Client, error) {
+	var clientOpts []client.Opt
+
+	if contextName != "default" {
+		context, err := GetContext(contextName)
+		if err != nil {
+			return nil, err
+		}
+
+		ctxEndpoint, err := GetContextEndpoint(context)
+		if err != nil {
+			return nil, err
+		}
+
+		helper := commandconnPkg.NewConnectionHelper(ctxEndpoint)
+		httpClient := &http.Client{
+			// No tls, no proxy
+			Transport: &http.Transport{
+				DialContext:     helper.Dialer,
+				IdleConnTimeout: 30 * time.Second,
+			},
+		}
+
+		clientOpts = append(clientOpts,
+			client.WithHTTPClient(httpClient),
+			client.WithHost(helper.Host),
+			client.WithDialContext(helper.Dialer),
+		)
+	}
+
+	version := os.Getenv("DOCKER_API_VERSION")
+	if version != "" {
+		clientOpts = append(clientOpts, client.WithVersion(version))
+	} else {
+		clientOpts = append(clientOpts, client.WithAPIVersionNegotiation())
+	}
+
+	cl, err := client.NewClientWithOpts(clientOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("created client for '%s'", contextName)
+
+	return cl, nil
+}

pkg/client/client_test.go

@@ -0,0 +1,46 @@
+package client_test
+
+import (
+	"fmt"
+	"testing"
+
+	"coopcloud.tech/abra/pkg/client"
+)
+
+// use at the start to ensure testContext[0, 1, ..., amnt-1] exist and
+// testContextFail[0, 1, ..., failAmnt-1] don't exist
+func ensureTestState(amnt, failAmnt int) error {
+	for i := 0; i < amnt; i++ {
+		err := client.CreateContext(fmt.Sprintf("testContext%d", i), "", "")
+		if err != nil {
+			return err
+		}
+	}
+	for i := 0; i < failAmnt; i++ {
+		if _, er := client.GetContext(fmt.Sprintf("testContextFail%d", i)); er == nil {
+			err := client.DeleteContext(fmt.Sprintf("testContextFail%d", i))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func TestNew(t *testing.T) {
+	err := ensureTestState(1, 1)
+	if err != nil {
+		t.Errorf("Couldn't ensure existence/nonexistence of contexts: %s", err)
+	}
+	contextName := "testContext0"
+	_, err = client.New(contextName)
+	if err != nil {
+		t.Errorf("couldn't initialise a new client with context %s: %s", contextName, err)
+	}
+	contextName = "testContextFail0"
+	_, err = client.New(contextName)
+	if err == nil {
+		t.Errorf("client.New(\"testContextFail0\") should have failed but didn't return an error")
+	}
+
+}

pkg/client/context.go

@@ -0,0 +1,128 @@
+package client
+
+import (
+	"errors"
+	"fmt"
+
+	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
+	command "github.com/docker/cli/cli/command"
+	dConfig "github.com/docker/cli/cli/config"
+	context "github.com/docker/cli/cli/context"
+	"github.com/docker/cli/cli/context/docker"
+	contextStore "github.com/docker/cli/cli/context/store"
+	cliflags "github.com/docker/cli/cli/flags"
+	"github.com/moby/term"
+	"github.com/sirupsen/logrus"
+)
+
+type Context = contextStore.Metadata
+
+func CreateContext(contextName string, user string, port string) error {
+	host := contextName
+	if user != "" {
+		host = fmt.Sprintf("%s@%s", user, host)
+	}
+	if port != "" {
+		host = fmt.Sprintf("%s:%s", host, port)
+	}
+	host = fmt.Sprintf("ssh://%s", host)
+	if err := createContext(contextName, host); err != nil {
+		return err
+	}
+	logrus.Debugf("created the '%s' context", contextName)
+	return nil
+}
+
+// createContext interacts with Docker Context to create a Docker context config
+func createContext(name string, host string) error {
+	s := NewDefaultDockerContextStore()
+	contextMetadata := contextStore.Metadata{
+		Endpoints: make(map[string]interface{}),
+		Name:      name,
+	}
+
+	contextTLSData := contextStore.ContextTLSData{
+		Endpoints: make(map[string]contextStore.EndpointTLSData),
+	}
+
+	dockerEP, dockerTLS, err := commandconnPkg.GetDockerEndpointMetadataAndTLS(host)
+	if err != nil {
+		return err
+	}
+
+	contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP
+	if dockerTLS != nil {
+		contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerTLS
+	}
+
+	if err := s.CreateOrUpdate(contextMetadata); err != nil {
+		return err
+	}
+
+	if err := s.ResetTLSMaterial(name, &contextTLSData); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DeleteContext(name string) error {
+	if name == "default" {
+		return errors.New("context 'default' cannot be removed")
+	}
+
+	if _, err := GetContext(name); err != nil {
+		return err
+	}
+
+	// remove any context that might be loaded
+	// TODO: Check if the context we are removing is the active one rather than doing it all the time
+	cfg := dConfig.LoadDefaultConfigFile(nil)
+	cfg.CurrentContext = ""
+	if err := cfg.Save(); err != nil {
+		return err
+	}
+
+	return NewDefaultDockerContextStore().Remove(name)
+}
+
+func GetContext(contextName string) (contextStore.Metadata, error) {
+	ctx, err := NewDefaultDockerContextStore().GetMetadata(contextName)
+	if err != nil {
+		return contextStore.Metadata{}, err
+	}
+
+	return ctx, nil
+}
+
+func GetContextEndpoint(ctx contextStore.Metadata) (string, error) {
+	endpointmeta, ok := ctx.Endpoints["docker"].(context.EndpointMetaBase)
+	if !ok {
+		err := errors.New("context lacks Docker endpoint")
+		return "", err
+	}
+	return endpointmeta.Host, nil
+}
+
+func newContextStore(dir string, config contextStore.Config) contextStore.Store {
+	return contextStore.New(dir, config)
+}
+
+func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
+	_, _, stderr := term.StdStreams()
+	dockerConfig := dConfig.LoadDefaultConfigFile(stderr)
+	contextDir := dConfig.ContextStoreDir()
+	storeConfig := command.DefaultContextStoreConfig()
+	store := newContextStore(contextDir, storeConfig)
+
+	opts := &cliflags.CommonOptions{Context: "default"}
+
+	dockerContextStore := &command.ContextStoreWithDefault{
+		Store: store,
+		Resolver: func() (*command.DefaultContext, error) {
+			return command.ResolveDefaultContext(opts, dockerConfig, storeConfig, stderr)
+		},
+	}
+
+	return dockerContextStore
+}

pkg/client/context_test.go

@@ -0,0 +1,80 @@
+package client_test
+
+import (
+	"testing"
+
+	"coopcloud.tech/abra/pkg/client"
+	dContext "github.com/docker/cli/cli/context"
+	dCliContextStore "github.com/docker/cli/cli/context/store"
+)
+
+type TestContext struct {
+	context           dCliContextStore.Metadata
+	expected_endpoint string
+}
+
+func dockerContext(host, key string) TestContext {
+	dockerContext := dCliContextStore.Metadata{
+		Name:     "foo",
+		Metadata: nil,
+		Endpoints: map[string]interface{}{
+			key: dContext.EndpointMetaBase{
+				Host:          host,
+				SkipTLSVerify: false,
+			},
+		},
+	}
+	return TestContext{
+		context:           dockerContext,
+		expected_endpoint: host,
+	}
+}
+
+func TestCreateContext(t *testing.T) {
+	err := client.CreateContext("testContext0", "wronguser", "wrongport")
+	if err == nil {
+		t.Error("client.CreateContext(\"testContextCreate\", \"wronguser\", \"wrongport\") should have failed but didn't return an error")
+	}
+	err = client.CreateContext("testContext0", "", "")
+	if err != nil {
+		t.Errorf("Couldn't create context: %s", err)
+	}
+}
+
+func TestDeleteContext(t *testing.T) {
+	ensureTestState(1, 1)
+	err := client.DeleteContext("default")
+	if err == nil {
+		t.Errorf("client.DeleteContext(\"default\") should have failed but didn't return an error")
+	}
+
+	err = client.DeleteContext("testContext0")
+	if err != nil {
+		t.Errorf("client.DeleteContext(\"testContext0\") failed: %s", err)
+	}
+	err = client.DeleteContext("testContextFail0")
+	if err == nil {
+		t.Errorf("client.DeleteContext(\"testContextFail0\") should have failed (attempt to delete non-existent context) but didn't return an error")
+	}
+}
+
+func TestGetContextEndpoint(t *testing.T) {
+	var testDockerContexts = []TestContext{
+		dockerContext("ssh://foobar", "docker"),
+		dockerContext("ssh://foobar", "k8"),
+	}
+	for _, context := range testDockerContexts {
+		endpoint, err := client.GetContextEndpoint(context.context)
+		if err != nil {
+			if err.Error() != "context lacks Docker endpoint" {
+				t.Error(err)
+			}
+		} else {
+			if endpoint != context.expected_endpoint {
+				t.Errorf("did not get correct context endpoint. Expected: %s, received: %s", context.expected_endpoint, endpoint)
+			}
+		}
+
+	}
+
+}

pkg/client/registry.go

@@ -0,0 +1,170 @@
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/web"
+	"github.com/docker/distribution/reference"
+)
+
+type RawTag struct {
+	Layer string
+	Name  string
+}
+
+type RawTags []RawTag
+
+var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
+
+func GetRegistryTags(image string) (RawTags, error) {
+	var tags RawTags
+
+	tagsUrl := fmt.Sprintf(registryURL, image)
+	if err := web.ReadJSON(tagsUrl, &tags); err != nil {
+		return tags, err
+	}
+
+	return tags, nil
+}
+
+// getRegv2Token retrieves a registry v2 authentication token.
+func getRegv2Token(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
+	req, err := http.NewRequest("GET", authTokenURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	client := &http.Client{Timeout: web.Timeout}
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", nil
+	}
+
+	tokenRes := struct {
+		Token  string
+		Expiry string
+		Issued string
+	}{}
+
+	if err := json.Unmarshal(body, &tokenRes); err != nil {
+		return "", err
+	}
+
+	return tokenRes.Token, nil
+}
+
+// GetTagDigest retrieves an image digest from a v2 registry
+func GetTagDigest(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	tag := image.(reference.NamedTagged).Tag()
+	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
+
+	req, err := http.NewRequest("GET", manifestURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := getRegv2Token(image)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header = http.Header{
+		"Accept": []string{
+			"application/vnd.docker.distribution.manifest.v2+json",
+			"application/vnd.docker.distribution.manifest.list.v2+json",
+		},
+		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
+	}
+
+	client := &http.Client{Timeout: web.Timeout}
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+
+	registryResT1 := struct {
+		SchemaVersion int
+		MediaType     string
+		Manifests     []struct {
+			MediaType string
+			Size      int
+			Digest    string
+			Platform  struct {
+				Architecture string
+				Os           string
+			}
+		}
+	}{}
+
+	registryResT2 := struct {
+		SchemaVersion int
+		MediaType     string
+		Config        struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+		Layers []struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+	}{}
+
+	if err := json.Unmarshal(body, &registryResT1); err != nil {
+		return "", err
+	}
+
+	var digest string
+	for _, manifest := range registryResT1.Manifests {
+		if string(manifest.Platform.Architecture) == "amd64" {
+			digest = strings.Split(manifest.Digest, ":")[1][:7]
+		}
+	}
+
+	if digest == "" {
+		if err := json.Unmarshal(body, &registryResT2); err != nil {
+			return "", err
+		}
+		digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
+	}
+
+	if digest == "" {
+		return "", fmt.Errorf("Unable to retrieve amd64 digest for '%s'", image)
+	}
+
+	return digest, nil
+}

pkg/client/secret.go

@@ -0,0 +1,25 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/swarm"
+)
+
+func StoreSecret(secretName, secretValue, server string) error {
+	cl, err := New(server)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	ann := swarm.Annotations{Name: secretName}
+	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
+
+	// We don't bother with the secret IDs for now
+	if _, err := cl.SecretCreate(ctx, spec); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/client/volumes.go

@@ -0,0 +1,51 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+)
+
+func GetVolumes(ctx context.Context, server string, appName string) ([]*types.Volume, error) {
+
+	cl, err := New(server)
+	if err != nil {
+		return nil, err
+	}
+
+	fs := filters.NewArgs()
+	fs.Add("name", appName)
+
+	volumeListOKBody, err := cl.VolumeList(ctx, fs)
+	volumeList := volumeListOKBody.Volumes
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	return volumeList, nil
+}
+
+func GetVolumeNames(volumes []*types.Volume) []string {
+	var volumeNames []string
+	for _, vol := range volumes {
+		volumeNames = append(volumeNames, vol.Name)
+	}
+	return volumeNames
+}
+
+func RemoveVolumes(ctx context.Context, server string, volumeNames []string, force bool) error {
+	cl, err := New(server)
+	if err != nil {
+		return err
+	}
+	for _, volName := range volumeNames {
+		err := cl.VolumeRemove(ctx, volName, force)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+
+}

pkg/compose/compose.go

@@ -0,0 +1,140 @@
+package compose
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+)
+
+// UpdateTag updates an image tag in-place on file system local compose files.
+func UpdateTag(pattern, image, tag, recipeName string) error {
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("considering '%s' config(s) for tag update", strings.Join(composeFiles, ", "))
+
+	for _, composeFile := range composeFiles {
+		opts := stack.Deploy{Composefiles: []string{composeFile}}
+
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
+		sampleEnv, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			return err
+		}
+
+		compose, err := loader.LoadComposefile(opts, sampleEnv)
+		if err != nil {
+			return err
+		}
+
+		for _, service := range compose.Services {
+			if service.Image == "" {
+				continue // may be a compose.$optional.yml file
+			}
+
+			img, _ := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return err
+			}
+
+			composeImage := reference.Path(img)
+			if strings.Contains(composeImage, "library") {
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
+				// postgres:<tag>, i.e. images which do not have a username in the
+				// first position of the string
+				composeImage = strings.Split(composeImage, "/")[1]
+			}
+			composeTag := img.(reference.NamedTagged).Tag()
+
+			logrus.Debugf("parsed '%s' from '%s'", composeTag, service.Image)
+
+			if image == composeImage {
+				bytes, err := ioutil.ReadFile(composeFile)
+				if err != nil {
+					return err
+				}
+
+				old := fmt.Sprintf("%s:%s", composeImage, composeTag)
+				new := fmt.Sprintf("%s:%s", composeImage, tag)
+				replacedBytes := strings.Replace(string(bytes), old, new, -1)
+
+				logrus.Debugf("updating '%s' to '%s' in '%s'", old, new, compose.Filename)
+
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// UpdateLabel updates a label in-place on file system local compose files.
+func UpdateLabel(pattern, serviceName, label, recipeName string) error {
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("considering '%s' config(s) for label update", strings.Join(composeFiles, ", "))
+
+	for _, composeFile := range composeFiles {
+		opts := stack.Deploy{Composefiles: []string{composeFile}}
+
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
+		sampleEnv, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			return err
+		}
+
+		compose, err := loader.LoadComposefile(opts, sampleEnv)
+		if err != nil {
+			return err
+		}
+
+		serviceExists := false
+		var service composetypes.ServiceConfig
+		for _, s := range compose.Services {
+			if s.Name == serviceName {
+				service = s
+				serviceExists = true
+			}
+		}
+
+		if !serviceExists {
+			continue
+		}
+
+		for oldLabel, value := range service.Deploy.Labels {
+			if strings.HasPrefix(oldLabel, "coop-cloud") {
+				bytes, err := ioutil.ReadFile(composeFile)
+				if err != nil {
+					return err
+				}
+
+				old := fmt.Sprintf("coop-cloud.${STACK_NAME}.version=%s", value)
+				replacedBytes := strings.Replace(string(bytes), old, label, -1)
+
+				logrus.Debugf("updating '%s' to '%s' in '%s'", old, label, compose.Filename)
+
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}

pkg/config/app.go

@@ -0,0 +1,370 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/sirupsen/logrus"
+)
+
+// Type aliases to make code hints easier to understand
+
+// AppEnv is a map of the values in an apps env config
+type AppEnv = map[string]string
+
+// AppName is AppName
+type AppName = string
+
+// AppFile represents app env files on disk without reading the contents
+type AppFile struct {
+	Path   string
+	Server string
+}
+
+// AppFiles is a slice of appfiles
+type AppFiles map[AppName]AppFile
+
+// App reprents an app with its env file read into memory
+type App struct {
+	Name   AppName
+	Type   string
+	Domain string
+	Env    AppEnv
+	Server string
+	Path   string
+}
+
+// StackName gets what the docker safe stack name is for the app
+func (a App) StackName() string {
+	if _, exists := a.Env["STACK_NAME"]; exists {
+		return a.Env["STACK_NAME"]
+	}
+	stackName := SanitiseAppName(a.Name)
+	a.Env["STACK_NAME"] = stackName
+	return stackName
+}
+
+// SORTING TYPES
+
+// ByServer sort a slice of Apps
+type ByServer []App
+
+func (a ByServer) Len() int      { return len(a) }
+func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServer) Less(i, j int) bool {
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByServerAndType sort a slice of Apps
+type ByServerAndType []App
+
+func (a ByServerAndType) Len() int      { return len(a) }
+func (a ByServerAndType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServerAndType) Less(i, j int) bool {
+	if a[i].Server == a[j].Server {
+		return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
+	}
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByType sort a slice of Apps
+type ByType []App
+
+func (a ByType) Len() int      { return len(a) }
+func (a ByType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByType) Less(i, j int) bool {
+	return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
+}
+
+// ByName sort a slice of Apps
+type ByName []App
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
+}
+
+func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
+	env, err := ReadEnv(appFile.Path)
+	if err != nil {
+		return App{}, fmt.Errorf("env file for '%s' couldn't be read: %s", name, err.Error())
+	}
+
+	logrus.Debugf("read env '%s' from '%s'", env, appFile.Path)
+
+	app, err := newApp(env, name, appFile)
+	if err != nil {
+		return App{}, fmt.Errorf("env file for '%s' has issues: %s", name, err.Error())
+	}
+
+	return app, nil
+}
+
+// newApp creates new App object
+func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+	// Checking for type as it is required - apps wont work without it
+	domain := env["DOMAIN"]
+	apptype, ok := env["TYPE"]
+	if !ok {
+		return App{}, errors.New("missing TYPE variable")
+	}
+
+	return App{
+		Name:   name,
+		Domain: domain,
+		Type:   apptype,
+		Env:    env,
+		Server: appFile.Server,
+		Path:   appFile.Path,
+	}, nil
+}
+
+// LoadAppFiles gets all app files for a given set of servers or all servers
+func LoadAppFiles(servers ...string) (AppFiles, error) {
+	appFiles := make(AppFiles)
+	if len(servers) == 1 {
+		if servers[0] == "" {
+			// Empty servers flag, one string will always be passed
+			var err error
+			servers, err = getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	logrus.Debugf("collecting metadata from '%v' servers: '%s'", len(servers), strings.Join(servers, ", "))
+
+	for _, server := range servers {
+		serverDir := path.Join(ABRA_SERVER_FOLDER, server)
+		files, err := getAllFilesInDirectory(serverDir)
+		if err != nil {
+			return nil, err
+		}
+		for _, file := range files {
+			appName := strings.TrimSuffix(file.Name(), ".env")
+			appFilePath := path.Join(ABRA_SERVER_FOLDER, server, file.Name())
+			appFiles[appName] = AppFile{
+				Path:   appFilePath,
+				Server: server,
+			}
+		}
+	}
+	return appFiles, nil
+}
+
+// GetApp loads an apps settings, reading it from file, in preparation to use it
+//
+// ONLY use when ready to use the env file to keep IO down
+func GetApp(apps AppFiles, name AppName) (App, error) {
+	appFile, exists := apps[name]
+	if !exists {
+		return App{}, fmt.Errorf("cannot find app with name '%s'", name)
+	}
+
+	app, err := readAppEnvFile(appFile, name)
+	if err != nil {
+		return App{}, err
+	}
+
+	return app, nil
+}
+
+// GetApps returns a slice of Apps with their env files read from a given slice of AppFiles
+func GetApps(appFiles AppFiles) ([]App, error) {
+	var apps []App
+
+	for name := range appFiles {
+		app, err := GetApp(appFiles, name)
+		if err != nil {
+			return nil, err
+		}
+		apps = append(apps, app)
+	}
+
+	return apps, nil
+}
+
+// GetAppServiceNames retrieves a list of app service names.
+func GetAppServiceNames(appName string) ([]string, error) {
+	var serviceNames []string
+
+	appFiles, err := LoadAppFiles("")
+	if err != nil {
+		return serviceNames, err
+	}
+
+	app, err := GetApp(appFiles, appName)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	composeFiles, err := GetAppComposeFiles(app.Type, app.Env)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	compose, err := GetAppComposeConfig(app.Type, opts, app.Env)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	for _, service := range compose.Services {
+		serviceNames = append(serviceNames, service.Name)
+	}
+
+	return serviceNames, nil
+}
+
+// GetAppNames retrieves a list of app names.
+func GetAppNames() ([]string, error) {
+	var appNames []string
+
+	appFiles, err := LoadAppFiles("")
+	if err != nil {
+		return appNames, err
+	}
+
+	apps, err := GetApps(appFiles)
+	if err != nil {
+		return appNames, err
+	}
+
+	for _, app := range apps {
+		appNames = append(appNames, app.Name)
+	}
+
+	return appNames, nil
+}
+
+// TemplateAppEnvSample copies the example env file for the app into the users env files
+func TemplateAppEnvSample(appType, appName, server, domain, recipe string) error {
+	envSamplePath := path.Join(ABRA_DIR, "apps", appType, ".env.sample")
+	envSample, err := ioutil.ReadFile(envSamplePath)
+	if err != nil {
+		return err
+	}
+
+	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
+	if _, err := os.Stat(appEnvPath); err == nil {
+		return fmt.Errorf("%s already exists?", appEnvPath)
+	}
+
+	envSample = []byte(strings.Replace(string(envSample), fmt.Sprintf("%s.example.com", recipe), domain, -1))
+	envSample = []byte(strings.Replace(string(envSample), "example.com", domain, -1))
+
+	err = ioutil.WriteFile(appEnvPath, envSample, 0755)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("copied '%s' to '%s'", envSamplePath, appEnvPath)
+
+	return nil
+}
+
+// SanitiseAppName makes a app name usable with Docker by replacing illegal characters
+func SanitiseAppName(name string) string {
+	return strings.ReplaceAll(name, ".", "_")
+}
+
+// GetAppStatuses queries servers to check the deployment status of given apps
+func GetAppStatuses(appFiles AppFiles) (map[string]map[string]string, error) {
+	statuses := make(map[string]map[string]string)
+
+	var unique []string
+	servers := make(map[string]struct{})
+	for _, appFile := range appFiles {
+		if _, ok := servers[appFile.Server]; !ok {
+			servers[appFile.Server] = struct{}{}
+			unique = append(unique, appFile.Server)
+		}
+	}
+
+	bar := formatter.CreateProgressbar(len(servers), "querying remote servers...")
+	ch := make(chan stack.StackStatus, len(servers))
+	for server := range servers {
+		go func(s string) {
+			ch <- stack.GetAllDeployedServices(s)
+			bar.Add(1)
+		}(server)
+	}
+
+	for range servers {
+		status := <-ch
+		for _, service := range status.Services {
+			result := make(map[string]string)
+			name := service.Spec.Labels[convert.LabelNamespace]
+
+			if _, ok := statuses[name]; !ok {
+				result["status"] = "deployed"
+			}
+
+			labelKey := fmt.Sprintf("coop-cloud.%s.version", name)
+			if version, ok := service.Spec.Labels[labelKey]; ok {
+				result["version"] = version
+			} else {
+				//FIXME: we only need to check containers with the version label not
+				//       every single container and then skip when we see no label perf gains
+				//       to be had here
+				continue
+			}
+
+			statuses[name] = result
+		}
+	}
+
+	logrus.Debugf("retrieved app statuses: '%s'", statuses)
+
+	return statuses, nil
+}
+
+// GetAppComposeFiles gets the list of compose files for an app which should be
+// merged into a composetypes.Config while respecting the COMPOSE_FILE env var.
+func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
+	var composeFiles []string
+
+	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
+		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
+		path := fmt.Sprintf("%s/%s/compose.yml", APPS_DIR, recipe)
+		composeFiles = append(composeFiles, path)
+		return composeFiles, nil
+	}
+
+	composeFileEnvVar := appEnv["COMPOSE_FILE"]
+	envVars := strings.Split(composeFileEnvVar, ":")
+	logrus.Debugf("COMPOSE_FILE detected ('%s'), loading '%s'", composeFileEnvVar, strings.Join(envVars, ", "))
+	for _, file := range strings.Split(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", APPS_DIR, recipe, file)
+		composeFiles = append(composeFiles, path)
+	}
+
+	logrus.Debugf("retrieved '%s' configs for '%s'", strings.Join(composeFiles, ", "), recipe)
+
+	return composeFiles, nil
+}
+
+// GetAppComposeConfig retrieves a compose specification for a recipe. This
+// specification is the result of a merge of all the compose.**.yml files in
+// the recipe repository.
+func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) {
+	compose, err := loader.LoadComposefile(opts, appEnv)
+	if err != nil {
+		return &composetypes.Config{}, err
+	}
+
+	logrus.Debugf("retrieved '%s' for '%s'", compose.Filename, recipe)
+
+	return compose, nil
+}

pkg/config/app_test.go

@@ -0,0 +1,37 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestNewApp(t *testing.T) {
+	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}
+
+func TestReadAppEnvFile(t *testing.T) {
+	app, err := readAppEnvFile(expectedAppFile, appName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}
+
+func TestGetApp(t *testing.T) {
+	// TODO: Test failures as well as successes
+	app, err := GetApp(expectedAppFiles, appName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}

pkg/config/env.go

@@ -0,0 +1,167 @@
+package config
+
+import (
+	"bufio"
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/Autonomic-Cooperative/godotenv"
+	"github.com/sirupsen/logrus"
+)
+
+var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
+var ABRA_SERVER_FOLDER = path.Join(ABRA_DIR, "servers")
+var APPS_JSON = path.Join(ABRA_DIR, "apps.json")
+var APPS_DIR = path.Join(ABRA_DIR, "apps")
+var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
+
+// GetServers retrieves all servers.
+func GetServers() ([]string, error) {
+	var servers []string
+
+	servers, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+	if err != nil {
+		return servers, err
+	}
+
+	logrus.Debugf("retrieved '%v' servers: '%s'", len(servers), servers)
+
+	return servers, nil
+}
+
+// ReadEnv loads an app envivornment into a map.
+func ReadEnv(filePath string) (AppEnv, error) {
+	var envFile AppEnv
+
+	envFile, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("read '%s' from '%s'", envFile, filePath)
+
+	return envFile, nil
+}
+
+// ReadServerNames retrieves all server names.
+func ReadServerNames() ([]string, error) {
+	serverNames, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("read '%s' from '%s'", strings.Join(serverNames, ","), ABRA_SERVER_FOLDER)
+
+	return serverNames, nil
+}
+
+// getAllFilesInDirectory returns filenames of all files in directory
+func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
+	var realFiles []fs.FileInfo
+
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		// Follow any symlinks
+		filePath := path.Join(directory, file.Name())
+
+		if filepath.Ext(strings.TrimSpace(filePath)) != ".env" {
+			continue
+		}
+
+		realPath, err := filepath.EvalSymlinks(filePath)
+		if err != nil {
+			logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
+		} else {
+			realFile, err := os.Stat(realPath)
+			if err != nil {
+				return nil, err
+			}
+			if !realFile.IsDir() {
+				realFiles = append(realFiles, file)
+			}
+		}
+	}
+
+	return realFiles, nil
+}
+
+// getAllFoldersInDirectory returns both folder and symlink paths
+func getAllFoldersInDirectory(directory string) ([]string, error) {
+	var folders []string
+
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+	if len(files) == 0 {
+		return nil, fmt.Errorf("directory is empty: '%s'", directory)
+	}
+
+	for _, file := range files {
+		// Check if file is directory or symlink
+		if file.IsDir() || file.Mode()&fs.ModeSymlink != 0 {
+			filePath := path.Join(directory, file.Name())
+			realDir, err := filepath.EvalSymlinks(filePath)
+			if err != nil {
+				logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
+			} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
+				// path is a directory
+				folders = append(folders, file.Name())
+			}
+		}
+	}
+
+	return folders, nil
+}
+
+// EnsureAbraDirExists checks for the abra config folder and throws error if not
+func EnsureAbraDirExists() error {
+	if _, err := os.Stat(ABRA_DIR); os.IsNotExist(err) {
+		logrus.Debugf("'%s' does not exist, creating it", ABRA_DIR)
+		if err := os.Mkdir(ABRA_DIR, 0777); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ReadAbraShEnvVars reads env vars from an abra.sh recipe file.
+func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
+	envVars := make(map[string]string)
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return envVars, nil
+		}
+		return envVars, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "export") {
+			splitVals := strings.Split(line, "export ")
+			envVarDef := splitVals[len(splitVals)-1]
+			keyVal := strings.Split(envVarDef, "=")
+			if len(keyVal) != 2 {
+				return envVars, fmt.Errorf("couldn't parse %s", line)
+			}
+			envVars[keyVal[0]] = keyVal[1]
+		}
+	}
+
+	logrus.Debugf("read '%s' from '%s'", envVars, abraSh)
+
+	return envVars, nil
+}

pkg/config/env_test.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"os"
+	"path"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+
+// make sure these are in alphabetical order
+var tFolders = []string{"folder1", "folder2"}
+var tFiles = []string{"bar.env", "foo.env"}
+
+var appName = "ecloud"
+var serverName = "evil.corp"
+
+var expectedAppEnv = AppEnv{
+	"DOMAIN": "ecloud.evil.corp",
+	"TYPE":   "ecloud",
+}
+
+var expectedApp = App{
+	Name:   appName,
+	Type:   expectedAppEnv["TYPE"],
+	Domain: expectedAppEnv["DOMAIN"],
+	Env:    expectedAppEnv,
+	Path:   expectedAppFile.Path,
+	Server: expectedAppFile.Server,
+}
+
+var expectedAppFile = AppFile{
+	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
+	Server: serverName,
+}
+
+var expectedAppFiles = map[string]AppFile{
+	appName: expectedAppFile,
+}
+
+// var expectedServerNames = []string{"evil.corp"}
+
+func TestGetAllFoldersInDirectory(t *testing.T) {
+	folders, err := getAllFoldersInDirectory(testFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(folders, tFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	}
+}
+
+func TestGetAllFilesInDirectory(t *testing.T) {
+	files, err := getAllFilesInDirectory(testFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fileNames []string
+	for _, file := range files {
+		fileNames = append(fileNames, file.Name())
+	}
+	if !reflect.DeepEqual(fileNames, tFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	}
+}
+
+func TestReadEnv(t *testing.T) {
+	env, err := ReadEnv(expectedAppFile.Path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(env, expectedAppEnv) {
+		t.Fatalf(
+			"did not get expected application settings. Expected: DOMAIN=%s TYPE=%s; Got: DOMAIN=%s TYPE=%s",
+			expectedAppEnv["DOMAIN"],
+			expectedAppEnv["TYPE"],
+			env["DOMAIN"],
+			env["TYPE"],
+		)
+	}
+}

pkg/dns/common.go

@@ -0,0 +1,28 @@
+package dns
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+)
+
+// NewToken constructs a new DNS provider token.
+func NewToken(provider, providerTokenEnvVar string) (string, error) {
+	if token, present := os.LookupEnv(providerTokenEnvVar); present {
+		return token, nil
+	}
+
+	logrus.Debugf("no %s in environment, asking via stdin", providerTokenEnvVar)
+
+	var token string
+	prompt := &survey.Input{
+		Message: fmt.Sprintf("%s API token?", provider),
+	}
+	if err := survey.AskOne(prompt, &token); err != nil {
+		return "", err
+	}
+
+	return token, nil
+}

pkg/dns/gandi/gandi.go

@@ -0,0 +1,15 @@
+package gandi
+
+import (
+	"coopcloud.tech/abra/pkg/dns"
+	"github.com/libdns/gandi"
+)
+
+// New constructs a new DNs provider.
+func New() (gandi.Provider, error) {
+	token, err := dns.NewToken("Gandi", "GANDI_TOKEN")
+	if err != nil {
+		return gandi.Provider{}, err
+	}
+	return gandi.Provider{APIToken: token}, nil
+}

pkg/git/clone.go

@@ -0,0 +1,98 @@
+package git
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// Clone runs a git clone which accounts for different default branches.
+func Clone(dir, url string) error {
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		logrus.Debugf("'%s' does not exist, attempting to git clone from '%s'", dir, url)
+		_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
+		if err != nil {
+			logrus.Debugf("cloning '%s' default branch failed, attempting from main branch", url)
+			_, err := git.PlainClone(dir, false, &git.CloneOptions{
+				URL:           url,
+				Tags:          git.AllTags,
+				ReferenceName: plumbing.ReferenceName("refs/heads/main"),
+			})
+			if err != nil {
+				if strings.Contains(err.Error(), "authentication required") {
+					name := filepath.Base(dir)
+					return fmt.Errorf("unable to clone %s, does %s exist?", name, url)
+				}
+				return err
+			}
+		}
+		logrus.Debugf("'%s' has been git cloned successfully", dir)
+	} else {
+		logrus.Debugf("'%s' already exists, doing nothing", dir)
+	}
+
+	return nil
+}
+
+// EnsureUpToDate ensures that a git repo on disk has the latest changes (git-fetch).
+func EnsureUpToDate(dir string) error {
+	repo, err := git.PlainOpen(dir)
+	if err != nil {
+		return err
+	}
+
+	branch := "master"
+	if _, err := repo.Branch("master"); err != nil {
+		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", dir)
+			return err
+		}
+		branch = "main"
+	}
+
+	logrus.Debugf("choosing '%s' as main git branch in '%s'", branch, dir)
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	refName := fmt.Sprintf("refs/heads/%s", branch)
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: plumbing.ReferenceName(refName),
+	}
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", refName, dir)
+		return err
+	}
+
+	logrus.Debugf("successfully checked out '%s' in '%s'", branch, dir)
+
+	remote, err := repo.Remote("origin")
+	if err != nil {
+		return err
+	}
+
+	fetchOpts := &git.FetchOptions{
+		RemoteName: "origin",
+		RefSpecs:   []config.RefSpec{"refs/heads/*:refs/remotes/origin/*"},
+		Force:      true,
+	}
+	if err := remote.Fetch(fetchOpts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return err
+		}
+	}
+
+	logrus.Debugf("successfully fetched all changes in '%s'", dir)
+
+	return nil
+}

pkg/git/read.go

@@ -0,0 +1,55 @@
+package git
+
+import (
+	"path"
+
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// GetRecipeHead retrieves latest HEAD metadata.
+func GetRecipeHead(recipeName string) (*plumbing.Reference, error) {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return nil, err
+	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return nil, err
+	}
+
+	return head, nil
+}
+
+// IsClean checks if a repo has unstaged changes
+func IsClean(recipeName string) (bool, error) {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return false, err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return false, err
+	}
+
+	status, err := worktree.Status()
+	if err != nil {
+		return false, err
+	}
+
+	if status.String() != "" {
+		logrus.Debugf("discovered git status for %s repository: %s", recipeName, status.String())
+	} else {
+		logrus.Debugf("discovered clean git status for %s repository", recipeName)
+	}
+
+	return status.IsClean(), nil
+}

pkg/limit/limit.go

@@ -0,0 +1,20 @@
+package limit // https://github.com/tidwall/limiter
+
+// Limiter is for limiting the number of concurrent operations. This
+type Limiter struct{ sem chan struct{} }
+
+// New returns a new Limiter. The limit param is the maximum number of
+// concurrent operations.
+func New(limit int) *Limiter {
+	return &Limiter{make(chan struct{}, limit)}
+}
+
+// Begin an operation.
+func (l *Limiter) Begin() {
+	l.sem <- struct{}{}
+}
+
+// End the operation.
+func (l *Limiter) End() {
+	<-l.sem
+}

pkg/recipe/recipe.go

@@ -0,0 +1,238 @@
+package recipe
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/compose"
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/sirupsen/logrus"
+)
+
+// Recipe represents a recipe.
+type Recipe struct {
+	Name   string
+	Config *composetypes.Config
+}
+
+// UpdateLabel updates a recipe label
+func (r Recipe) UpdateLabel(serviceName, label string) error {
+	pattern := fmt.Sprintf("%s/%s/compose**yml", config.APPS_DIR, r.Name)
+	if err := compose.UpdateLabel(pattern, serviceName, label, r.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+// UpdateTag updates a recipe tag
+func (r Recipe) UpdateTag(image, tag string) error {
+	pattern := fmt.Sprintf("%s/%s/compose**yml", config.APPS_DIR, r.Name)
+	if err := compose.UpdateTag(pattern, image, tag, r.Name); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Tags list the recipe tags
+func (r Recipe) Tags() ([]string, error) {
+	var tags []string
+
+	recipeDir := path.Join(config.ABRA_DIR, "apps", r.Name)
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return tags, err
+	}
+
+	gitTags, err := repo.Tags()
+	if err != nil {
+		return tags, err
+	}
+
+	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
+		tags = append(tags, strings.TrimPrefix(string(ref.Name()), "refs/tags/"))
+		return nil
+	}); err != nil {
+		return tags, err
+	}
+
+	logrus.Debugf("detected '%s' as tags for recipe '%s'", strings.Join(tags, ", "), r.Name)
+
+	return tags, nil
+}
+
+// Get retrieves a recipe.
+func Get(recipeName string) (Recipe, error) {
+	if err := EnsureExists(recipeName); err != nil {
+		return Recipe{}, err
+	}
+
+	pattern := fmt.Sprintf("%s/%s/compose**yml", config.APPS_DIR, recipeName)
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return Recipe{}, err
+	}
+
+	envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
+	sampleEnv, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	config, err := loader.LoadComposefile(opts, sampleEnv)
+	if err != nil {
+		return Recipe{}, err
+	}
+
+	return Recipe{Name: recipeName, Config: config}, nil
+}
+
+// EnsureExists checks whether a recipe has been cloned locally or not.
+func EnsureExists(recipe string) error {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(recipe))
+	url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe)
+	if err := gitPkg.Clone(recipeDir, url); err != nil {
+		return err
+	}
+	return nil
+}
+
+// EnsureVersion checks whether a specific version exists for a recipe.
+func EnsureVersion(recipeName, version string) error {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	isClean, err := gitPkg.IsClean(recipeName)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
+	}
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return err
+	}
+
+	tags, err := repo.Tags()
+	if err != nil {
+		return nil
+	}
+
+	var parsedTags []string
+	var tagRef plumbing.ReferenceName
+	if err := tags.ForEach(func(ref *plumbing.Reference) (err error) {
+		parsedTags = append(parsedTags, ref.Name().Short())
+		if ref.Name().Short() == version {
+			tagRef = ref.Name()
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	logrus.Debugf("read '%s' as tags for recipe '%s'", strings.Join(parsedTags, ", "), recipeName)
+
+	if tagRef.String() == "" {
+		return fmt.Errorf("%s is not available?", version)
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	opts := &git.CheckoutOptions{
+		Branch: tagRef,
+		Create: false,
+		Force:  true,
+	}
+	if err := worktree.Checkout(opts); err != nil {
+		return err
+	}
+
+	logrus.Debugf("successfully checked '%s' out to '%s' in '%s'", recipeName, tagRef.Short(), recipeDir)
+
+	return nil
+}
+
+// EnsureLatest makes sure the latest commit is checkout on for a local recipe repository.
+func EnsureLatest(recipeName string) error {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	isClean, err := gitPkg.IsClean(recipeName)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
+	}
+
+	logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	branch := "master"
+	if _, err := repo.Branch("master"); err != nil {
+		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", path.Join(config.APPS_DIR, recipeName))
+			return err
+		}
+		branch = "main"
+	}
+
+	refName := fmt.Sprintf("refs/heads/%s", branch)
+	checkOutOpts := &git.CheckoutOptions{
+		Create: false,
+		Force:  true,
+		Branch: plumbing.ReferenceName(refName),
+	}
+
+	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", branch, recipeDir)
+		return err
+	}
+
+	return nil
+}
+
+// ChaosVersion constructs a chaos mode recipe version.
+func ChaosVersion(recipeName string) (string, error) {
+	var version string
+
+	head, err := gitPkg.GetRecipeHead(recipeName)
+	if err != nil {
+		return version, err
+	}
+
+	version = head.String()[:8]
+
+	isClean, err := gitPkg.IsClean(recipeName)
+	if err != nil {
+		return version, err
+	}
+
+	if !isClean {
+		version = fmt.Sprintf("%s + unstaged changes", version)
+	}
+
+	return version, nil
+}

pkg/secret/pass.go

@@ -0,0 +1,53 @@
+package secret
+
+import (
+	"errors"
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+)
+
+// PassInsertSecret inserts a secret into a pass store.
+func PassInsertSecret(secretValue, secretName, appName, server string) error {
+	if _, err := exec.LookPath("pass"); err != nil {
+		return errors.New("pass command not found on $PATH, is it installed?")
+	}
+
+	cmd := fmt.Sprintf(
+		"echo %s | pass insert hosts/%s/%s/%s -m",
+		secretValue, server, appName, secretName,
+	)
+
+	logrus.Debugf("attempting to run '%s'", cmd)
+
+	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
+		return err
+	}
+
+	logrus.Infof("'%s' inserted into pass store", secretName)
+
+	return nil
+}
+
+// PassRmSecret deletes a secret from a pass store.
+func PassRmSecret(secretName, appName, server string) error {
+	if _, err := exec.LookPath("pass"); err != nil {
+		return errors.New("pass command not found on $PATH, is it installed?")
+	}
+
+	cmd := fmt.Sprintf(
+		"pass rm --force hosts/%s/%s/%s",
+		server, appName, secretName,
+	)
+
+	logrus.Debugf("attempting to run '%s'", cmd)
+
+	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
+		return err
+	}
+
+	logrus.Infof("'%s' removed from pass store", secretName)
+
+	return nil
+}

pkg/secret/secret.go

@@ -0,0 +1,172 @@
+// Package secret provides functionality for generating and storing secrets
+// both in a remote swarm and locally within supported storage such as pass
+// stores.
+package secret
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/schultz-is/passgen"
+	"github.com/sirupsen/logrus"
+)
+
+// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
+// secret definition.
+type secretValue struct {
+	Version string
+	Length  int
+}
+
+// GeneratePasswords generates passwords.
+func GeneratePasswords(count, length uint) ([]string, error) {
+	passwords, err := passgen.GeneratePasswords(
+		count,
+		length,
+		passgen.AlphabetDefault,
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("generated '%s'", strings.Join(passwords, ", "))
+
+	return passwords, nil
+}
+
+// GeneratePassphrases generates human readable and rememberable passphrases.
+func GeneratePassphrases(count uint) ([]string, error) {
+	passphrases, err := passgen.GeneratePassphrases(
+		count,
+		passgen.PassphraseWordCountDefault,
+		rune('-'),
+		passgen.PassphraseCasingDefault,
+		passgen.WordListDefault,
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("generated '%s'", strings.Join(passphrases, ", "))
+
+	return passphrases, nil
+}
+
+// ReadSecretEnvVars reads secret env vars from an app env var config.
+func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
+	secretEnvVars := make(map[string]string)
+
+	for envVar := range appEnv {
+		regex := regexp.MustCompile(`^SECRET.*VERSION.*`)
+		if string(regex.Find([]byte(envVar))) != "" {
+			secretEnvVars[envVar] = appEnv[envVar]
+		}
+	}
+
+	logrus.Debugf("read '%s' as secrets from '%s'", secretEnvVars, appEnv)
+
+	return secretEnvVars
+}
+
+// TODO: should probably go in the config/app package?
+func ParseSecretEnvVarName(secretEnvVar string) string {
+	withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_")
+	withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION")
+	name := strings.ToLower(withoutSuffix)
+	logrus.Debugf("parsed '%s' as name from '%s'", name, secretEnvVar)
+	return name
+}
+
+// TODO: should probably go in the config/app package?
+func ParseGeneratedSecretName(secret string, appEnv config.App) string {
+	name := fmt.Sprintf("%s_", appEnv.StackName())
+	withoutAppName := strings.TrimPrefix(secret, name)
+	idx := strings.LastIndex(withoutAppName, "_")
+	parsed := withoutAppName[:idx]
+	logrus.Debugf("parsed '%s' as name from '%s'", parsed, secret)
+	return parsed
+}
+
+// TODO: should probably go in the config/app package?
+func ParseSecretEnvVarValue(secret string) (secretValue, error) {
+	values := strings.Split(secret, "#")
+	if len(values) == 0 {
+		return secretValue{}, fmt.Errorf("unable to parse '%s'", secret)
+	}
+
+	if len(values) == 1 {
+		return secretValue{Version: values[0], Length: 0}, nil
+	}
+
+	split := strings.Split(values[1], "=")
+	parsed := split[len(split)-1]
+	stripped := strings.ReplaceAll(parsed, " ", "")
+	length, err := strconv.Atoi(stripped)
+	if err != nil {
+		return secretValue{}, err
+	}
+	version := strings.ReplaceAll(values[0], " ", "")
+
+	logrus.Debugf("parsed version '%s' and length '%v' from '%s'", version, length, secret)
+
+	return secretValue{Version: version, Length: length}, nil
+}
+
+// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
+func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
+	secrets := make(map[string]string)
+
+	ch := make(chan error, len(secretEnvVars))
+	for secretEnvVar := range secretEnvVars {
+		go func(s string) {
+			secretName := ParseSecretEnvVarName(s)
+			secretValue, err := ParseSecretEnvVarValue(secretEnvVars[s])
+			if err != nil {
+				ch <- err
+				return
+			}
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version)
+			logrus.Debugf("attempting to generate and store '%s' on '%s'", secretRemoteName, server)
+			if secretValue.Length > 0 {
+				passwords, err := GeneratePasswords(1, uint(secretValue.Length))
+				if err != nil {
+					ch <- err
+					return
+				}
+				if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
+					ch <- err
+					return
+				}
+				secrets[secretName] = passwords[0]
+			} else {
+				passphrases, err := GeneratePassphrases(1)
+				if err != nil {
+					ch <- err
+					return
+				}
+				if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
+					ch <- err
+				}
+				secrets[secretName] = passphrases[0]
+			}
+			ch <- nil
+		}(secretEnvVar)
+	}
+
+	for range secretEnvVars {
+		err := <-ch
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	logrus.Debugf("generated and stored '%s' on '%s'", secrets, server)
+
+	return secrets, nil
+}

pkg/server/server.go

@@ -0,0 +1,27 @@
+package server
+
+import (
+	"os"
+	"path"
+
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+)
+
+// CreateServerDir creates a server directory under ~/.abra.
+func CreateServerDir(serverName string) error {
+	serverPath := path.Join(config.ABRA_DIR, "servers", serverName)
+
+	if err := os.Mkdir(serverPath, 0755); err != nil {
+		if !os.IsExist(err) {
+			return err
+		}
+
+		logrus.Infof("%s already exists", serverPath)
+		return nil
+	}
+
+	logrus.Infof("successfully created %s", serverPath)
+
+	return nil
+}

pkg/ssh/ssh.go

@@ -0,0 +1,226 @@
+package ssh
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os/user"
+	"sync"
+	"time"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/kevinburke/ssh_config"
+	"github.com/sfreiberg/simplessh"
+	"github.com/sirupsen/logrus"
+)
+
+// HostConfig is a SSH host config.
+type HostConfig struct {
+	Host         string
+	IdentityFile string
+	Port         string
+	User         string
+}
+
+// GetHostConfig retrieves a ~/.ssh/config config for a host.
+func GetHostConfig(hostname, username, port string) (HostConfig, error) {
+	var hostConfig HostConfig
+
+	var host, idf string
+
+	if host = ssh_config.Get(hostname, "Hostname"); host == "" {
+		logrus.Debugf("no hostname found in SSH config, assuming %s", hostname)
+		host = hostname
+	}
+
+	if username == "" {
+		if username = ssh_config.Get(hostname, "User"); username == "" {
+			systemUser, err := user.Current()
+			if err != nil {
+				return hostConfig, err
+			}
+			logrus.Debugf("no username found in SSH config or passed on command-line, assuming %s", username)
+			username = systemUser.Username
+		}
+	}
+
+	if port == "" {
+		if port = ssh_config.Get(hostname, "Port"); port == "" {
+			logrus.Debugf("no port found in SSH config or passed on command-line, assuming 22")
+			port = "22"
+		}
+	}
+
+	idf = ssh_config.Get(hostname, "IdentityFile")
+
+	hostConfig.Host = host
+	if idf != "" {
+		hostConfig.IdentityFile = idf
+	}
+	hostConfig.Port = port
+	hostConfig.User = username
+
+	logrus.Debugf("constructed SSH config %s for %s", hostConfig, hostname)
+
+	return hostConfig, nil
+}
+
+// New creates a new SSH client connection.
+func New(domainName, sshAuth, username, port string) (*simplessh.Client, error) {
+	var client *simplessh.Client
+
+	hostConfig, err := GetHostConfig(domainName, username, port)
+	if err != nil {
+		return client, err
+	}
+
+	if sshAuth == "identity-file" {
+		var err error
+		client, err = simplessh.ConnectWithAgentTimeout(hostConfig.Host, hostConfig.User, 5*time.Second)
+		if err != nil {
+			return client, err
+		}
+	} else {
+		password := ""
+		prompt := &survey.Password{
+			Message: "SSH password?",
+		}
+		if err := survey.AskOne(prompt, &password); err != nil {
+			return client, err
+		}
+
+		var err error
+		client, err = simplessh.ConnectWithPasswordTimeout(hostConfig.Host, hostConfig.User, password, 5*time.Second)
+		if err != nil {
+			return client, err
+		}
+	}
+
+	return client, nil
+}
+
+// sudoWriter supports sudo command handling.
+// https://github.com/sfreiberg/simplessh/blob/master/simplessh.go
+type sudoWriter struct {
+	b     bytes.Buffer
+	pw    string
+	stdin io.Writer
+	m     sync.Mutex
+}
+
+// Write satisfies the write interface for sudoWriter.
+// https://github.com/sfreiberg/simplessh/blob/master/simplessh.go
+func (w *sudoWriter) Write(p []byte) (int, error) {
+	if string(p) == "sudo_password" {
+		w.stdin.Write([]byte(w.pw + "\n"))
+		w.pw = ""
+		return len(p), nil
+	}
+
+	w.m.Lock()
+	defer w.m.Unlock()
+
+	return w.b.Write(p)
+}
+
+// RunSudoCmd runs SSH commands and streams output.
+// https://github.com/sfreiberg/simplessh/blob/master/simplessh.go
+func RunSudoCmd(cmd, passwd string, cl *simplessh.Client) error {
+	session, err := cl.SSHClient.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	cmd = "sudo -p " + "sudo_password" + " -S " + cmd
+
+	w := &sudoWriter{
+		pw: passwd,
+	}
+	w.stdin, err = session.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	session.Stdout = w
+	session.Stderr = w
+
+	done := make(chan struct{})
+	scanner := bufio.NewScanner(session.Stdin)
+
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			fmt.Println(line)
+		}
+		done <- struct{}{}
+	}()
+
+	if err := session.Start(cmd); err != nil {
+		return err
+	}
+
+	<-done
+
+	if err := session.Wait(); err != nil {
+		return err
+	}
+
+	return err
+}
+
+// Exec runs a command on a remote and streams output.
+// https://github.com/sfreiberg/simplessh/blob/master/simplessh.go
+func Exec(cmd string, cl *simplessh.Client) error {
+	session, err := cl.SSHClient.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	stdout, err := session.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	stderr, err := session.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	stdoutDone := make(chan struct{})
+	stdoutScanner := bufio.NewScanner(stdout)
+
+	go func() {
+		for stdoutScanner.Scan() {
+			line := stdoutScanner.Text()
+			fmt.Println(line)
+		}
+		stdoutDone <- struct{}{}
+	}()
+
+	stderrDone := make(chan struct{})
+	stderrScanner := bufio.NewScanner(stderr)
+
+	go func() {
+		for stderrScanner.Scan() {
+			line := stderrScanner.Text()
+			fmt.Println(line)
+		}
+		stderrDone <- struct{}{}
+	}()
+
+	if err := session.Start(cmd); err != nil {
+		return err
+	}
+
+	<-stdoutDone
+	<-stderrDone
+
+	if err := session.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/upstream/commandconn/commandconn.go

@@ -0,0 +1,295 @@
+// Package commandconn provides a net.Conn implementation that can be used for
+// proxying (or emulating) stream via a custom command.
+//
+// For example, to provide an http.Client that can connect to a Docker daemon
+// running in a Docker container ("DIND"):
+//
+//  httpClient := &http.Client{
+//  	Transport: &http.Transport{
+//  		DialContext: func(ctx context.Context, _network, _addr string) (net.Conn, error) {
+//  			return commandconn.New(ctx, "docker", "exec", "-it", containerID, "docker", "system", "dial-stdio")
+//  		},
+//  	},
+//  }
+
+package commandconn
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	exec "golang.org/x/sys/execabs"
+)
+
+func createSession(cmd *exec.Cmd) {
+	// for supporting ssh connection helper with ProxyCommand
+	// https://github.com/docker/cli/issues/1707
+	cmd.SysProcAttr.Setsid = true
+}
+
+// New returns net.Conn
+func New(ctx context.Context, cmd string, args ...string) (net.Conn, error) {
+	var (
+		c   commandConn
+		err error
+	)
+	c.cmd = exec.CommandContext(ctx, cmd, args...)
+	// we assume that args never contains sensitive information
+	logrus.Debugf("commandconn: starting %s with %v", cmd, args)
+	c.cmd.Env = os.Environ()
+	c.cmd.SysProcAttr = &syscall.SysProcAttr{}
+	setPdeathsig(c.cmd)
+	createSession(c.cmd)
+	c.stdin, err = c.cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	c.stdout, err = c.cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	c.cmd.Stderr = &stderrWriter{
+		stderrMu:    &c.stderrMu,
+		stderr:      &c.stderr,
+		debugPrefix: fmt.Sprintf("commandconn (%s):", cmd),
+	}
+	c.localAddr = dummyAddr{network: "dummy", s: "dummy-0"}
+	c.remoteAddr = dummyAddr{network: "dummy", s: "dummy-1"}
+	return &c, c.cmd.Start()
+}
+
+// commandConn implements net.Conn
+type commandConn struct {
+	cmd           *exec.Cmd
+	cmdExited     bool
+	cmdWaitErr    error
+	cmdMutex      sync.Mutex
+	stdin         io.WriteCloser
+	stdout        io.ReadCloser
+	stderrMu      sync.Mutex
+	stderr        bytes.Buffer
+	stdioClosedMu sync.Mutex // for stdinClosed and stdoutClosed
+	stdinClosed   bool
+	stdoutClosed  bool
+	localAddr     net.Addr
+	remoteAddr    net.Addr
+}
+
+// killIfStdioClosed kills the cmd if both stdin and stdout are closed.
+func (c *commandConn) killIfStdioClosed() error {
+	c.stdioClosedMu.Lock()
+	stdioClosed := c.stdoutClosed && c.stdinClosed
+	c.stdioClosedMu.Unlock()
+	if !stdioClosed {
+		return nil
+	}
+	return c.kill()
+}
+
+// killAndWait tries sending SIGTERM to the process before sending SIGKILL.
+func killAndWait(cmd *exec.Cmd) error {
+	var werr error
+	if runtime.GOOS != "windows" {
+		werrCh := make(chan error)
+		go func() { werrCh <- cmd.Wait() }()
+		cmd.Process.Signal(syscall.SIGTERM)
+		select {
+		case werr = <-werrCh:
+		case <-time.After(3 * time.Second):
+			cmd.Process.Kill()
+			werr = <-werrCh
+		}
+	} else {
+		cmd.Process.Kill()
+		werr = cmd.Wait()
+	}
+	return werr
+}
+
+// kill returns nil if the command terminated, regardless to the exit status.
+func (c *commandConn) kill() error {
+	var werr error
+	c.cmdMutex.Lock()
+	if c.cmdExited {
+		werr = c.cmdWaitErr
+	} else {
+		werr = killAndWait(c.cmd)
+		c.cmdWaitErr = werr
+		c.cmdExited = true
+	}
+	c.cmdMutex.Unlock()
+	if werr == nil {
+		return nil
+	}
+	wExitErr, ok := werr.(*exec.ExitError)
+	if ok {
+		if wExitErr.ProcessState.Exited() {
+			return nil
+		}
+	}
+	return errors.Wrapf(werr, "commandconn: failed to wait")
+}
+
+func (c *commandConn) onEOF(eof error) error {
+	// when we got EOF, the command is going to be terminated
+	var werr error
+	c.cmdMutex.Lock()
+	if c.cmdExited {
+		werr = c.cmdWaitErr
+	} else {
+		werrCh := make(chan error)
+		go func() { werrCh <- c.cmd.Wait() }()
+		select {
+		case werr = <-werrCh:
+			c.cmdWaitErr = werr
+			c.cmdExited = true
+		case <-time.After(10 * time.Second):
+			c.cmdMutex.Unlock()
+			c.stderrMu.Lock()
+			stderr := c.stderr.String()
+			c.stderrMu.Unlock()
+			return errors.Errorf("command %v did not exit after %v: stderr=%q", c.cmd.Args, eof, stderr)
+		}
+	}
+	c.cmdMutex.Unlock()
+	if werr == nil {
+		return eof
+	}
+	c.stderrMu.Lock()
+	stderr := c.stderr.String()
+	c.stderrMu.Unlock()
+	return errors.Errorf("command %v has exited with %v, please make sure the URL is valid, and Docker 18.09 or later is installed on the remote host: stderr=%s", c.cmd.Args, werr, stderr)
+}
+
+func ignorableCloseError(err error) bool {
+	errS := err.Error()
+	ss := []string{
+		os.ErrClosed.Error(),
+	}
+	for _, s := range ss {
+		if strings.Contains(errS, s) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *commandConn) CloseRead() error {
+	// NOTE: maybe already closed here
+	if err := c.stdout.Close(); err != nil && !ignorableCloseError(err) {
+		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// logrus.Warnf("commandConn.CloseRead: %v", err)
+	}
+	c.stdioClosedMu.Lock()
+	c.stdoutClosed = true
+	c.stdioClosedMu.Unlock()
+	if err := c.killIfStdioClosed(); err != nil {
+		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// logrus.Warnf("commandConn.CloseRead: %v", err)
+	}
+	return nil
+}
+
+func (c *commandConn) Read(p []byte) (int, error) {
+	n, err := c.stdout.Read(p)
+	if err == io.EOF {
+		err = c.onEOF(err)
+	}
+	return n, err
+}
+
+func (c *commandConn) CloseWrite() error {
+	// NOTE: maybe already closed here
+	if err := c.stdin.Close(); err != nil && !ignorableCloseError(err) {
+		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// logrus.Warnf("commandConn.CloseWrite: %v", err)
+	}
+	c.stdioClosedMu.Lock()
+	c.stdinClosed = true
+	c.stdioClosedMu.Unlock()
+	if err := c.killIfStdioClosed(); err != nil {
+		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// logrus.Warnf("commandConn.CloseWrite: %v", err)
+	}
+	return nil
+}
+
+func (c *commandConn) Write(p []byte) (int, error) {
+	n, err := c.stdin.Write(p)
+	if err == io.EOF {
+		err = c.onEOF(err)
+	}
+	return n, err
+}
+
+func (c *commandConn) Close() error {
+	var err error
+	if err = c.CloseRead(); err != nil {
+		logrus.Warnf("commandConn.Close: CloseRead: %v", err)
+	}
+	if err = c.CloseWrite(); err != nil {
+		// TODO: muted because https://github.com/docker/compose/issues/8544
+		// logrus.Warnf("commandConn.Close: CloseWrite: %v", err)
+	}
+	return err
+}
+
+func (c *commandConn) LocalAddr() net.Addr {
+	return c.localAddr
+}
+func (c *commandConn) RemoteAddr() net.Addr {
+	return c.remoteAddr
+}
+func (c *commandConn) SetDeadline(t time.Time) error {
+	logrus.Debugf("unimplemented call: SetDeadline(%v)", t)
+	return nil
+}
+func (c *commandConn) SetReadDeadline(t time.Time) error {
+	logrus.Debugf("unimplemented call: SetReadDeadline(%v)", t)
+	return nil
+}
+func (c *commandConn) SetWriteDeadline(t time.Time) error {
+	logrus.Debugf("unimplemented call: SetWriteDeadline(%v)", t)
+	return nil
+}
+
+type dummyAddr struct {
+	network string
+	s       string
+}
+
+func (d dummyAddr) Network() string {
+	return d.network
+}
+
+func (d dummyAddr) String() string {
+	return d.s
+}
+
+type stderrWriter struct {
+	stderrMu    *sync.Mutex
+	stderr      *bytes.Buffer
+	debugPrefix string
+}
+
+func (w *stderrWriter) Write(p []byte) (int, error) {
+	logrus.Debugf("%s%s", w.debugPrefix, string(p))
+	w.stderrMu.Lock()
+	if w.stderr.Len() > 4096 {
+		w.stderr.Reset()
+	}
+	n, err := w.stderr.Write(p)
+	w.stderrMu.Unlock()
+	return n, err
+}

pkg/upstream/commandconn/connection.go

@@ -0,0 +1,82 @@
+package commandconn
+
+import (
+	"context"
+	"net"
+	"net/url"
+
+	"github.com/docker/cli/cli/connhelper"
+	"github.com/docker/cli/cli/connhelper/ssh"
+	"github.com/docker/cli/cli/context/docker"
+	dCliContextStore "github.com/docker/cli/cli/context/store"
+	dClient "github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// GetConnectionHelper returns Docker-specific connection helper for the given URL.
+// GetConnectionHelper returns nil without error when no helper is registered for the scheme.
+//
+// ssh://<user>@<host> URL requires Docker 18.09 or later on the remote host.
+func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
+	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=5"})
+}
+
+func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) {
+	u, err := url.Parse(daemonURL)
+	if err != nil {
+		return nil, err
+	}
+	switch scheme := u.Scheme; scheme {
+	case "ssh":
+		sp, err := ssh.ParseURL(daemonURL)
+		if err != nil {
+			return nil, errors.Wrap(err, "ssh host connection is not valid")
+		}
+		return &connhelper.ConnectionHelper{
+			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return New(ctx, "ssh", append(sshFlags, sp.Args("docker", "system", "dial-stdio")...)...)
+			},
+			Host: "http://docker.example.com",
+		}, nil
+	}
+	// Future version may support plugins via ~/.docker/config.json. e.g. "dind"
+	// See docker/cli#889 for the previous discussion.
+	return nil, err
+}
+
+func NewConnectionHelper(daemonURL string) *connhelper.ConnectionHelper {
+	helper, err := GetConnectionHelper(daemonURL)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	return helper
+}
+
+func getDockerEndpoint(host string) (docker.Endpoint, error) {
+	skipTLSVerify := false
+	ep := docker.Endpoint{
+		EndpointMeta: docker.EndpointMeta{
+			Host:          host,
+			SkipTLSVerify: skipTLSVerify,
+		},
+	}
+	// try to resolve a docker client, validating the configuration
+	opts, err := ep.ClientOpts()
+	if err != nil {
+		return docker.Endpoint{}, err
+	}
+	if _, err := dClient.NewClientWithOpts(opts...); err != nil {
+		return docker.Endpoint{}, err
+	}
+	return ep, nil
+}
+
+func GetDockerEndpointMetadataAndTLS(host string) (docker.EndpointMeta, *dCliContextStore.EndpointTLSData, error) {
+	ep, err := getDockerEndpoint(host)
+	if err != nil {
+		return docker.EndpointMeta{}, nil, err
+	}
+	return ep.EndpointMeta, ep.TLSData.ToStoreTLSData(), nil
+}

pkg/upstream/commandconn/pdeathsig_linux.go

@@ -0,0 +1,10 @@
+package commandconn
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+func setPdeathsig(cmd *exec.Cmd) {
+	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL
+}

pkg/upstream/commandconn/pdeathsig_nolinux.go

@@ -0,0 +1,11 @@
+//go:build !linux
+// +build !linux
+
+package commandconn
+
+import (
+	"os/exec"
+)
+
+func setPdeathsig(cmd *exec.Cmd) {
+}