chore: make deps

Updated by "Update PO files to match POT (msgmerge)" add-on in Weblate.

Translation: Co-op Cloud/abra
Translate-URL: https://translate.coopcloud.tech/projects/co-op-cloud/abra/

.dockerignore

@@ -0,0 +1,7 @@
+*.swo
+*.swp
+.dockerignore
+Dockerfile
+abra
+dist
+tags

.drone.yml

@@ -1,13 +1,149 @@
 ---
 kind: pipeline
-name: linters
+name: coopcloud.tech/abra
 steps:
-  - name: run shellcheck
-    image: debian:buster
+  - name: make check
+    image: golang:1.24
     commands:
-      - apt update
-      - apt install -y shellcheck
-      - shellcheck abra installer
+      - make check
+
+  - name: xgettext-go
+    image: git.coopcloud.tech/toolshed/drone-xgettext-go:latest
+    settings:
+      keyword: i18n.G
+      keyword_ctx: i18n.GC
+      out: pkg/i18n/locales/abra.pot
+      comments_tag: translators
+    depends_on:
+      - make check
+    when:
+      event:
+        exclude:
+          - tag
+
+  - name: xgettext-go status
+    image: golang:1.24-alpine3.22
+    commands:
+      - apk add patchutils git make
+      - cd /drone/src
+      - sed -i "s/charset=CHARSET/charset=UTF-8/g" pkg/i18n/locales/*.pot
+      - git diff pkg/i18n/locales/abra.pot | grepdiff --output-matching=hunk POT-Creation-Date | git apply --reverse --allow-empty
+      - git diff
+      - git diff-files --exit-code
+    depends_on:
+      - xgettext-go
+    when:
+      event:
+        exclude:
+          - tag
+
+  - name: make test
+    image: golang:1.24
+    environment:
+      ABRA_DIR: $HOME/.abra
+      CATL_URL: https://git.coopcloud.tech/toolshed/recipes-catalogue-json.git
+    commands:
+      - mkdir -p $HOME/.abra
+      - git clone $CATL_URL $HOME/.abra/catalogue
+      - make test
+    depends_on:
+      - make check
+
+  - name: fetch
+    image: docker:git
+    commands:
+      - git fetch --tags
+    depends_on:
+      - make check
+      - make test
+    when:
+      event: tag
+
+  - name: release
+    image: goreleaser/goreleaser:v2.5.1
+    environment:
+      GITEA_TOKEN:
+        from_secret: goreleaser_gitea_token
+    volumes:
+      - name: deps
+        path: /go
+    commands:
+      - goreleaser release
+    depends_on:
+      - fetch
+    when:
+      event: tag
+
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: abra-bot
+      password:
+        from_secret: git_coopcloud_tech_token_abra_bot
+      repo: git.coopcloud.tech/toolshed/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      branch:
+        - main
+    depends_on:
+      - make check
+      - make test
+
+  - name: on-demand integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      request_pty: true
+      script:
+        - |
+          wget https://git.coopcloud.tech/toolshed/abra/raw/branch/main/scripts/tests/run-ci-int -O run-ci-int
+          chmod +x run-ci-int
+          sh run-ci-int
+    when:
+      ref:
+        - refs/heads/int-*
+    depends_on:
+      - make check
+      - make test
+
+  - name: nightly integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      request_pty: true
+      script:
+        - |
+          wget https://git.coopcloud.tech/toolshed/abra/raw/branch/main/scripts/tests/run-ci-int -O run-ci-int
+          chmod +x run-ci-int
+          sh run-ci-int
+    when:
+      event:
+        - cron:
+      cron:
+        # @daily https://docs.drone.io/cron/
+        - integration 
+
+volumes:
+  - name: deps
+    temp: {}
+
 trigger:
-  branch:
-    - main
+  action:
+    exclude:
+      - synchronized

.envrc.sample

@@ -0,0 +1,7 @@
+# integration test suite
+# export ABRA_DIR="$HOME/.abra_test"
+# export TEST_SERVER=test.example.com
+# export ABRA_CI=1
+
+# release automation
+# export GITEA_TOKEN=

.gitignore

@@ -0,0 +1,9 @@
+*.tar.gz
+*fmtcoverage.html
+.e2e.env
+.envrc
+.vscode/
+/abra
+/bin
+dist/
+tests/integration/.bats

.goreleaser.yml

@@ -0,0 +1,51 @@
+---
+gitea_urls:
+  api: https://git.coopcloud.tech/api/v1
+  download: https://git.coopcloud.tech/
+  skip_tls_verify: false
+
+before:
+  hooks:
+    - go mod tidy
+
+builds:
+  - id: abra
+    binary: abra
+    dir: cmd/abra
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+      - "-s"
+      - "-w"
+
+checksum:
+  name_template: "checksums.txt"
+
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+
+changelog:
+  sort: desc
+  filters:
+    exclude:
+      - "^Merge"
+      - "^Revert"
+      - "^WIP:"
+      - "^chore(deps):"
+      - "^style:"
+      - "^test:"
+      - "^tests:"

AUTHORS.md

@@ -0,0 +1,25 @@
+# authors
+
+> If you're looking at this and you hack on `abra` and you're not listed here,
+> please do add yourself! This is a community project, let's show some 💞
+
+- 3wordchant
+- ammaratef45
+- apfelwurm
+- basebuilder
+- cassowary
+- chasqui
+- codegod100
+- decentral1se
+- fauno
+- frando
+- iexos
+- kawaiipunk
+- knoflook
+- mayel
+- moritz
+- p4u1
+- rix
+- roxxers
+- vera
+- yksflip

CHANGELOG.md

@@ -1,15 +0,0 @@
-# abra 0.2.0 (2020-09-24)
-
-- Prepare for swarm install script using script.d ([#12](https://git.autonomic.zone/compose-stacks/planning/issues/12))
-
-# abra 0.1.2 (2020-09-22)
-
-- Add upgrade command ([#10](https://git.autonomic.zone/autonomic-cooperative/abra/issues/10))
-
-# abra 0.1.1 (2020-09-22)
-
-- Add installer script ([#9](https://git.autonomic.zone/autonomic-cooperative/abra/issues/9))
-
-# abra 0.1.0 (2020-09-22)
-
-- Initial pre-alpha release

Dockerfile

@@ -0,0 +1,30 @@
+# Build image
+FROM golang:1.24-alpine AS build
+
+ENV GOPRIVATE=coopcloud.tech
+
+RUN apk add --no-cache \
+  gcc \
+  git \
+  make \
+  musl-dev
+
+COPY . /app
+
+WORKDIR /app
+
+RUN CGO_ENABLED=0 make build
+
+# Release image ("slim")
+FROM alpine:3.19.1
+
+RUN apk add --no-cache \
+  ca-certificates \
+  git \
+  openssh
+
+RUN update-ca-certificates
+
+COPY --from=build /app/abra /abra
+
+ENTRYPOINT ["/abra"]

LICENSE

@@ -0,0 +1,15 @@
+Abra: The Co-op Cloud utility belt
+Copyright (C) 2022 Co-op Cloud <helo@coopcloud.tech>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

Makefile

@@ -1,14 +1,87 @@
-default: install
+ABRA         := ./cmd/abra
+XGETTEXT     := ./bin/xgettext-go
+COMMIT       := $(shell git rev-list -1 HEAD)
+GOPATH       := $(shell go env GOPATH)
+GOVERSION    := 1.24
+LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
+DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+GCFLAGS      := "all=-l -B"
+DOMAIN       := abra
+POFILES      := $(wildcard pkg/i18n/locales/*.po)
+MOFILES      := $(patsubst %.po,%.mo,$(POFILES))
+LINGUAS      := $(basename $(POFILES))
 
-dev_install:
-	ln -sf $(PWD)/abra ~/.local/bin
+export GOPRIVATE=coopcloud.tech
+
+all: format check build
+
+run:
+	@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
 
 install:
-	install abra /usr/bin/abra
+	@go install -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
 
-get_yq:
-	wget https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_amd64 && \
-		chmod +x yq_linux_amd64 && \
-		mv yq_linux_amd64 yq
+build:
+	@go build -v -gcflags=$(GCFLAGS) -ldflags=$(DIST_LDFLAGS) $(ABRA)
 
-.PHONY: dev_install install get_yq
+build-docker:
+	@docker run -it -v $(PWD):/abra golang:$(GOVERSION) \
+		bash -c 'cd /abra; ./scripts/docker/build.sh'
+
+clean:
+	@rm '$(GOPATH)/bin/abra'
+
+format:
+	@gofmt -s -w $$(find . -type f -name '*.go' | grep -v "/vendor/")
+
+check:
+	@test -z $$(gofmt -l $$(find . -type f -name '*.go' | grep -v "/vendor/")) || \
+	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+
+test:
+	@go test ./... -cover -v
+
+find-tests:
+	@find . -name "*_test.go"
+
+loc:
+	@find . -name "*.go" | xargs wc -l
+
+deps:
+	@go get -t -u ./...
+
+.PHONY: i18n
+i18n: update-pot update-pot-po-metadata update-po build-mo
+
+.PHONY: update-po
+update-po:
+	@set -eu; \
+	for lang in $(LINGUAS); do \
+		msgmerge --backup=none -U $$lang.po pkg/i18n/locales/$(DOMAIN).pot; \
+	done
+
+.PHONY: update-pot
+update-pot: $(XGETTEXT)
+	@${XGETTEXT} \
+		-o pkg/i18n/locales/$(DOMAIN).pot \
+		--keyword=i18n.G \
+		--keyword-ctx=i18n.GC \
+		--sort-output \
+		--add-comments-tag="translators" \
+		$$(find . -name "*.go" -not -path "*vendor*" | sort)
+
+${XGETTEXT}:
+	@mkdir -p ./bin && \
+	wget -O ./bin/xgettext-go https://git.coopcloud.tech/toolshed/xgettext-go/raw/branch/main/xgettext-go && \
+	chmod +x ./bin/xgettext-go
+
+.PHONY: update-pot-po-metadata
+update-pot-po-metadata:
+	@sed -i "s/charset=CHARSET/charset=UTF-8/g" pkg/i18n/locales/*.po pkg/i18n/locales/*.pot
+
+.PHONY: build-mo
+build-mo:
+	@set -eu; \
+	for lang in $(POFILES); do \
+		msgfmt $$lang -o $$(echo $$lang | sed 's/.po/.mo/g') --statistics; \
+	done

README.md

@@ -1,31 +1,14 @@
-# abra
+# `abra`
 
-[![Build Status](https://drone.autonomic.zone/api/badges/autonomic-cooperative/abra/status.svg)](https://drone.autonomic.zone/autonomic-cooperative/abra)
+[![Build Status](https://build.coopcloud.tech/api/badges/toolshed/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/toolshed/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/toolshed/abra)](https://goreportcard.com/report/git.coopcloud.tech/toolshed/abra)
+[![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
+[![Translation status](https://translate.coopcloud.tech/widget/co-op-cloud/svg-badge.svg)](https://translate.coopcloud.tech/engage/co-op-cloud/)
 
-Docker stack magic 🎩🐇
+The Co-op Cloud utility belt 🎩🐇
 
-## Install
+<a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-```sh
-curl -fsSL https://install.abra.autonomic.zone | bash
-```
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community 💖
 
-Specific releases are available via the project [release page](https://git.autonomic.zone/autonomic-cooperative/abra/releases).
-
-## Changes
-
-See [CHANGELOG.md](./CHANGELOG.md).
-
-## Hacking
-
-```sh
-git clone ssh://git@git.autonomic.zone:2222/autonomic-cooperative/abra.git
-cd abra
-make dev_install
-```
-
-See [autonomic-cooperative/installer-scripts](https://git.autonomic.zone/autonomic-cooperative/installer-scripts) for the installer script deployment. To make a release, just add an entry to [CHANGELOG.md](./CHANGELOG.md) and the [abra-installer](./script.d/abra-installer) / [swarm-installer](./script.d/swarm-installer) (following [semver](https://semver.org/) please) and then `git tag x.x.x && git push origin main --tags`. If you want the [installer-scripts](https://git.autonomic.zone/autonomic-cooperative/installer-scripts) deployment to pick that up, you'll need to change the version number in the [Makefile](https://git.autonomic.zone/autonomic-cooperative/installer-scripts/src/branch/main/Makefile) and run `make` in that repository and push the changes.
-
-## Examples
-
-- `abra run mariadb mysqldump gitea -p'GdIbMeS09SURRktBnm3jcTufsL5z0MPd' | gzip > ../git.autonomic.zone_mariadb_`date +%F`.sql.gz`
+Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

abra

@@ -1,282 +0,0 @@
-#!/bin/bash
-
-PROGRAM_NAME=$(basename "$0")
-ABRA_CONFIG=abra.yml
-
-if [ -z "$COMPOSE_FILE" ]; then
-	COMPOSE_FILE="compose.yml"
-fi
-
-yml_pattern_exists() {
-	PATTERN=$1
-
-	if ! type yq > /dev/null 2>&1; then
-		echo "$(tput setaf 1)ERROR: yq program is not installed$(tput sgr0)"
-		exit
-	fi
-
-	if [ -f $ABRA_CONFIG ]; then
-			RESULT=$(yq read $ABRA_CONFIG "$PATTERN")
-
-			if [ "$RESULT" != 0 ]; then
-				return 0
-			fi
-	fi
-
-	return 1
-}
-
-if [ "$1" == "-a" ]; then
-	STACK_NAME=$2
-	shift 2
-fi
-
-if [ -f abra.yml ]; then
-	if yml_pattern_exists stack_name; then
-		STACK_NAME=$(yq read abra.yml stack_name)
-	fi
-fi
-
-if [ -z "$STACK_NAME" ]; then
-	echo "$(tput setaf 1)ERROR: \$STACK_NAME must be set (e.g. export STACK_NAME=my_cool_app)$(tput sgr0 )"
-	exit
-fi
-
-if type direnv > /dev/null 2>&1 && ! direnv status | grep -q 'Found RC allowed true'; then
-	echo "$(tput setaf 1)ERROR: direnv is blocked, run direnv allow$(tput sgr0)"
-	exit
-fi
-
-if [ -f abra-commands.sh ]; then
-	# shellcheck disable=SC1091
-	. abra-commands.sh
-fi
-
-sub_help() {
-	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] <subcommand> [options]"
-	echo ""
-	echo "Subcommands:"
-	echo "    cp SRC_PATH SERVICE:DEST_PATH         copy files to a container"
-	echo "    deploy                                let 'em rip"
-	echo "    logs SERVICE [ARGS]                   tail logs from a deployed service"
-	echo "    run SERVICE CMD                       run a command in the specified service's container"
-	echo "    run_args SERVICE ARGS CMD             run, passing extra args to docker exec"
-	echo "    secret_generate SECRET VERSION [CMD]  generate a secret, store it in pass & as a Docker secret"
-	echo "    upgrade                               upgrade to the latest version"
-	echo "    ... (custom commands)"
-	echo ""
-	echo "Make sure \$STACK_NAME is set using direnv or -a"
-	echo ""
-	echo "Runs compose.yml by default, set e.g. COMPOSE_FILE=\"compose.yml:compose2.yml\" to override"
-}
-
-sub_secret_help() {
-	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] secret <subcommand> [options]"
-	echo ""
-	echo "Subcommands:"
-	echo "    generate [PW]                         generate & store secret"
-}
-
-sub_secret_generate(){
-	SECRET=$1
-	VERSION=$2
-	PW=${3:-pwqgen}
-
-	if [ -z "$SECRET" ] || [ -z "$VERSION" ]; then
-		echo "Usage: $PROGRAM_NAME secret_generate SECRET VERSION"
-		exit
-	fi
-
-	$PW | tee \
-		>(docker secret create "${STACK_NAME}_${SECRET}_${VERSION}" -) \
-		>(pass insert "hosts/autonomic-swarm/${STACK_NAME}/${SECRET}" -m)
-}
-
-sub_secret() {
-	SUBCOMMAND=$1
-	shift
-	# shellcheck disable=SC2068
-	parse_subcommand "$SUBCOMMAND" "secret" $@
-}
-
-sub_run_args(){
-	SERVICE=$1
-	DOCKER_ARGS=$2
-
-	shift 2
-
-	if [ -z "$SERVICE" ]; then
-		echo "Usage: $PROGRAM_NAME run SERVICE [CMD]"
-		exit
-	fi
-
-	CONTAINER=$(docker container ls --format "table {{.ID}},{{.Names}}" \
-		| grep "${STACK_NAME}_${SERVICE}" | cut -d',' -f1)
-
-	if [ -z "$CONTAINER" ]; then
-		echo "Container not found! 🚨"
-		exit
-	fi
-
-	# shellcheck disable=SC2086
-	docker exec $DOCKER_ARGS -it "$CONTAINER" "$@"
-
-	return
-}
-
-sub_run(){
-	SERVICE=$1
-
-	shift
-
-	sub_run_args "$SERVICE" "" "$@"
-}
-
-sub_deploy (){
-	echo "About to deploy:"
-	echo "  Compose: $(tput setaf 3)${PWD}/${COMPOSE_FILE}$(tput sgr0)"
-	if [ -n "$DOMAIN" ]; then
-		echo "  Domain: $(tput setaf 2)${DOMAIN}$(tput sgr0)"
-	fi
-	echo "  Stack: $(tput setaf 1)${STACK_NAME}$(tput sgr0)"
-
-	read -rp "Continue? (y/[n])? " choice
-
-	case "$choice" in
-		y|Y ) ;;
-		n|N ) return;;
-		* ) return;;
-	esac
-
-	# shellcheck disable=SC2086
-	if docker stack deploy -c ${COMPOSE_FILE/:/ -c } "$STACK_NAME"; then
-		if [ -n "$DOMAIN" ]; then
-			echo "$(tput setaf 2)Yay! App should be available at https://${DOMAIN}$(tput sgr0)"
-		else
-			echo "$(tput setaf 2)Yay! That worked. No \$DOMAIN defined, check logs.(tput sgr0)"
-		fi
-	else
-		echo "$(tput setaf 1)Oh no! Something went wrong 😕 Check errors above$(tput sgr0)"
-	fi
-}
-
-sub_logs (){
-	SERVICE=$1
-
-	shift
-
-	if [ $# -eq 0 ]; then
-		LOGS_ARGS="\
-		--follow \
-		--no-trunc \
-		--details \
-		--timestamps"
-	else
-		# shellcheck disable=SC2124
-		LOGS_ARGS=$@
-	fi
-
-	# shellcheck disable=SC2086
-	docker service logs "${STACK_NAME}_${SERVICE}" $LOGS_ARGS
-}
-
-sub_cp() {
-	SOURCE=$1
-	DEST=$2
-
-	SERVICE=$(echo "$SOURCE" | grep -o '^[^:]\+:' || echo "$DEST" | grep -o '^[^:]\+:')
-	SERVICE=$(echo "$SERVICE" | tr -d ':')
-
-	if [ -z "$SERVICE" ]; then
-		echo "$(tput setaf 1)ERROR: Can't find SERVICE in either SRC or DEST$(tput sgr0)"
-		echo ""
-		echo "Usage: $PROGRAM_NAME cp SERVICE:SRC_PATH DEST_PATH"
-		echo "       $PROGRAM_NAME cp SRC_PATH SERVICE:DEST_PATH"
-		exit
-	fi
-
-	CONTAINER=$(docker container ls --format "table {{.ID}},{{.Names}}" \
-		| grep "${STACK_NAME}_${SERVICE}" | cut -d',' -f1)
-
-	if [ -z "$CONTAINER" ]; then
-		echo "$(tput setaf 1)ERROR: Can't find a ${STACK_NAME}_${SERVICE}$(tput sgr0)"
-		exit
-	fi
-
-	CP_ARGS=$(echo "$SOURCE $DEST" | sed "s/$SERVICE/$CONTAINER/")
-
-	# shellcheck disable=SC2086
-	docker cp $CP_ARGS
-}
-
-sub_context_help() {
-	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] context <subcommand> [options]"
-	echo ""
-	echo "Subcommands:"
-	echo "    init HOST [USER] [PORT]               set up remote Docker context"
-	echo "    use                                   activate remote Docker context"
-}
-
-sub_context_init() {
-	HOST="$1"
-	USERNAME="$2"
-	PORT="$3"
-
-	if [ -n "$PORT" ]; then
-		PORT=":$PORT"
-	fi
-
-	if [ -n "$USERNAME" ]; then
-		USERNAME="$USERNAME@"
-	fi
-
-	docker context create "$HOST" \
-		--docker "host=ssh://$USERNAME$HOST$PORT"
-}
-
-sub_context_use() {
-	docker context use "$1"
-}
-
-sub_upgrade() {
-	curl -fsSL https://install.abra.autonomic.zone | bash
-}
-
-sub_context() {
-	SUBCOMMAND2=$1
-	shift
-	# shellcheck disable=SC2068
-	parse_subcommand "$SUBCOMMAND2" "context" $@
-}
-
-parse_subcommand() {
-	SUBCOMMAND="$1"
-	PREFIX=$2
-
-	if [ -n "$PREFIX" ]; then
-		PPREFIX="_$2"
-		SPREFIX="$2 "
-		SSPREFIX=" $2"
-	fi
-
-	case $SUBCOMMAND in
-		"" | "-h" | "--help")
-			"sub${PPREFIX}_help"
-			;;
-		*)
-			shift 2
-			"sub${PPREFIX}_${SUBCOMMAND}" "$@"
-			if [ $? = 127 ]; then
-				echo "Error: '$SPREFIX$SUBCOMMAND' is not a known subcommand." >&2
-				echo "       Run '$PROGRAM_NAME$SSPREFIX --help' for a list of known subcommands." >&2
-				exit 1
-			fi
-			;;
-	esac
-}
-
-SUBCOMMAND=$1
-shift
-
-# shellcheck disable=SC2086,SC2068
-parse_subcommand $SUBCOMMAND "" $@

cli/app/app.go

@@ -0,0 +1,20 @@
+package app
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appAliases = i18n.GC("a", "abra app")
+
+var AppCommand = &cobra.Command{
+	// translators: `app` command group
+	Use:     i18n.G("app [cmd] [args] [flags]"),
+	Aliases: strings.Split(appAliases, ","),
+	// translators: Short description for `app` command group
+	Short: i18n.G("Manage apps"),
+}

cli/app/backup.go

@@ -0,0 +1,339 @@
+package app
+
+import (
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app backup list` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appBackupListAliases = i18n.G("ls")
+
+var AppBackupListCommand = &cobra.Command{
+	// translators: `app backup list` command
+	Use:     i18n.G("list <domain> [flags]"),
+	Aliases: strings.Split(appBackupListAliases, ","),
+	// translators: Short description for `app backup list` command
+	Short: i18n.G("List the contents of a snapshot"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if snapshot != "" {
+			log.Debug(i18n.G("including SNAPSHOT=%s in backupbot exec invocation", snapshot))
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if showAllPaths {
+			log.Debug(i18n.G("including SHOW_ALL=%v in backupbot exec invocation", showAllPaths))
+			execEnv = append(execEnv, fmt.Sprintf("SHOW_ALL=%v", showAllPaths))
+		}
+
+		if timestamps {
+			log.Debug(i18n.G("including TIMESTAMPS=%v in backupbot exec invocation", timestamps))
+			execEnv = append(execEnv, fmt.Sprintf("TIMESTAMPS=%v", timestamps))
+		}
+
+		if _, err = internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+// translators: `abra app backup download` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appBackupDownloadAliases = i18n.G("d")
+
+var AppBackupDownloadCommand = &cobra.Command{
+	// translators: `app backup download` command
+	Use:     i18n.G("download <domain> [flags]"),
+	Aliases: strings.Split(appBackupDownloadAliases, ","),
+	// translators: Short description for `app backup download` command
+	Short: i18n.G("Download a snapshot"),
+	Long: i18n.G(`Downloads a backup.tar.gz to the current working directory.
+
+"--volumes/-v" includes data contained in volumes alongide paths specified in
+"backupbot.backup.path" labels.`),
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if snapshot != "" {
+			log.Debug(i18n.G("including SNAPSHOT=%s in backupbot exec invocation", snapshot))
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if includePath != "" {
+			log.Debug(i18n.G("including INCLUDE_PATH=%s in backupbot exec invocation", includePath))
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
+
+		if includeSecrets {
+			log.Debug(i18n.G("including SECRETS=%v in backupbot exec invocation", includeSecrets))
+			execEnv = append(execEnv, fmt.Sprintf("SECRETS=%v", includeSecrets))
+		}
+
+		if includeVolumes {
+			log.Debug(i18n.G("including VOLUMES=%v in backupbot exec invocation", includeVolumes))
+			execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%v", includeVolumes))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+
+		remoteBackupDir := "/tmp/backup.tar.gz"
+		currentWorkingDir := "."
+		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+// translators: `abra app backup create` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appBackupCreateAliases = i18n.G("c")
+
+var AppBackupCreateCommand = &cobra.Command{
+	// translators: `app backup create` command
+	Use:     i18n.G("create <domain> [flags]"),
+	Aliases: strings.Split(appBackupCreateAliases, ","),
+	// translators: Short description for `app backup create` command
+	Short: i18n.G("Create a new snapshot"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if retries != "" {
+			log.Debug(i18n.G("including RETRIES=%s in backupbot exec invocation", retries))
+			execEnv = append(execEnv, fmt.Sprintf("RETRIES=%s", retries))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+// translators: `abra app backup snapshots` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appBackupSnapshotsAliases = i18n.G("s")
+
+var AppBackupSnapshotsCommand = &cobra.Command{
+	// translators: `app backup snapshots` command
+	Use:     i18n.G("snapshots <domain> [flags]"),
+	Aliases: strings.Split(appBackupSnapshotsAliases, ","),
+	// translators: Short description for `app backup snapshots` command
+	Short: i18n.G("List all snapshots"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if _, err = internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+// translators: `abra app backup` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appBackupAliases = i18n.G("b")
+
+var AppBackupCommand = &cobra.Command{
+	// translators: `app backup` command group
+	Use:     i18n.G("backup [cmd] [args] [flags]"),
+	Aliases: strings.Split(appBackupAliases, ","),
+	// translators: Short description for `app backup` command group
+	Short: i18n.G("Manage app backups"),
+}
+
+var (
+	snapshot       string
+	retries        string
+	includePath    string
+	showAllPaths   bool
+	timestamps     bool
+	includeSecrets bool
+	includeVolumes bool
+)
+
+func init() {
+	AppBackupListCommand.Flags().StringVarP(
+		&snapshot,
+		i18n.G("snapshot"),
+		i18n.G("s"),
+		"",
+		i18n.G("list specific snapshot"),
+	)
+
+	AppBackupListCommand.Flags().BoolVarP(
+		&showAllPaths,
+		i18n.G("all"),
+		i18n.GC("a", "app backup list"),
+		false,
+		i18n.G("show all paths"),
+	)
+
+	AppBackupListCommand.Flags().BoolVarP(
+		&timestamps,
+		i18n.G("timestamps"),
+		i18n.G("t"),
+		false,
+		i18n.G("include timestamps"),
+	)
+
+	AppBackupDownloadCommand.Flags().StringVarP(
+		&snapshot,
+		i18n.G("snapshot"),
+		i18n.G("s"),
+		"",
+		i18n.G("list specific snapshot"),
+	)
+
+	AppBackupDownloadCommand.Flags().StringVarP(
+		&includePath,
+		i18n.G("path"),
+		i18n.G("p"),
+		"",
+		i18n.G("volumes path"),
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&includeSecrets,
+		i18n.G("secrets"),
+		i18n.G("S"),
+		false,
+		i18n.G("include secrets"),
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&includeVolumes,
+		i18n.G("volumes"),
+		i18n.G("v"),
+		false,
+		i18n.G("include volumes"),
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	AppBackupCreateCommand.Flags().StringVarP(
+		&retries,
+		i18n.G("retries"),
+		i18n.G("r"),
+		"1",
+		i18n.G("number of retry attempts"),
+	)
+
+	AppBackupCreateCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/check.go

@@ -0,0 +1,99 @@
+package app
+
+import (
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app check` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appCheckAliases = i18n.G("chk")
+
+var AppCheckCommand = &cobra.Command{
+	// translators: `app check` command
+	Use:     i18n.G("check <domain> [flags]"),
+	Aliases: strings.Split(appCheckAliases, ","),
+	// translators: Short description for `app check` command
+	Short: i18n.G("Ensure an app is well configured"),
+	Long: i18n.G(`Compare env vars in both the app ".env" and recipe ".env.sample" file.
+
+The goal is to ensure that recipe ".env.sample" env vars are defined in your
+app ".env" file. Only env var definitions in the ".env.sample" which are
+uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include
+these env vars, then "check" will complain.
+
+Recipe maintainers may or may not provide defaults for env vars within their
+recipes regardless of commenting or not (e.g. through the use of
+${FOO:<default>} syntax). "check" does not confirm or deny this for you.`),
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table.
+			Headers(
+				fmt.Sprintf("%s .env.sample", app.Recipe.Name),
+				fmt.Sprintf("%s.env", app.Name),
+			).
+			StyleFunc(func(row, col int) lipgloss.Style {
+				switch {
+				case col == 1:
+					return lipgloss.NewStyle().Padding(0, 1, 0, 1).Align(lipgloss.Center)
+				default:
+					return lipgloss.NewStyle().Padding(0, 1, 0, 1)
+				}
+			})
+
+		envVars, err := appPkg.CheckEnv(app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if envVar.Present {
+				val := []string{envVar.Name, "✅"}
+				table.Row(val...)
+			} else {
+				val := []string{envVar.Name, "❌"}
+				table.Row(val...)
+			}
+		}
+
+		if err := formatter.PrintTable(table); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+func init() {
+	AppCheckCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/cmd.go

@@ -0,0 +1,289 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"slices"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app cmd` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appCmdAliases = i18n.G("cmd")
+
+var AppCmdCommand = &cobra.Command{
+	// translators: `app command` command
+	Use:     i18n.G("command <domain> [service | --local] <cmd> [[args] [flags] | [flags] -- [args]]"),
+	Aliases: strings.Split(appCmdAliases, ","),
+	// translators: Short description for `app cmd` command
+	Short: i18n.G("Run app commands"),
+	Long: i18n.G(`Run an app specific command.
+
+These commands are bash functions, defined in the abra.sh of the recipe itself.
+They can be run within the context of a service (e.g. app) or locally on your
+work station by passing "--local/-l".
+
+N.B. If using the "--" style to pass arguments, flags (e.g. "--local/-l") must
+be passed *before* the "--". It is possible to pass arguments without the "--"
+as long as no dashes are present (i.e. "foo" works without "--", "-foo"
+does not).`),
+	Example: i18n.G(`  # pass <cmd> args/flags without "--"
+  abra app cmd 1312.net app my_cmd_arg foo --user bar
+
+  # pass <cmd> args/flags with "--"
+  abra app cmd 1312.net app my_cmd_args --user bar -- foo -vvv
+
+  # drop the [service] arg if using "--local/-l"
+  abra app cmd 1312.net my_cmd --local`),
+	Args: func(cmd *cobra.Command, args []string) error {
+		if local {
+			if !(len(args) >= 2) {
+				return errors.New(i18n.G("requires at least 2 arguments with --local/-l"))
+			}
+
+			if slices.Contains(os.Args, "--") {
+				if cmd.ArgsLenAtDash() > 2 {
+					return errors.New(i18n.G("accepts at most 2 args with --local/-l"))
+				}
+			}
+
+			// NOTE(d1): it is unclear how to correctly validate this case
+			//
+			// abra app cmd 1312.net app test_cmd_args foo --local
+			// FATAL <recipe> doesn't have a app function
+			//
+			// "app" should not be there, but there is no reliable way to detect arg
+			// count when the user can pass an arbitrary amount of recipe command
+			// arguments
+			return nil
+		}
+
+		if !(len(args) >= 3) {
+			return errors.New(i18n.G("requires at least 3 arguments"))
+		}
+
+		return nil
+	},
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			if !local {
+				return autocomplete.ServiceNameComplete(args[0])
+			}
+			return autocomplete.CommandNameComplete(args[0])
+		case 2:
+			if !local {
+				return autocomplete.CommandNameComplete(args[0])
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		if local && remoteUser != "" {
+			log.Fatal(i18n.G("cannot use --local & --user together"))
+		}
+
+		hasCmdArgs, parsedCmdArgs := parseCmdArgs(args, local)
+
+		if _, err := os.Stat(app.Recipe.AbraShPath); err != nil {
+			if os.IsNotExist(err) {
+				log.Fatal(i18n.G("%s does not exist for %s?", app.Recipe.AbraShPath, app.Name))
+			}
+			log.Fatal(err)
+		}
+
+		if local {
+			cmdName := args[1]
+			if err := internal.EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+				log.Fatal(err)
+			}
+
+			log.Debug(i18n.G("--local detected, running %s on local work station", cmdName))
+
+			var exportEnv string
+			for k, v := range app.Env {
+				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
+			}
+
+			var sourceAndExec string
+			if hasCmdArgs {
+				log.Debug(i18n.G("parsed following command arguments: %s", parsedCmdArgs))
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, app.Recipe.AbraShPath, cmdName, parsedCmdArgs)
+			} else {
+				log.Debug(i18n.G("did not detect any command arguments"))
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, app.Recipe.AbraShPath, cmdName)
+			}
+
+			shell := "/bin/bash"
+			if _, err := os.Stat(shell); errors.Is(err, os.ErrNotExist) {
+				log.Debug(i18n.G("%s does not exist locally, use /bin/sh as fallback", shell))
+				shell = "/bin/sh"
+			}
+			cmd := exec.Command(shell, "-c", sourceAndExec)
+
+			if err := internal.RunCmd(cmd); err != nil {
+				log.Fatal(err)
+			}
+
+			return
+		}
+
+		cmdName := args[2]
+		if err := internal.EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+			log.Fatal(err)
+		}
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		matchingServiceName := false
+		targetServiceName := args[1]
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
+			}
+		}
+
+		if !matchingServiceName {
+			log.Fatal(i18n.G("no service %s for %s?", targetServiceName, app.Name))
+		}
+
+		log.Debug(i18n.G("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName))
+
+		if hasCmdArgs {
+			log.Debug(i18n.G("parsed following command arguments: %s", parsedCmdArgs))
+		} else {
+			log.Debug(i18n.G("did not detect any command arguments"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := internal.RunCmdRemote(
+			cl,
+			app,
+			disableTTY,
+			app.Recipe.AbraShPath,
+			targetServiceName, cmdName, parsedCmdArgs, remoteUser); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+// translators: `abra app command list` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appCmdListAliases = i18n.G("ls")
+
+var AppCmdListCommand = &cobra.Command{
+	// translators: `app cmd list` command
+	Use:     i18n.G("list <domain> [flags]"),
+	Aliases: strings.Split(appCmdListAliases, ","),
+	// translators: Short description for `app cmd list` command
+	Short: i18n.G("List all available commands"),
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cmdNames, err := appPkg.ReadAbraShCmdNames(app.Recipe.AbraShPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		sort.Strings(cmdNames)
+
+		for _, cmdName := range cmdNames {
+			fmt.Println(cmdName)
+		}
+	},
+}
+
+func parseCmdArgs(args []string, isLocal bool) (bool, string) {
+	var (
+		parsedCmdArgs string
+		hasCmdArgs    bool
+	)
+
+	if isLocal {
+		if len(args) > 2 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[2:], " "))
+		}
+	} else {
+		if len(args) > 3 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[3:], " "))
+		}
+	}
+
+	return hasCmdArgs, parsedCmdArgs
+}
+
+var (
+	local      bool
+	remoteUser string
+	disableTTY bool
+)
+
+func init() {
+	AppCmdCommand.Flags().BoolVarP(
+		&local,
+		i18n.G("local"),
+		i18n.G("l"),
+		false,
+		i18n.G("run command locally"),
+	)
+
+	AppCmdCommand.Flags().StringVarP(
+		&remoteUser,
+		i18n.G("user"),
+		i18n.G("u"),
+		"",
+		i18n.G("request remote user"),
+	)
+
+	AppCmdCommand.Flags().BoolVarP(
+		&disableTTY,
+		i18n.G("tty"),
+		i18n.G("T"),
+		false,
+		i18n.G("disable remote TTY"),
+	)
+
+	AppCmdCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/cmd_test.go

@@ -0,0 +1,31 @@
+package app
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseCmdArgs(t *testing.T) {
+	tests := []struct {
+		input          []string
+		shouldParse    bool
+		expectedOutput string
+	}{
+		// `--` is not parsed when passed in from the command-line e.g. -- foo bar baz
+		// so we need to eumlate that as missing when testing if bash args are passed in
+		// see https://git.coopcloud.tech/toolshed/organising/issues/336 for more
+		{[]string{"foo.com", "app", "test"}, false, ""},
+		{[]string{"foo.com", "app", "test", "foo"}, true, "foo "},
+		{[]string{"foo.com", "app", "test", "foo", "bar", "baz"}, true, "foo bar baz "},
+	}
+
+	for _, test := range tests {
+		ok, parsed := parseCmdArgs(test.input, false)
+		if ok != test.shouldParse {
+			t.Fatalf("[%s] should not parse", strings.Join(test.input, " "))
+		}
+		if parsed != test.expectedOutput {
+			t.Fatalf("%s does not match %s", parsed, test.expectedOutput)
+		}
+	}
+}

cli/app/config.go

@@ -0,0 +1,65 @@
+package app
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app config` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appConfigAliases = i18n.G("cfg")
+
+var AppConfigCommand = &cobra.Command{
+	// translators: `app config` command
+	Use:     i18n.G("config <domain> [flags]"),
+	Aliases: strings.Split(appConfigAliases, ","),
+	// translators: Short description for `app config` command
+	Short:   i18n.G("Edit app config"),
+	Example: i18n.G("  abra config 1312.net"),
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		files, err := appPkg.LoadAppFiles("")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		appName := args[0]
+		appFile, exists := files[appName]
+		if !exists {
+			log.Fatal(i18n.G("cannot find app with name %s", appName))
+		}
+
+		ed, ok := os.LookupEnv("EDITOR")
+		if !ok {
+			edPrompt := &survey.Select{
+				Message: i18n.G("which editor do you wish to use?"),
+				Options: []string{"vi", "vim", "nvim", "nano", "pico", "emacs"},
+			}
+			if err := survey.AskOne(edPrompt, &ed); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		c := exec.Command(ed, appFile.Path)
+		c.Stdin = os.Stdin
+		c.Stdout = os.Stdout
+		c.Stderr = os.Stderr
+		if err := c.Run(); err != nil {
+			log.Fatal(err)
+		}
+	},
+}

cli/app/cp.go

@@ -0,0 +1,389 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	containertypes "github.com/docker/docker/api/types/container"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app cp` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appCpAliases = i18n.G("c")
+
+var AppCpCommand = &cobra.Command{
+	// translators: `app cp` command
+	Use:     i18n.G("cp <domain> <src> <dst> [flags]"),
+	Aliases: strings.Split(appCpAliases, ","),
+	// translators: Short description for `app cp` command
+	Short: i18n.G("Copy files to/from a deployed app service"),
+	Example: i18n.G(`  # copy myfile.txt to the root of the app service
+  abra app cp 1312.net myfile.txt app:/
+
+  # copy that file back to your current working directory locally
+  abra app cp 1312.net app:/myfile.txt ./`),
+	Args: cobra.ExactArgs(3),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		src := args[1]
+		dst := args[2]
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Debug(i18n.G("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server))
+
+		if toContainer {
+			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
+		} else {
+			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+var errServiceMissing = errors.New(i18n.G("one of <src>/<dest> arguments must take $SERVICE:$PATH form"))
+
+// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH
+func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) {
+	parsedSrc := strings.SplitN(src, ":", 2)
+	parsedDst := strings.SplitN(dst, ":", 2)
+	if len(parsedSrc)+len(parsedDst) != 3 {
+		return "", "", "", false, errServiceMissing
+	}
+	if len(parsedSrc) == 2 {
+		return parsedSrc[1], dst, parsedSrc[0], false, nil
+	}
+	if len(parsedDst) == 2 {
+		return src, parsedDst[1], parsedDst[0], true, nil
+	}
+	return "", "", "", false, errServiceMissing
+}
+
+// CopyToContainer copies a file or directory from the local file system to the container.
+// See the possible copy modes and their documentation.
+func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := os.Stat(srcPath)
+	if err != nil {
+		return errors.New(i18n.G("local %s ", err))
+	}
+
+	dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath)
+	dstExists := true
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			dstExists = false
+		} else {
+			return errors.New(i18n.G("remote path: %s", err))
+		}
+	}
+
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists)
+	if err != nil {
+		return err
+	}
+	movePath := ""
+	switch mode {
+	case CopyModeDirToDir:
+		// Add the src directory to the destination path
+		_, srcDir := path.Split(srcPath)
+		dstPath = path.Join(dstPath, srcDir)
+
+		// Make sure the dst directory exits.
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			return err
+		}
+		if _, err := container.RunExec(dcli, cl, containerID, &containertypes.ExecOptions{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mkdir", "-p", dstPath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return errors.New(i18n.G("create remote directory: %s", err))
+		}
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		movePath = dstPath
+		dstPath = "/tmp"
+	}
+
+	toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(srcPath, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	log.Debug(i18n.G("copy %s from local to %s on container", srcPath, dstPath))
+	copyOpts := containertypes.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
+		return err
+	}
+
+	if movePath != "" {
+		_, srcFile := path.Split(srcPath)
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			return err
+		}
+		if _, err := container.RunExec(dcli, cl, containerID, &containertypes.ExecOptions{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mv", path.Join("/tmp", srcFile), movePath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return errors.New(i18n.G("create remote directory: %s", err))
+		}
+	}
+
+	return nil
+}
+
+// CopyFromContainer copies a file or directory from the given container to the local file system.
+// See the possible copy modes and their documentation.
+func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return errors.New(i18n.G("remote: %s does not exist", srcPath))
+		} else {
+			return errors.New(i18n.G("remote path: %s", err))
+		}
+	}
+
+	dstStat, err := os.Stat(dstPath)
+	dstExists := true
+	var dstMode os.FileMode
+	if err != nil {
+		if os.IsNotExist(err) {
+			dstExists = false
+		} else {
+			return errors.New(i18n.G("remote path: %s", err))
+		}
+	} else {
+		dstMode = dstStat.Mode()
+	}
+
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists)
+	if err != nil {
+		return err
+	}
+
+	moveDstDir := ""
+	moveDstFile := ""
+	switch mode {
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		moveDstFile = dstPath
+		dstPath = "/tmp"
+	case CopyModeFilesToDir:
+		// Copy the directory to the temp directory and move it to its
+		// dstPath afterwards.
+		moveDstDir = path.Join(dstPath, "/")
+		dstPath = "/tmp"
+
+		// Make sure the temp directory always gets removed
+		defer os.Remove(path.Join("/tmp"))
+	}
+
+	content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath)
+	if err != nil {
+		return errors.New(i18n.G("copy: %s", err))
+	}
+	defer content.Close()
+	if err := archive.Untar(content, dstPath, &archive.TarOptions{
+		NoOverwriteDirNonDir: true,
+		Compression:          archive.Gzip,
+		NoLchown:             true,
+	}); err != nil {
+		return errors.New(i18n.G("untar: %s", err))
+	}
+
+	if moveDstFile != "" {
+		_, srcFile := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil {
+			return err
+		}
+	}
+	if moveDstDir != "" {
+		_, srcDir := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var (
+	ErrCopyDirToFile  = errors.New(i18n.G("can't copy dir to file"))
+	ErrDstDirNotExist = errors.New(i18n.G("destination directory does not exist"))
+)
+
+type CopyMode int
+
+const (
+	// Copy a src file to a dest file. The src and dest file names are the same.
+	//  <dir_src>/<file> + <dir_dst>/<file> -> <dir_dst>/<file>
+	CopyModeFileToFile = CopyMode(iota)
+	// Copy a src file to a dest file. The src and dest file names are  not the same.
+	//  <dir_src>/<file_src> + <dir_dst>/<file_dst> -> <dir_dst>/<file_dst>
+	CopyModeFileToFileRename
+	// Copy a src file to dest directory. The dest file gets created in the dest
+	// folder with the src filename.
+	//  <dir_src>/<file> + <dir_dst> -> <dir_dst>/<file>
+	CopyModeFileToDir
+	// Copy a src directory to dest directory.
+	//  <dir_src> + <dir_dst> -> <dir_dst>/<dir_src>
+	CopyModeDirToDir
+	// Copy all files in the src directory to the dest directory. This works recursively.
+	//  <dir_src>/ + <dir_dst> -> <dir_dst>/<files_from_dir_src>
+	CopyModeFilesToDir
+)
+
+// copyMode takes a src and dest path and file mode to determine the copy mode.
+// See the possible copy modes and their documentation.
+func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) {
+	_, srcFile := path.Split(srcPath)
+	_, dstFile := path.Split(dstPath)
+	if srcMode.IsDir() {
+		if !dstExists {
+			return -1, ErrDstDirNotExist
+		}
+		if dstMode.IsDir() {
+			if strings.HasSuffix(srcPath, "/") {
+				return CopyModeFilesToDir, nil
+			}
+			return CopyModeDirToDir, nil
+		}
+		return -1, ErrCopyDirToFile
+	}
+
+	if dstMode.IsDir() {
+		return CopyModeFileToDir, nil
+	}
+
+	if srcFile != dstFile {
+		return CopyModeFileToFileRename, nil
+	}
+
+	return CopyModeFileToFile, nil
+}
+
+// moveDir moves all files from a source path to the destination path recursively.
+func moveDir(sourcePath, destPath string) error {
+	return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath))
+		if info.IsDir() {
+			err := os.Mkdir(newPath, info.Mode())
+			if err != nil {
+				if os.IsExist(err) {
+					return nil
+				}
+				return err
+			}
+		}
+		if info.Mode().IsRegular() {
+			return moveFile(p, newPath)
+		}
+		return nil
+	})
+}
+
+// moveFile moves a file from a source path to a destination path.
+func moveFile(sourcePath, destPath string) error {
+	inputFile, err := os.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	outputFile, err := os.Create(destPath)
+	if err != nil {
+		inputFile.Close()
+		return err
+	}
+	defer outputFile.Close()
+	_, err = io.Copy(outputFile, inputFile)
+	inputFile.Close()
+	if err != nil {
+		return err
+	}
+
+	// Remove file after succesfull copy.
+	err = os.Remove(sourcePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func init() {
+	AppCpCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/cp_test.go

@@ -0,0 +1,113 @@
+package app
+
+import (
+	"os"
+	"testing"
+)
+
+func TestParse(t *testing.T) {
+	tests := []struct {
+		src         string
+		dst         string
+		srcPath     string
+		dstPath     string
+		service     string
+		toContainer bool
+		err         error
+	}{
+		{src: "foo", dst: "bar", err: errServiceMissing},
+		{src: "app:foo", dst: "app:bar", err: errServiceMissing},
+		{src: "app:foo", dst: "bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: false},
+		{src: "foo", dst: "app:bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: true},
+	}
+
+	for i, tc := range tests {
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(tc.src, tc.dst)
+		if srcPath != tc.srcPath {
+			t.Errorf("[%d] srcPath: want (%s), got(%s)", i, tc.srcPath, srcPath)
+		}
+		if dstPath != tc.dstPath {
+			t.Errorf("[%d] dstPath: want (%s), got(%s)", i, tc.dstPath, dstPath)
+		}
+		if service != tc.service {
+			t.Errorf("[%d] service: want (%s), got(%s)", i, tc.service, service)
+		}
+		if toContainer != tc.toContainer {
+			t.Errorf("[%d] toConainer: want (%t), got(%t)", i, tc.toContainer, toContainer)
+		}
+		if err == nil && tc.err != nil && err.Error() != tc.err.Error() {
+			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
+		}
+	}
+}
+
+func TestCopyMode(t *testing.T) {
+	tests := []struct {
+		srcPath   string
+		dstPath   string
+		srcMode   os.FileMode
+		dstMode   os.FileMode
+		dstExists bool
+		mode      CopyMode
+		err       error
+	}{
+		{
+			srcPath:   "foo.txt",
+			dstPath:   "foo.txt",
+			srcMode:   os.ModePerm,
+			dstMode:   os.ModePerm,
+			dstExists: true,
+			mode:      CopyModeFileToFile,
+		},
+		{
+			srcPath:   "foo.txt",
+			dstPath:   "bar.txt",
+			srcMode:   os.ModePerm,
+			dstExists: true,
+			mode:      CopyModeFileToFileRename,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModeDir,
+			dstExists: true,
+			mode:      CopyModeDirToDir,
+		},
+		{
+			srcPath:   "foo/",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModeDir,
+			dstExists: true,
+			mode:      CopyModeFilesToDir,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstExists: false,
+			mode:      -1,
+			err:       ErrDstDirNotExist,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModePerm,
+			dstExists: true,
+			mode:      -1,
+			err:       ErrCopyDirToFile,
+		},
+	}
+
+	for i, tc := range tests {
+		mode, err := copyMode(tc.srcPath, tc.dstPath, tc.srcMode, tc.dstMode, tc.dstExists)
+		if mode != tc.mode {
+			t.Errorf("[%d] mode: want (%d), got(%d)", i, tc.mode, mode)
+		}
+		if err != tc.err {
+			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
+		}
+	}
+}

cli/app/deploy.go

@@ -0,0 +1,440 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/secret"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/deploy"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app deploy` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appDeployAliases = i18n.G("d")
+
+var AppDeployCommand = &cobra.Command{
+	// translators: `app deploy` command
+	Use:     i18n.G("deploy <domain> [version] [flags]"),
+	Aliases: strings.Split(appDeployAliases, ","),
+	// translators: Short description for `app deploy` command
+	Short: i18n.G("Deploy an app"),
+	Long: i18n.G(`Deploy an app.
+
+This command supports chaos operations. Use "--chaos/-C" to deploy your recipe
+checkout as-is. Recipe commit hashes are also supported as values for
+"[version]". Please note, "upgrade"/"rollback" do not support chaos operations.`),
+	Example: i18n.G(`  # standard deployment
+  abra app deploy 1312.net
+
+  # chaos deployment
+  abra app deploy 1312.net --chaos
+  
+  # deploy specific version
+  abra app deploy 1312.net 2.0.0+1.2.3
+
+  # deploy a specific git hash
+  abra app deploy 1312.net 886db76d`),
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				errMsg := i18n.G("autocomplete failed: %s", err)
+				return []string{errMsg}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.RecipeVersionComplete(app.Recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			deployWarnMessages []string
+			toDeployVersion    string
+		)
+
+		app := internal.ValidateApp(args)
+
+		if err := validateArgsAndFlags(args); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Debug(i18n.G("checking whether %s is already deployed", app.StackName()))
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if deployMeta.IsDeployed && !(internal.Force || internal.Chaos) {
+			log.Fatal(i18n.G("%s is already deployed", app.Name))
+		}
+
+		toDeployVersion, err = getDeployVersion(args, deployMeta, app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			isChaosCommit, err := app.Recipe.EnsureVersion(toDeployVersion)
+			if err != nil {
+				log.Fatal(i18n.G("ensure recipe: %s", err))
+			}
+			if isChaosCommit {
+				log.Warnf(i18n.G("version '%s' appears to be a chaos commit, but --chaos/-C was not provided", toDeployVersion))
+				internal.Chaos = true
+			}
+		}
+
+		if err := lint.LintForErrors(app.Recipe); err != nil {
+			if internal.Chaos {
+				log.Warn(err)
+			} else {
+				log.Fatal(err)
+			}
+		}
+
+		if err := validateSecrets(cl, app); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := deploy.MergeAbraShEnv(app.Recipe, app.Env); err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		stackName := app.StackName()
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
+		}
+		compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		appPkg.SetRecipeLabel(compose, stackName, app.Recipe.Name)
+		appPkg.SetChaosLabel(compose, stackName, internal.Chaos)
+		if internal.Chaos {
+			appPkg.SetChaosVersionLabel(compose, stackName, toDeployVersion)
+		}
+
+		versionLabel := toDeployVersion
+		if internal.Chaos {
+			for _, service := range compose.Services {
+				if service.Name == "app" {
+					labelKey := fmt.Sprintf("coop-cloud.%s.version", stackName)
+					// NOTE(d1): keep non-chaos version labbeling when doing chaos ops
+					versionLabel = service.Deploy.Labels[labelKey]
+				}
+			}
+		}
+		appPkg.SetVersionLabel(compose, stackName, versionLabel)
+
+		newRecipeWithDeployVersion := fmt.Sprintf("%s:%s", app.Recipe.Name, toDeployVersion)
+		appPkg.ExposeAllEnv(stackName, compose, app.Env, newRecipeWithDeployVersion)
+
+		envVars, err := appPkg.CheckEnv(app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				deployWarnMessages = append(deployWarnMessages,
+					i18n.G("%s missing from %s.env", envVar.Name, app.Domain),
+				)
+			}
+		}
+
+		if !internal.NoDomainChecks {
+			if domainName, ok := app.Env["DOMAIN"]; ok {
+				if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+					log.Fatal(err)
+				}
+			} else {
+				log.Debug(i18n.G("skipping domain checks, no DOMAIN=... configured"))
+			}
+		} else {
+			log.Debug(i18n.G("skipping domain checks"))
+		}
+
+		deployedVersion := config.MISSING_DEFAULT
+		if deployMeta.IsDeployed {
+			deployedVersion = deployMeta.Version
+			if deployMeta.IsChaos {
+				deployedVersion = deployMeta.ChaosVersion
+			}
+		}
+
+		// Gather secrets
+		secretInfo, err := deploy.GatherSecretsForDeploy(cl, app, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Gather configs
+		configInfo, err := deploy.GatherConfigsForDeploy(cl, app, compose, app.Env, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Gather images
+		imageInfo, err := deploy.GatherImagesForDeploy(cl, app, compose, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Show deploy overview
+		if err := internal.DeployOverview(
+			app,
+			deployedVersion,
+			toDeployVersion,
+			"",
+			deployWarnMessages,
+			secretInfo,
+			configInfo,
+			imageInfo,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		f, err := app.Filters(true, false, serviceNames...)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(
+			cl,
+			deployOpts,
+			compose,
+			app.Name,
+			app.Server,
+			internal.DontWaitConverge,
+			internal.NoInput,
+			f,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_DEPLOY_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			log.Debug(i18n.G("run the following post-deploy commands: %s", postDeployCmds))
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				log.Fatal(i18n.G("attempting to run post deploy commands, saw: %s", err))
+			}
+		}
+
+		if err := app.WriteRecipeVersion(toDeployVersion, false); err != nil {
+			log.Fatal(i18n.G("writing recipe version failed: %s", err))
+		}
+	},
+}
+
+func getLatestVersionOrCommit(app appPkg.App) (string, error) {
+	versions, err := app.Recipe.Tags()
+	if err != nil {
+		return "", err
+	}
+
+	if len(versions) > 0 && !internal.Chaos {
+		return versions[len(versions)-1], nil
+	}
+
+	head, err := app.Recipe.Head()
+	if err != nil {
+		return "", err
+	}
+
+	return formatter.SmallSHA(head.String()), nil
+}
+
+// validateArgsAndFlags ensures compatible args/flags.
+func validateArgsAndFlags(args []string) error {
+	if len(args) == 2 && args[1] != "" && internal.Chaos {
+		return errors.New(i18n.G("cannot use [version] and --chaos together"))
+	}
+
+	if len(args) == 2 && args[1] != "" && internal.DeployLatest {
+		return errors.New(i18n.G("cannot use [version] and --latest together"))
+	}
+
+	if internal.DeployLatest && internal.Chaos {
+		return errors.New(i18n.G("cannot use --chaos and --latest together"))
+	}
+
+	return nil
+}
+
+func validateSecrets(cl *dockerClient.Client, app appPkg.App) error {
+	composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+	if err != nil {
+		return err
+	}
+
+	secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+	if err != nil {
+		return err
+	}
+
+	secStats, err := secret.PollSecretsStatus(cl, app)
+	if err != nil {
+		return err
+	}
+
+	for _, secStat := range secStats {
+		if !secStat.CreatedOnRemote {
+			secretConfig := secretsConfig[secStat.LocalName]
+			if secretConfig.SkipGenerate {
+				return errors.New(i18n.G("secret not inserted (#generate=false): %s", secStat.LocalName))
+			}
+			return errors.New(i18n.G("secret not generated: %s", secStat.LocalName))
+		}
+	}
+
+	return nil
+}
+
+func getDeployVersion(cliArgs []string, deployMeta stack.DeployMeta, app appPkg.App) (string, error) {
+	// Chaos mode overrides everything
+	if internal.Chaos {
+		v, err := app.Recipe.ChaosVersion()
+		if err != nil {
+			return "", err
+		}
+		log.Debug(i18n.G("version: taking chaos version: %s", v))
+		return v, nil
+	}
+
+	// Check if the deploy version is set with a cli argument
+	if len(cliArgs) == 2 && cliArgs[1] != "" {
+		log.Debug(i18n.G("version: taking version from cli arg: %s", cliArgs[1]))
+		return cliArgs[1], nil
+	}
+
+	// Check if the recipe has a version in the .env file
+	if app.Recipe.EnvVersion != "" && !internal.DeployLatest {
+		if strings.HasSuffix(app.Recipe.EnvVersionRaw, "+U") {
+			// NOTE(d1): use double-line 5 spaces ("FATA ") trick to make a more
+			// informative error message. it's ugly but that's our logging situation
+			// atm
+			return "", errors.New(i18n.G(`cannot redeploy previous chaos version (%s), did you mean to use "--chaos"?
+     to return to a regular release, specify a release tag, commit SHA or use "--latest"`,
+				formatter.BoldDirtyDefault(app.Recipe.EnvVersionRaw)))
+		}
+		log.Debug(i18n.G("version: taking version from .env file: %s", app.Recipe.EnvVersion))
+		return app.Recipe.EnvVersion, nil
+	}
+
+	// Take deployed version
+	if deployMeta.IsDeployed && !internal.DeployLatest {
+		log.Debug(i18n.G("version: taking deployed version: %s", deployMeta.Version))
+		return deployMeta.Version, nil
+	}
+
+	v, err := getLatestVersionOrCommit(app)
+	log.Debug(i18n.G("version: taking new recipe version: %s", v))
+	if err != nil {
+		return "", err
+	}
+	return v, nil
+}
+
+func init() {
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.Force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("perform action without further prompt"),
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.NoDomainChecks,
+		i18n.G("no-domain-checks"),
+		i18n.G("D"),
+		false,
+		i18n.G("disable public DNS checks"),
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.DontWaitConverge,
+		i18n.G("no-converge-checks"),
+		i18n.G("c"),
+		false,
+		i18n.G("disable converge logic checks"),
+	)
+
+	AppDeployCommand.PersistentFlags().BoolVarP(
+		&internal.DeployLatest,
+		i18n.G("latest"),
+		i18n.G("l"),
+		false,
+		i18n.G("deploy latest recipe version"),
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.ShowUnchanged,
+		i18n.G("show-unchanged"),
+		i18n.G("U"),
+		false,
+		i18n.G("show all configs & images, including unchanged ones"),
+	)
+}

cli/app/env.go

@@ -0,0 +1,344 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app env` aliases. use a comma separated list of aliases
+// with no spaces in between
+var appEnvAliases = i18n.G("e")
+
+// translators: `abra app env list` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appEnvListAliases = i18n.G("l,ls")
+
+// translators: `abra app env pull` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appEnvPullAliases = i18n.G("pl,p")
+
+var AppEnvListCommand = &cobra.Command{
+	// translators: `app env list` command
+	Use:     i18n.G("list <domain> [flags]"),
+	Aliases: strings.Split(appEnvListAliases, ","),
+	// translators: Short description for `app env list` command
+	Short:   i18n.G("List all app environment values"),
+	Example: i18n.G("  abra app env list 1312.net"),
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		var envKeys []string
+		for k := range app.Env {
+			envKeys = append(envKeys, k)
+		}
+
+		sort.Strings(envKeys)
+
+		var rows [][]string
+		for _, k := range envKeys {
+			rows = append(rows, []string{k, app.Env[k]})
+		}
+
+		overview := formatter.CreateOverview(i18n.G("ENV OVERVIEW"), rows)
+		fmt.Println(overview)
+	},
+}
+
+var AppEnvPullCommand = &cobra.Command{
+	// translators: `app pull` command
+	Use:     i18n.G("pull <domain> [flags]"),
+	Aliases: strings.Split(appEnvPullAliases, ","),
+	// translators: Short description for `app env pull` command
+	Short: i18n.G("Pull app environment values from a deployed app"),
+	Long: i18n.G(`Pull app environment values from a deploymed app.
+
+A convenient command for when you've lost your app environment file or want to
+synchronize your local app environment values with what is deployed live.`),
+	Example: i18n.G(`  # pull existing .env file and overwrite local values
+  abra app env pull 1312.net --force
+
+  # pull lost app .env file
+  abra app env pull my.gitea.net --server 1312.net`),
+	Args: cobra.MaximumNArgs(2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		appName := args[0]
+
+		appEnvPath := path.Join(config.ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
+		if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
+			log.Fatal(i18n.G("%s already exists?", appEnvPath))
+		}
+
+		if server == "" {
+			log.Fatal(i18n.G("unable to determine server of app %s, please pass --server/-s", appName))
+		}
+
+		serverDir := filepath.Join(config.SERVERS_DIR, server)
+		if _, err := os.Stat(serverDir); os.IsNotExist(err) {
+			log.Fatal(i18n.G("unknown server %s, run \"abra server add %s\"?", server, server))
+		}
+
+		store := contextPkg.NewDefaultDockerContextStore()
+		contexts, err := store.Store.List()
+		if err != nil {
+			log.Fatal(i18n.G("unable to look up server context for %s: %s", server, err))
+		}
+
+		var contextCreated bool
+		if server == "default" {
+			contextCreated = true
+		}
+
+		for _, context := range contexts {
+			if context.Name == server {
+				contextCreated = true
+			}
+		}
+
+		if !contextCreated {
+			log.Fatal(i18n.G("%s missing context, run \"abra server add %s\"?", server, server))
+		}
+
+		cl, err := client.New(server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, appPkg.StackName(appName))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is not deployed?", appName))
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(appName), "app"))
+		targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, internal.NoInput)
+		if err != nil {
+			log.Fatal(i18n.G("unable to retrieve container for %s: %s", appName, err))
+		}
+
+		inspectResult, err := cl.ContainerInspect(context.Background(), targetContainer.ID)
+		if err != nil {
+			log.Fatal(i18n.G("unable to inspect container for %s: %s", appName, err))
+		}
+
+		deploymentEnv := make(map[string]string)
+		for _, envVar := range inspectResult.Config.Env {
+			split := strings.SplitN(envVar, "=", 2)
+			if len(split) != 2 {
+				log.Debug(i18n.G("no value attached to %s", envVar))
+				continue
+			}
+
+			key, val := split[0], split[1]
+			deploymentEnv[key] = val
+		}
+
+		log.Debug(i18n.G("pulled env values from %s deployment: %s", appName, deploymentEnv))
+
+		var (
+			recipeEnvVar string
+			recipeKey    string
+		)
+
+		if r, ok := deploymentEnv["TYPE"]; ok {
+			recipeKey = "TYPE"
+			recipeEnvVar = r
+		}
+
+		if r, ok := deploymentEnv["RECIPE"]; ok {
+			recipeKey = "RECIPE"
+			recipeEnvVar = r
+		}
+
+		if recipeEnvVar == "" {
+			log.Fatal(i18n.G("unable to determine recipe type from %s, env: %v", appName, inspectResult.Config.Env))
+		}
+
+		var recipeName = recipeEnvVar
+		if strings.Contains(recipeEnvVar, ":") {
+			split := strings.Split(recipeEnvVar, ":")
+			recipeName = split[0]
+		}
+
+		recipe := internal.ValidateRecipe(
+			[]string{recipeName},
+			cmd.Name(),
+		)
+
+		version := deployMeta.Version
+		if deployMeta.IsChaos {
+			version = deployMeta.ChaosVersion
+		}
+
+		if _, err := recipe.EnsureVersion(version); err != nil {
+			log.Fatal(err)
+		}
+
+		mergedEnv, err := recipe.SampleEnv()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Debug(i18n.G("retrieved env values from .env.sample of %s: %s", recipe.Name, mergedEnv))
+
+		for k, v := range deploymentEnv {
+			mergedEnv[k] = v
+		}
+
+		if !strings.Contains(recipeEnvVar, ":") {
+			mergedEnv[recipeKey] = fmt.Sprintf("%s:%s", mergedEnv[recipeKey], version)
+		}
+
+		log.Debug(i18n.G("final merged env values for %s are: %s", appName, mergedEnv))
+
+		envSample, err := os.ReadFile(recipe.SampleEnvPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = os.WriteFile(appEnvPath, envSample, 0o664)
+		if err != nil {
+			log.Fatal(i18n.G("unable to write new env %s: %s", appEnvPath, err))
+		}
+
+		read, err := os.ReadFile(appEnvPath)
+		if err != nil {
+			log.Fatal(i18n.G("unable to read new env %s: %s", appEnvPath, err))
+		}
+
+		sampleEnv, err := recipe.SampleEnv()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var composeFileUpdated bool
+		newContents := string(read)
+		for key, val := range mergedEnv {
+			if sampleEnv[key] == val {
+				continue
+			}
+
+			if key == "COMPOSE_FILE" {
+				composeFileUpdated = true
+				continue
+			}
+
+			if m, _ := regexp.MatchString(fmt.Sprintf(`#%s=`, key), newContents); m {
+				log.Debug(i18n.G("uncommenting %s", key))
+				re := regexp.MustCompile(fmt.Sprintf(`#%s=`, key))
+				newContents = re.ReplaceAllString(newContents, fmt.Sprintf("%s=", key))
+			}
+
+			if m, _ := regexp.MatchString(fmt.Sprintf(`# %s=`, key), newContents); m {
+				log.Debug(i18n.G("uncommenting %s", key))
+				re := regexp.MustCompile(fmt.Sprintf(`# %s=`, key))
+				newContents = re.ReplaceAllString(newContents, fmt.Sprintf("%s=", key))
+			}
+
+			if m, _ := regexp.MatchString(fmt.Sprintf(`%s=".*"`, key), newContents); m {
+				log.Debug(i18n.G(`inserting %s="%s" (double quotes)`, key, val))
+				re := regexp.MustCompile(fmt.Sprintf(`%s=".*"`, key))
+				newContents = re.ReplaceAllString(newContents, fmt.Sprintf(`%s="%s"`, key, val))
+				continue
+			}
+
+			if m, _ := regexp.MatchString(fmt.Sprintf(`%s='.*'`, key), newContents); m {
+				log.Debug(i18n.G(`inserting %s='%s' (single quotes)`, key, val))
+				re := regexp.MustCompile(fmt.Sprintf(`%s='.*'`, key))
+				newContents = re.ReplaceAllString(newContents, fmt.Sprintf(`%s='%s'`, key, val))
+				continue
+			}
+
+			if m, _ := regexp.MatchString(fmt.Sprintf("%s=.*", key), newContents); m {
+				log.Debug(i18n.G("inserting %s=%s (no quotes)", key, val))
+				re := regexp.MustCompile(fmt.Sprintf("%s=.*", key))
+				newContents = re.ReplaceAllString(newContents, fmt.Sprintf("%s=%s", key, val))
+			}
+		}
+
+		err = os.WriteFile(appEnvPath, []byte(newContents), 0)
+		if err != nil {
+			log.Fatal(i18n.G("unable to write new env %s: %s", appEnvPath, err))
+		}
+
+		log.Info(i18n.G("%s successfully created", appEnvPath))
+
+		if composeFileUpdated {
+			log.Warn(i18n.G("manual update required: COMPOSE_FILE=\"%s\"", mergedEnv["COMPOSE_FILE"]))
+		}
+	},
+}
+
+var AppEnvCommand = &cobra.Command{
+	// translators: `app env` command group
+	Use:     i18n.G("env [cmd] [args] [flags]"),
+	Aliases: strings.Split(appEnvAliases, ","),
+	// translators: Short description for `app env` command group
+	Short: i18n.G("Manage app environment values"),
+}
+
+var (
+	server string
+)
+
+func init() {
+	AppEnvPullCommand.Flags().BoolVarP(
+		&internal.Force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("perform action without further prompt"),
+	)
+
+	AppEnvPullCommand.Flags().StringVarP(
+		&server,
+		i18n.G("server"),
+		i18n.G("s"),
+		"",
+		i18n.G("server associated with deployed app"),
+	)
+
+	AppEnvPullCommand.RegisterFlagCompletionFunc(
+		i18n.G("server"),
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.ServerNameComplete()
+		},
+	)
+}

cli/app/labels.go

@@ -0,0 +1,147 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app labels` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appLabelsAliases = i18n.G("lb")
+
+var AppLabelsCommand = &cobra.Command{
+	// translators: `app labels` command
+	Use:     i18n.G("labels <domain> [flags]"),
+	Aliases: strings.Split(appLabelsAliases, ","),
+	// translators: Short description for `app labels` command
+	Short:   i18n.G("Show deployment labels"),
+	Long:    i18n.G("Both local recipe and live deployment labels are shown."),
+	Example: "  " + i18n.G("abra app labels 1312.net"),
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		remoteLabels, err := getLabels(cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		rows := [][]string{
+			{i18n.G("DEPLOYED LABELS"), "---"},
+		}
+
+		remoteLabelKeys := make([]string, 0, len(remoteLabels))
+		for k := range remoteLabels {
+			remoteLabelKeys = append(remoteLabelKeys, k)
+		}
+
+		sort.Strings(remoteLabelKeys)
+
+		for _, k := range remoteLabelKeys {
+			rows = append(rows, []string{
+				k,
+				remoteLabels[k],
+			})
+		}
+
+		if len(remoteLabelKeys) == 0 {
+			rows = append(rows, []string{i18n.G("unknown")})
+		}
+
+		rows = append(rows, []string{i18n.G("RECIPE LABELS"), "---"})
+
+		config, err := app.Recipe.GetComposeConfig(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var localLabelKeys []string
+		var appServiceConfig composetypes.ServiceConfig
+		for _, service := range config.Services {
+			if service.Name == "app" {
+				appServiceConfig = service
+
+				for k := range service.Deploy.Labels {
+					localLabelKeys = append(localLabelKeys, k)
+				}
+			}
+		}
+
+		sort.Strings(localLabelKeys)
+
+		for _, k := range localLabelKeys {
+			rows = append(rows, []string{
+				k,
+				appServiceConfig.Deploy.Labels[k],
+			})
+		}
+
+		overview := formatter.CreateOverview(i18n.G("LABELS OVERVIEW"), rows)
+		fmt.Println(overview)
+	},
+}
+
+// getLabels reads docker labels from running services in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
+func getLabels(cl *dockerClient.Client, stackName string) (map[string]string, error) {
+	labels := make(map[string]string)
+
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return labels, err
+	}
+
+	for _, service := range services {
+		if service.Spec.Name != fmt.Sprintf("%s_app", stackName) {
+			continue
+		}
+
+		for k, v := range service.Spec.Labels {
+			labels[k] = v
+		}
+	}
+
+	return labels, nil
+}
+
+func init() {
+	AppLabelsCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/list.go

@@ -0,0 +1,341 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/tagcmp"
+	"github.com/spf13/cobra"
+)
+
+type appStatus struct {
+	Server       string `json:"server"`
+	Recipe       string `json:"recipe"`
+	AppName      string `json:"appName"`
+	Domain       string `json:"domain"`
+	Status       string `json:"status"`
+	Chaos        string `json:"chaos"`
+	ChaosVersion string `json:"chaosVersion"`
+	Version      string `json:"version"`
+	Upgrade      string `json:"upgrade"`
+}
+
+type serverStatus struct {
+	Apps             []appStatus `json:"apps"`
+	AppCount         int         `json:"appCount"`
+	VersionCount     int         `json:"versionCount"`
+	UnversionedCount int         `json:"unversionedCount"`
+	LatestCount      int         `json:"latestCount"`
+	UpgradeCount     int         `json:"upgradeCount"`
+}
+
+// translators: `abra app list` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appListAliases = i18n.G("ls")
+
+var AppListCommand = &cobra.Command{
+	// translators: `app list` command
+	Use:     i18n.G("list [flags]"),
+	Aliases: strings.Split(appListAliases, ","),
+	// translators: Short description for `app list` command
+	Short: i18n.G("List all managed apps"),
+	Long: i18n.G(`Generate a report of all managed apps.
+
+Use "--status/-S" flag to query all servers for the live deployment status.`),
+	Example: i18n.G(`  # list apps of all servers without live status
+  abra app ls
+
+  # list apps of a specific server with live status
+  abra app ls -s 1312.net -S
+
+  # list apps of all servers which match a specific recipe
+  abra app ls -r gitea`),
+	Args: cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		appFiles, err := appPkg.LoadAppFiles(listAppServer)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		apps, err := appPkg.GetApps(appFiles, recipeFilter)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		sort.Sort(appPkg.ByServerAndRecipe(apps))
+
+		statuses := make(map[string]map[string]string)
+		if status {
+			alreadySeen := make(map[string]bool)
+			for _, app := range apps {
+				if _, ok := alreadySeen[app.Server]; !ok {
+					alreadySeen[app.Server] = true
+				}
+			}
+
+			statuses, err = appPkg.GetAppStatuses(apps, internal.MachineReadable)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		var totalServersCount int
+		var totalAppsCount int
+		allStats := make(map[string]serverStatus)
+		for _, app := range apps {
+			var stats serverStatus
+			var ok bool
+			if stats, ok = allStats[app.Server]; !ok {
+				stats = serverStatus{}
+				if recipeFilter == "" {
+					// count server, no filtering
+					totalServersCount++
+				}
+			}
+
+			if app.Recipe.Name == recipeFilter || recipeFilter == "" {
+				if recipeFilter != "" {
+					// only count server if matches filter
+					totalServersCount++
+				}
+
+				appStats := appStatus{}
+				stats.AppCount++
+				totalAppsCount++
+
+				if status {
+					if err := app.Recipe.EnsureUpToDate(); err != nil {
+						log.Warnf(
+							"Failed to ensure repo is up to date for recipe: %s err: %s",
+							app.Recipe.Name,
+							err,
+						)
+					}
+					status := i18n.G("unknown")
+					version := i18n.G("unknown")
+					chaos := i18n.G("unknown")
+					chaosVersion := i18n.G("unknown")
+					if statusMeta, ok := statuses[app.StackName()]; ok {
+						if currentVersion, exists := statusMeta["version"]; exists {
+							if currentVersion != "" {
+								version = currentVersion
+							}
+						}
+						if chaosDeploy, exists := statusMeta["chaos"]; exists {
+							chaos = chaosDeploy
+						}
+						if chaosDeployVersion, exists := statusMeta["chaosVersion"]; exists {
+							chaosVersion = chaosDeployVersion
+						}
+						if statusMeta["status"] != "" {
+							status = statusMeta["status"]
+						}
+						stats.VersionCount++
+					} else {
+						stats.UnversionedCount++
+					}
+
+					appStats.Status = status
+					appStats.Chaos = chaos
+					appStats.ChaosVersion = chaosVersion
+					appStats.Version = version
+
+					var newUpdates []string
+					if version != "unknown" && chaos == "false" {
+						if err := app.Recipe.EnsureExists(); err != nil {
+							log.Fatal(i18n.G("unable to clone %s: %s", app.Name, err))
+						}
+
+						updates, err := app.Recipe.Tags()
+						if err != nil {
+							log.Fatal(i18n.G("unable to retrieve tags for %s: %s", app.Name, err))
+						}
+
+						parsedVersion, err := tagcmp.Parse(version)
+						if err != nil {
+							log.Fatal(err)
+						}
+
+						for _, update := range updates {
+							if ok := tagcmp.IsParsable(update); !ok {
+								log.Debug(i18n.G("unable to parse %s, skipping as upgrade option", update))
+								continue
+							}
+
+							parsedUpdate, err := tagcmp.Parse(update)
+							if err != nil {
+								log.Fatal(err)
+							}
+
+							if update != version && parsedUpdate.IsGreaterThan(parsedVersion) {
+								newUpdates = append(newUpdates, update)
+							}
+						}
+					}
+
+					if len(newUpdates) == 0 {
+						if version == "unknown" {
+							appStats.Upgrade = i18n.G("unknown")
+						} else {
+							appStats.Upgrade = i18n.G("latest")
+							stats.LatestCount++
+						}
+					} else {
+						newUpdates = internal.SortVersionsDesc(newUpdates)
+						appStats.Upgrade = strings.Join(newUpdates, "\n")
+						stats.UpgradeCount++
+					}
+				}
+
+				appStats.Server = app.Server
+				appStats.Recipe = app.Recipe.Name
+				appStats.AppName = app.Name
+				appStats.Domain = app.Domain
+
+				stats.Apps = append(stats.Apps, appStats)
+			}
+			allStats[app.Server] = stats
+		}
+
+		if internal.MachineReadable {
+			jsonstring, err := json.Marshal(allStats)
+			if err != nil {
+				log.Fatal(err)
+			} else {
+				fmt.Println(string(jsonstring))
+			}
+
+			return
+		}
+
+		alreadySeen := make(map[string]bool)
+		for _, app := range apps {
+			if _, ok := alreadySeen[app.Server]; ok {
+				continue
+			}
+
+			serverStat := allStats[app.Server]
+
+			headers := []string{i18n.G("RECIPE"), i18n.G("DOMAIN"), i18n.G("SERVER")}
+			if status {
+				headers = append(headers, []string{
+					i18n.G("STATUS"),
+					i18n.G("CHAOS"),
+					i18n.G("VERSION"),
+					i18n.G("UPGRADE"),
+				}...,
+				)
+			}
+
+			table, err := formatter.CreateTable()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			table.Headers(headers...)
+
+			var rows [][]string
+			for _, appStat := range serverStat.Apps {
+				row := []string{appStat.Recipe, appStat.Domain, appStat.Server}
+				if status {
+					chaosStatus := appStat.Chaos
+					if chaosStatus != "unknown" {
+						chaosEnabled, err := strconv.ParseBool(chaosStatus)
+						if err != nil {
+							log.Fatal(err)
+						}
+						if chaosEnabled && appStat.ChaosVersion != "unknown" {
+							chaosStatus = appStat.ChaosVersion
+						}
+					}
+
+					row = append(row, []string{
+						appStat.Status,
+						chaosStatus,
+						appStat.Version,
+						appStat.Upgrade}...,
+					)
+				}
+
+				rows = append(rows, row)
+			}
+
+			table.Rows(rows...)
+
+			if len(rows) > 0 {
+				if err := formatter.PrintTable(table); err != nil {
+					log.Fatal(err)
+				}
+
+				if len(allStats) > 1 && len(rows) > 0 {
+					fmt.Println() // newline separator for multiple servers
+				}
+			}
+
+			alreadySeen[app.Server] = true
+		}
+	},
+}
+
+var (
+	status        bool
+	recipeFilter  string
+	listAppServer string
+)
+
+func init() {
+	AppListCommand.Flags().BoolVarP(
+		&status,
+		i18n.G("status"),
+		i18n.G("S"),
+		false,
+		i18n.G("show app deployment status"),
+	)
+
+	AppListCommand.Flags().StringVarP(
+		&recipeFilter,
+		i18n.G("recipe"),
+		i18n.G("r"),
+		"",
+		i18n.G("show apps of a specific recipe"),
+	)
+
+	AppListCommand.RegisterFlagCompletionFunc(
+		i18n.G("recipe"),
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.RecipeNameComplete()
+		},
+	)
+
+	AppListCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+
+	AppListCommand.Flags().StringVarP(
+		&listAppServer,
+		i18n.G("server"),
+		i18n.G("s"),
+		"",
+		i18n.G("show apps of a specific server"),
+	)
+
+	AppListCommand.RegisterFlagCompletionFunc(
+		i18n.G("server"),
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.ServerNameComplete()
+		},
+	)
+}

cli/app/logs.go

@@ -0,0 +1,113 @@
+package app
+
+import (
+	"context"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/logs"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app logs` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appLogsAliases = i18n.G("l")
+
+var AppLogsCommand = &cobra.Command{
+	// translators: `app logs` command
+	Use:     i18n.G("logs <domain> [service] [flags]"),
+	Aliases: strings.Split(appLogsAliases, ","),
+	// translators: Short description for `app logs` command
+	Short: i18n.G("Tail app logs"),
+	Args:  cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				return []string{i18n.G("autocomplete failed: %s", err)}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.ServiceNameComplete(app.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+		stackName := app.StackName()
+
+		if err := app.Recipe.EnsureExists(); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is not deployed?", app.Name))
+		}
+
+		var serviceNames []string
+		if len(args) == 2 {
+			serviceNames = []string{args[1]}
+		}
+
+		f, err := app.Filters(true, false, serviceNames...)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		opts := logs.TailOpts{
+			AppName:  app.Name,
+			Services: serviceNames,
+			StdErr:   stdErr,
+			Since:    sinceLogs,
+			Filters:  f,
+		}
+
+		if err := logs.TailLogs(cl, opts); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+var (
+	stdErr    bool
+	sinceLogs string
+)
+
+func init() {
+	AppLogsCommand.Flags().BoolVarP(
+		&stdErr,
+		i18n.G("stderr"),
+		i18n.G("s"),
+		false,
+		i18n.G("only tail stderr"),
+	)
+
+	AppLogsCommand.Flags().StringVarP(
+		&sinceLogs,
+		i18n.G("since"),
+		i18n.G("S"),
+		"",
+		i18n.G("tail logs since YYYY-MM-DDTHH:MM:SSZ"),
+	)
+}

cli/app/move.go

@@ -0,0 +1,354 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/secret"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/volume"
+	dockerclient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app move` aliases. use a comma separated list of aliases
+// with no spaces in between
+var appMoveAliases = i18n.G("m")
+
+var AppMoveCommand = &cobra.Command{
+	// translators: `app move` command
+	Use:     i18n.G("move <domain> <server> [flags]"),
+	Aliases: strings.Split(appMoveAliases, ","),
+	// translators: Short description for `app move` command
+	Short: i18n.G("Moves an app to a different server"),
+	Long: i18n.G(`Move an app to a differnt server.
+
+This command will migrate an app config and copy secrets and volumes from the
+old server to the new one. The app MUST be deployed on the old server before
+doing the move. The app will be undeployed from the current server but not
+deployed on the new server.
+
+The "tar" command is required on both the old and new server as well as "sudo"
+permissions. The "rsync" command is required on your local machine for
+transferring volumes.
+
+Do not forget to update your DNS records. Don't panic, it might take a while
+for the dust to settle after you move an app. If anything goes wrong, you can
+always move the app config file to the original server and deploy it there
+again. No data is removed from the old server.
+
+Use "--dry-run/-r" to see which secrets and volumes will be moved.`),
+	Example: i18n.G(`  # move an app
+  abra app move nextcloud.1312.net myserver.com`),
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			return autocomplete.ServerNameComplete()
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if len(args) <= 1 {
+			log.Fatal(i18n.G("no server provided?"))
+		}
+		newServer := internal.ValidateServer([]string{args[1]})
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		currentServerClient, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), currentServerClient, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s must first be deployed on %s before moving", app.Name, app.Server))
+		}
+
+		resources, err := getAppResources(currentServerClient, app)
+		if err != nil {
+			log.Fatal(i18n.G("unable to retrieve %s resources on %s: %s", app.Name, app.Server, err))
+		}
+
+		internal.MoveOverview(app, newServer, resources.SecretNames(), resources.VolumeNames())
+		if err := internal.PromptProcced(); err != nil {
+			log.Fatal(i18n.G("bailing out: %s", err))
+		}
+
+		log.Info(i18n.G("undeploying %s on %s", app.Name, app.Server))
+		rmOpts := stack.Remove{
+			Namespaces: []string{app.StackName()},
+			Detach:     false,
+		}
+		if err := stack.RunRemove(context.Background(), currentServerClient, rmOpts); err != nil {
+			log.Fatal(i18n.G("failed to remove app from %s: %s", err, app.Server))
+		}
+
+		newServerClient, err := client.New(newServer)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, s := range resources.SecretList {
+			sname := strings.Split(strings.TrimPrefix(s.Spec.Name, app.StackName()+"_"), "_")
+			secretName := strings.Join(sname[:len(sname)-1], "_")
+			data := resources.Secrets[secretName]
+			if err := client.StoreSecret(newServerClient, s.Spec.Name, data); err != nil {
+				if strings.Contains(err.Error(), "already exists") {
+					log.Info(i18n.G("skipping secret (because it already exists) on %s: %s", s.Spec.Name, newServer))
+					continue
+				}
+				log.Fatal(i18n.G("failed to store secret on %s: %s", err, newServer))
+			}
+			log.Info(i18n.G("created secret on %s: %s", s.Spec.Name, newServer))
+		}
+
+		for _, v := range resources.Volumes {
+			log.Info(i18n.G("moving volume %s from %s to %s", v.Name, app.Server, newServer))
+
+			// NOTE(p4u1): Need to create the volume before copying the data, because
+			// when docker creates a new volume it set the folder permissions to
+			// root, which might be wrong. This ensures we always have the correct
+			// folder permissions inside the volume.
+			log.Debug(i18n.G("creating volume %s on %s", v.Name, newServer))
+			_, err := newServerClient.VolumeCreate(context.Background(), volume.CreateOptions{
+				Name:   v.Name,
+				Driver: v.Driver,
+			})
+			if err != nil {
+				log.Fatal(i18n.G("failed to create volume %s on %s: %s", v.Name, newServer, err))
+			}
+
+			filename := fmt.Sprintf("%s_outgoing.tar.gz", v.Name)
+			log.Debug(i18n.G("creating %s on %s", filename, app.Server))
+			tarCmd := fmt.Sprintf("sudo tar --same-owner -czhpf %s -C /var/lib/docker/volumes %s", filename, v.Name)
+			cmd := exec.Command("ssh", app.Server, "-tt", tarCmd)
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("%s failed on %s: output:%s err:%s", tarCmd, app.Server, string(out), err))
+			}
+
+			log.Debug(i18n.G("rsyncing %s from %s to local machine", filename, app.Server))
+			cmd = exec.Command("rsync", "-a", "-v", fmt.Sprintf("%s:%s", app.Server, filename), filename)
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("failed to copy %s from %s to local machine: output:%s err:%s", filename, app.Server, string(out), err))
+			}
+
+			log.Debug(i18n.G("rsyncing %s to %s from local machine", filename, filename, newServer))
+			cmd = exec.Command("rsync", "-a", "-v", filename, fmt.Sprintf("%s:%s", newServer, filename))
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("failed to copy %s from local machine to %s: output:%s err:%s", filename, newServer, string(out), err))
+			}
+
+			log.Debug(i18n.G("extracting %s on %s", filename, newServer))
+			tarExtractCmd := fmt.Sprintf("sudo tar --same-owner -xzpf %s -C /var/lib/docker/volumes", filename)
+			cmd = exec.Command("ssh", newServer, "-tt", tarExtractCmd)
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("%s failed to extract %s on %s: output:%s err:%s", tarExtractCmd, filename, newServer, string(out), err))
+			}
+
+			// Remove tar files
+			log.Debug(i18n.G("removing %s from %s", filename, newServer))
+			cmd = exec.Command("ssh", newServer, "-tt", fmt.Sprintf("sudo rm -rf %s", filename))
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("failed to remove %s from %s: output:%s err:%s", filename, newServer, string(out), err))
+			}
+
+			log.Debug(i18n.G("removing %s from %s", filename, app.Server))
+			cmd = exec.Command("ssh", app.Server, "-tt", fmt.Sprintf("sudo rm -rf %s", filename))
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("failed to remove %s from %s: output:%s err:%s", filename, app.Server, string(out), err))
+			}
+
+			log.Debug(i18n.G("removing %s from local machine", filename))
+			cmd = exec.Command("rm", "-r", "-f", filename)
+			if out, err := cmd.CombinedOutput(); err != nil {
+				log.Fatal(i18n.G("failed to remove %s on local machine: output:%s err:%s", filename, string(out), err))
+			}
+		}
+
+		newServerPath := fmt.Sprintf("%s/servers/%s/%s.env", config.ABRA_DIR, newServer, app.Name)
+		log.Info(i18n.G("migrating app config from %s to %s", app.Server, newServerPath))
+		if err := copyFile(app.Path, newServerPath); err != nil {
+			log.Fatal(i18n.G("failed to migrate app config: %s", err))
+		}
+
+		if err := os.Remove(app.Path); err != nil {
+			log.Fatal(i18n.G("unable to remove %s: %s", app.Path, err))
+		}
+
+		log.Info(i18n.G("%s was successfully moved from %s to %s 🎉", app.Name, app.Server, newServer))
+	},
+}
+
+type AppResources struct {
+	Secrets    map[string]string
+	SecretList []swarm.Secret
+	Volumes    map[string]containertypes.MountPoint
+}
+
+func (a *AppResources) SecretNames() []string {
+	secrets := []string{}
+	for name := range a.Secrets {
+		secrets = append(secrets, name)
+	}
+	return secrets
+}
+
+func (a *AppResources) VolumeNames() []string {
+	volumes := []string{}
+	for name := range a.Volumes {
+		volumes = append(volumes, name)
+	}
+	return volumes
+}
+
+func getAppResources(cl *dockerclient.Client, app app.App) (*AppResources, error) {
+	filter, err := app.Filters(false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+
+	composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+	if err != nil {
+		return nil, err
+	}
+
+	secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+
+	secretConfigs, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+	if err != nil {
+		return nil, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles, Namespace: app.StackName()}
+	compose, err := appPkg.GetAppComposeConfig(app.Name, opts, app.Env)
+	if err != nil {
+		return nil, err
+	}
+
+	resources := &AppResources{
+		Secrets:    make(map[string]string),
+		SecretList: secretList,
+		Volumes:    make(map[string]containertypes.MountPoint),
+	}
+
+	for _, s := range services {
+		secretNames := map[string]string{}
+		for _, serviceCompose := range compose.Services {
+			stackService := fmt.Sprintf("%s_%s", app.StackName(), serviceCompose.Name)
+			if stackService != s.Spec.Name {
+				log.Debug(i18n.G("skipping %s as it does not match %s", stackService, s.Spec.Name))
+				continue
+			}
+
+			for _, secret := range serviceCompose.Secrets {
+				for _, s := range secretList {
+					stackSecret := fmt.Sprintf("%s_%s_%s", app.StackName(), secret.Source, secretConfigs[secret.Source].Version)
+					if s.Spec.Name == stackSecret {
+						secretNames[secret.Source] = s.ID
+						break
+					}
+				}
+			}
+		}
+
+		f := filters.NewArgs()
+		f.Add("name", s.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(context.Background(), cl, f, true)
+		if err != nil {
+			return nil, errors.New(i18n.G("unable to get container matching %s: %s", s.Spec.Name, err))
+		}
+
+		for _, m := range targetContainer.Mounts {
+			if m.Type == mount.TypeVolume {
+				resources.Volumes[m.Name] = m
+			}
+		}
+
+		for secretName, secretID := range secretNames {
+			if _, ok := resources.Secrets[secretName]; ok {
+				continue
+			}
+
+			log.Debug(i18n.G("extracting secret %s on %s", secretName, app.Server))
+
+			cmd := fmt.Sprintf("sudo cat /var/lib/docker/containers/%s/mounts/secrets/%s", targetContainer.ID, secretID)
+			out, err := exec.Command("ssh", app.Server, "-tt", cmd).Output()
+			if err != nil {
+				return nil, errors.New(i18n.G("%s failed on %s: output:%s err:%s", cmd, app.Server, string(out), err))
+			}
+
+			resources.Secrets[secretName] = string(out)
+		}
+	}
+
+	return resources, nil
+}
+
+func copyFile(src string, dst string) error {
+	// Read all content of src to data, may cause OOM for a large file.
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return err
+	}
+
+	// Write data to dst
+	err = os.WriteFile(dst, data, 0o644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func init() {
+	AppMoveCommand.Flags().BoolVarP(
+		&internal.Dry,
+		i18n.G("dry-run"),
+		i18n.G("r"),
+		false,
+		i18n.G("report changes that would be made"),
+	)
+}

cli/app/new.go

@@ -0,0 +1,402 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+var appNewDescription = i18n.G(`Creates a new app from a default recipe.
+
+This new app configuration is stored in your $ABRA_DIR directory under the
+appropriate server.
+
+This command does not deploy your app for you. You will need to run "abra app
+deploy <domain>" to do so.
+
+You can see what recipes are available (i.e. values for the [recipe] argument)
+by running "abra recipe ls".
+
+Recipe commit hashes are supported values for "[version]".
+
+Passing the "--secrets/-S" flag will automatically generate secrets for your
+app and store them encrypted at rest on the chosen target server. These
+generated secrets are only visible at generation time, so please take care to
+store them somewhere safe.
+
+You can use the "--pass/-P" to store these generated passwords locally in a
+pass store (see passwordstore.org for more). The pass command must be available
+on your $PATH.`)
+
+// translators: `abra app new` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appNewAliases = i18n.G("n")
+
+var AppNewCommand = &cobra.Command{
+	// translators: `app new` command
+	Use:     i18n.G("new [recipe] [version] [flags]"),
+	Aliases: strings.Split(appNewAliases, ","),
+	// translators: Short description for `app new` command
+	Short: i18n.G("Create a new app"),
+	Long:  appNewDescription,
+	Args:  cobra.RangeArgs(0, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.RecipeNameComplete()
+		case 1:
+			recipe := internal.ValidateRecipe(args, cmd.Name())
+			return autocomplete.RecipeVersionComplete(recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		recipe := internal.ValidateRecipe(args, cmd.Name())
+
+		if len(args) == 2 && internal.Chaos {
+			log.Fatal(i18n.G("cannot use [version] and --chaos together"))
+		}
+
+		var recipeVersion string
+		if len(args) == 2 {
+			recipeVersion = args[1]
+		}
+
+		chaosVersion := config.CHAOS_DEFAULT
+		if internal.Chaos {
+			var err error
+			chaosVersion, err = recipe.ChaosVersion()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			recipeVersion = chaosVersion
+		} else {
+			if err := recipe.EnsureIsClean(); err != nil {
+				log.Fatal(err)
+			}
+
+			var recipeVersions recipePkg.RecipeVersions
+			if recipeVersion == "" {
+				var err error
+				recipeVersions, _, err = recipe.GetRecipeVersions()
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+
+			if len(recipeVersions) > 0 {
+				latest := recipeVersions[len(recipeVersions)-1]
+				for tag := range latest {
+					recipeVersion = tag
+				}
+
+				if _, err := recipe.EnsureVersion(recipeVersion); err != nil {
+					log.Fatal(err)
+				}
+			} else {
+				if err := recipe.EnsureLatest(); err != nil {
+					log.Fatal(err)
+				}
+
+				if recipeVersion == "" {
+					head, err := recipe.Head()
+					if err != nil {
+						log.Fatal(i18n.G("failed to retrieve latest commit for %s: %s", recipe.Name, err))
+					}
+
+					recipeVersion = formatter.SmallSHA(head.String())
+				}
+			}
+		}
+
+		if err := ensureServerFlag(); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := ensureDomainFlag(recipe, newAppServer); err != nil {
+			log.Fatal(err)
+		}
+
+		sanitisedAppName := appPkg.SanitiseAppName(appDomain)
+		log.Debug(i18n.G("%s sanitised as %s for new app", appDomain, sanitisedAppName))
+
+		if err := appPkg.TemplateAppEnvSample(
+			recipe,
+			appDomain,
+			newAppServer,
+			appDomain,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		sampleEnv, err := recipe.SampleEnv()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := recipe.GetComposeFiles(sampleEnv)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secretsConfig, err := secret.ReadSecretsConfig(
+			recipe.SampleEnvPath,
+			composeFiles,
+			appPkg.StackName(appDomain),
+		)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var appSecrets AppSecrets
+		if generateSecrets {
+			if err := promptForSecrets(recipe.Name, secretsConfig); err != nil {
+				log.Fatal(err)
+			}
+
+			cl, err := client.New(newAppServer)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			appSecrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if newAppServer == "default" {
+			newAppServer = "local"
+		}
+
+		log.Info(i18n.G("%s created (version: %s)", appDomain, recipeVersion))
+
+		if len(secretsConfig) > 0 {
+			var (
+				hasSecretToGenerate bool
+				hasSecretToSkip     bool
+			)
+
+			for _, secretConfig := range secretsConfig {
+				if secretConfig.SkipGenerate {
+					hasSecretToSkip = true
+					continue
+				}
+
+				hasSecretToGenerate = true
+			}
+
+			if hasSecretToGenerate && !generateSecrets {
+				log.Warn(i18n.G("%s requires secret generation before deploy, run \"abra app secret generate %s --all\"", recipe.Name, appDomain))
+			}
+
+			if hasSecretToSkip {
+				log.Warn(i18n.G("%s requires secret insertion before deploy (#generate=false)", recipe.Name))
+			}
+		}
+
+		if len(appSecrets) > 0 {
+			rows := [][]string{}
+			for k, v := range appSecrets {
+				rows = append(rows, []string{k, v})
+			}
+
+			overview := formatter.CreateOverview(i18n.G("SECRETS OVERVIEW"), rows)
+
+			fmt.Println(overview)
+
+			log.Warn(i18n.G(
+				"secrets are %s shown again, please save them %s",
+				formatter.BoldUnderlineStyle.Render("NOT"),
+				formatter.BoldUnderlineStyle.Render("NOW"),
+			))
+		}
+
+		app, err := app.Get(appDomain)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := app.WriteRecipeVersion(recipeVersion, false); err != nil {
+			log.Fatal(i18n.G("writing recipe version failed: %s", err))
+		}
+	},
+}
+
+// AppSecrets represents all app secrest
+type AppSecrets map[string]string
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) {
+	// NOTE(d1): trim to match app.StackName() implementation
+	if len(sanitisedAppName) > config.MAX_SANITISED_APP_NAME_LENGTH {
+		log.Debug(i18n.G("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH]))
+		sanitisedAppName = sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH]
+	}
+
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, newAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if saveInPass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(
+				secretValue,
+				secretName,
+				appDomain,
+				newAppServer,
+			); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return secrets, nil
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag(recipe recipePkg.Recipe, server string) error {
+	if appDomain == "" && !internal.NoInput {
+		prompt := &survey.Input{
+			Message: i18n.G("Specify app domain"),
+			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
+		}
+		if err := survey.AskOne(prompt, &appDomain); err != nil {
+			return err
+		}
+	}
+
+	if appDomain == "" {
+		return errors.New(i18n.G("no domain provided"))
+	}
+
+	return nil
+}
+
+// promptForSecrets asks if we should generate secrets for a new app.
+func promptForSecrets(recipeName string, secretsConfig map[string]secret.Secret) error {
+	if len(secretsConfig) == 0 {
+		log.Debug(i18n.G("%s has no secrets to generate, skipping...", recipeName))
+		return nil
+	}
+
+	if !generateSecrets && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: i18n.G("Generate app secrets?"),
+		}
+		if err := survey.AskOne(prompt, &generateSecrets); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	servers, err := config.GetServers()
+	if err != nil {
+		return err
+	}
+
+	if len(servers) == 1 {
+		newAppServer = servers[0]
+		log.Info(i18n.G("single server detected, choosing %s automatically", newAppServer))
+		return nil
+	}
+
+	if newAppServer == "" && !internal.NoInput {
+		prompt := &survey.Select{
+			Message: i18n.G("Select app server:"),
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &newAppServer); err != nil {
+			return err
+		}
+	}
+
+	if newAppServer == "" {
+		return errors.New(i18n.G("no server provided"))
+	}
+
+	return nil
+}
+
+var (
+	newAppServer    string
+	appDomain       string
+	saveInPass      bool
+	generateSecrets bool
+)
+
+func init() {
+	AppNewCommand.Flags().StringVarP(
+		&newAppServer,
+		i18n.G("server"),
+		i18n.G("s"),
+		"",
+		i18n.G("specify server for new app"),
+	)
+
+	AppNewCommand.RegisterFlagCompletionFunc(
+		i18n.G("server"),
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.ServerNameComplete()
+		},
+	)
+
+	AppNewCommand.Flags().StringVarP(
+		&appDomain,
+		i18n.G("domain"),
+		i18n.G("D"),
+		"",
+		i18n.G("domain name for app"),
+	)
+
+	AppNewCommand.Flags().BoolVarP(
+		&saveInPass,
+		i18n.G("pass"),
+		i18n.G("p"),
+		false,
+		i18n.G("store secrets in a local pass store"),
+	)
+
+	AppNewCommand.Flags().BoolVarP(
+		&generateSecrets,
+		i18n.G("secrets"),
+		i18n.G("S"),
+		false,
+		i18n.G("automatically generate secrets"),
+	)
+
+	AppNewCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+}

cli/app/ps.go

@@ -0,0 +1,217 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	abraService "coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	dockerFormatter "github.com/docker/cli/cli/command/formatter"
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app ps` aliases. use a comma separated list of aliases
+// with no spaces in between
+var appPsAliases = i18n.G("p")
+
+var AppPsCommand = &cobra.Command{
+	// translators: `app ps` command
+	Use:     i18n.G("ps <domain> [flags]"),
+	Aliases: strings.Split(appPsAliases, ","),
+	// translators: Short description for `app ps` command
+	Short: i18n.G("Check app deployment status"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is not deployed?", app.Name))
+		}
+
+		chaosVersion := config.CHAOS_DEFAULT
+		statuses, err := appPkg.GetAppStatuses([]appPkg.App{app}, true)
+		if statusMeta, ok := statuses[app.StackName()]; ok {
+			if isChaos, exists := statusMeta["chaos"]; exists && isChaos == "true" {
+				if cVersion, exists := statusMeta["chaosVersion"]; exists {
+					chaosVersion = cVersion
+					if strings.HasSuffix(chaosVersion, config.DIRTY_DEFAULT) {
+						chaosVersion = formatter.BoldDirtyDefault(chaosVersion)
+					}
+				}
+			}
+		}
+
+		showPSOutput(app, cl, deployMeta.Version, chaosVersion)
+	},
+}
+
+// showPSOutput renders ps output.
+func showPSOutput(app appPkg.App, cl *dockerClient.Client, deployedVersion, chaosVersion string) {
+	composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	deployOpts := stack.Deploy{
+		Composefiles: composeFiles,
+		Namespace:    app.StackName(),
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+	compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	services := compose.Services
+	sort.Slice(services, func(i, j int) bool {
+		return services[i].Name < services[j].Name
+	})
+
+	var rows [][]string
+	allContainerStats := make(map[string]map[string]string)
+	for _, service := range services {
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
+
+		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
+		if err != nil {
+			log.Fatal(err)
+			return
+		}
+
+		var containerStats map[string]string
+		if len(containers) == 0 {
+			containerStats = map[string]string{
+				"version": deployedVersion,
+				"chaos":   chaosVersion,
+				"service": service.Name,
+				"image":   i18n.G("unknown"),
+				"created": i18n.G("unknown"),
+				"status":  i18n.G("unknown"),
+				"state":   i18n.G("unknown"),
+				"ports":   i18n.G("unknown"),
+			}
+		} else {
+			container := containers[0]
+			containerStats = map[string]string{
+				"version": deployedVersion,
+				"chaos":   chaosVersion,
+				"service": abraService.ContainerToServiceName(container.Names, app.StackName()),
+				"image":   formatter.RemoveSha(container.Image),
+				"created": formatter.HumanDuration(container.Created),
+				"status":  container.Status,
+				"state":   container.State,
+				"ports":   dockerFormatter.DisplayablePorts(container.Ports),
+			}
+		}
+
+		allContainerStats[containerStats["service"]] = containerStats
+
+		// NOTE(d1): don't clobber these variables for --machine output
+		dVersion := deployedVersion
+		cVersion := chaosVersion
+
+		if containerStats["service"] != "app" {
+			// NOTE(d1): don't repeat info which only relevant for the "app" service
+			dVersion = ""
+			cVersion = ""
+		}
+
+		row := []string{
+			containerStats["service"],
+			containerStats["status"],
+			containerStats["image"],
+			dVersion,
+			cVersion,
+		}
+
+		rows = append(rows, row)
+	}
+
+	if internal.MachineReadable {
+		rendered, err := json.Marshal(allContainerStats)
+		if err != nil {
+			log.Fatal(i18n.G("unable to convert to JSON: %s", err))
+		}
+
+		fmt.Println(string(rendered))
+
+		return
+	}
+
+	table, err := formatter.CreateTable()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	headers := []string{
+		i18n.G("SERVICE"),
+		i18n.G("STATUS"),
+		i18n.G("IMAGE"),
+		i18n.G("VERSION"),
+		i18n.G("CHAOS"),
+	}
+
+	table.
+		Headers(headers...).
+		Rows(rows...)
+
+	if err := formatter.PrintTable(table); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func init() {
+	AppPsCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+
+	AppPsCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/remove.go

@@ -0,0 +1,167 @@
+package app
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app remove` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appRemoveAliases = i18n.G("rm")
+
+var AppRemoveCommand = &cobra.Command{
+	// translators: `app remove` command
+	Use:     i18n.G("remove <domain> [flags]"),
+	Aliases: strings.Split(appRemoveAliases, ","),
+	// translators: Short description for `app remove` command
+	Short: i18n.G("Remove all app data, locally and remotely"),
+	Long: i18n.G(`Remove everything related to an app which is already undeployed.
+
+By default, it will prompt for confirmation before proceeding. All secrets,
+volumes and the local app env file will be deleted.
+
+Only run this command when you are sure you want to completely remove the app
+and all associated app data. This is a destructive action, Be Careful!
+
+If you would like to delete specific volumes or secrets, please use removal
+sub-commands under "app volume" and "app secret" instead.
+
+Please note, if you delete the local app env file without removing volumes and
+secrets first, Abra will *not* be able to help you remove them afterwards.
+
+To delete everything without prompt, use the "--force/-f" or the "--no-input/n"
+flag.`),
+	Example: i18n.G("  abra app remove 1312.net"),
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if !internal.Force && !internal.NoInput {
+			log.Warn(i18n.G("ALERTA ALERTA: deleting %s data and config (local/remote)", app.Name))
+
+			response := false
+			prompt := &survey.Confirm{Message: i18n.G("are you sure?")}
+			if err := survey.AskOne(prompt, &response); err != nil {
+				log.Fatal(err)
+			}
+
+			if !response {
+				log.Fatal(i18n.G("aborting as requested"))
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+		if deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name))
+		}
+
+		fs, err := app.Filters(false, false)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		configs, err := client.GetConfigs(cl, context.Background(), app.Server, fs)
+		if err != nil {
+			log.Fatal(err)
+		}
+		configNames := client.GetConfigNames(configs)
+
+		if len(configNames) > 0 {
+			if err := client.RemoveConfigs(cl, context.Background(), configNames, internal.Force); err != nil {
+				log.Fatal(i18n.G("removing configs failed: %s", err))
+			}
+
+			log.Info(i18n.G("%d config(s) removed successfully", len(configNames)))
+		} else {
+			log.Info(i18n.G("no configs to remove"))
+		}
+
+		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: fs})
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secrets := make(map[string]string)
+		var secretNames []string
+
+		for _, cont := range secretList {
+			secrets[cont.Spec.Annotations.Name] = cont.ID // we have to map the names to ID's
+			secretNames = append(secretNames, cont.Spec.Annotations.Name)
+		}
+
+		if len(secrets) > 0 {
+			for _, name := range secretNames {
+				err := cl.SecretRemove(context.Background(), secrets[name])
+				if err != nil {
+					log.Fatal(err)
+				}
+				log.Info(i18n.G("secret: %s removed", name))
+			}
+		} else {
+			log.Info(i18n.G("no secrets to remove"))
+		}
+
+		fs, err = app.Filters(false, true)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, fs)
+		if err != nil {
+			log.Fatal(err)
+		}
+		volumeNames := client.GetVolumeNames(volumeList)
+
+		if len(volumeNames) > 0 {
+			err := client.RemoveVolumes(cl, context.Background(), volumeNames, internal.Force, 5)
+			if err != nil {
+				log.Fatal(i18n.G("removing volumes failed: %s", err))
+			}
+
+			log.Info(i18n.G("%d volume(s) removed successfully", len(volumeNames)))
+		} else {
+			log.Info(i18n.G("no volumes to remove"))
+		}
+
+		if err = os.Remove(app.Path); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info(i18n.G("file: %s removed", app.Path))
+	},
+}
+
+func init() {
+	AppRemoveCommand.Flags().BoolVarP(
+		&internal.Force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("perform action without further prompt"),
+	)
+}

cli/app/restart.go

@@ -0,0 +1,174 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/ui"
+	upstream "coopcloud.tech/abra/pkg/upstream/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app restart` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appRestartAliases = i18n.G("re")
+
+var AppRestartCommand = &cobra.Command{
+	// translators: `app restart` command
+	Use:     i18n.G("restart <domain> [[service] | --all-services] [flags]"),
+	Aliases: strings.Split(appRestartAliases, ","),
+	// translators: Short description for `app restart` command
+	Short: i18n.G("Restart an app"),
+	Long: i18n.G(`This command restarts services within a deployed app.
+
+Run "abra app ps <domain>" to see a list of service names.
+
+Pass "--all-services/-a" to restart all services.`),
+	Example: i18n.G(`  # restart a single app service
+  abra app restart 1312.net app
+
+  # restart all app services
+  abra app restart 1312.net -a`),
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			if !allServices {
+				return autocomplete.ServiceNameComplete(args[0])
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		var serviceName string
+		if len(args) == 2 {
+			serviceName = args[1]
+		}
+
+		if serviceName == "" && !allServices {
+			log.Fatal(i18n.G("missing [service]"))
+		}
+
+		if serviceName != "" && allServices {
+			log.Fatal(i18n.G("cannot use [service] and --all-services/-a together"))
+		}
+
+		var serviceNames []string
+		if allServices {
+			var err error
+			serviceNames, err = appPkg.GetAppServiceNames(app.Name)
+			if err != nil {
+				log.Fatal(err)
+			}
+		} else {
+			serviceNames = append(serviceNames, serviceName)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is not deployed?", app.Name))
+		}
+
+		for _, serviceName := range serviceNames {
+			stackServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+
+			service, _, err := cl.ServiceInspectWithRaw(
+				context.Background(),
+				stackServiceName,
+				types.ServiceInspectOptions{},
+			)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			log.Debug(i18n.G("attempting to scale %s to 0", stackServiceName))
+
+			if err := upstream.RunServiceScale(context.Background(), cl, stackServiceName, 0); err != nil {
+				log.Fatal(err)
+			}
+
+			f, err := app.Filters(true, false, serviceName)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			waitOpts := stack.WaitOpts{
+				Services:   []ui.ServiceMeta{{Name: stackServiceName, ID: service.ID}},
+				AppName:    app.Name,
+				ServerName: app.Server,
+				Filters:    f,
+				NoInput:    internal.NoInput,
+				NoLog:      true,
+				Quiet:      true,
+			}
+
+			if err := stack.WaitOnServices(cmd.Context(), cl, waitOpts); err != nil {
+				log.Fatal(err)
+			}
+
+			log.Debug(i18n.G("%s has been scaled to 0", stackServiceName))
+			log.Debug(i18n.G("attempting to scale %s to 1", stackServiceName))
+
+			if err := upstream.RunServiceScale(context.Background(), cl, stackServiceName, 1); err != nil {
+				log.Fatal(err)
+			}
+
+			if err := stack.WaitOnServices(cmd.Context(), cl, waitOpts); err != nil {
+				log.Fatal(err)
+			}
+
+			log.Debug(i18n.G("%s has been scaled to 1", stackServiceName))
+			log.Info(i18n.G("%s service successfully restarted", serviceName))
+		}
+	},
+}
+
+var allServices bool
+
+func init() {
+	AppRestartCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+	AppRestartCommand.Flags().BoolVarP(
+		&allServices,
+		i18n.G("all-services"),
+		i18n.GC("a", "app restart"),
+		false,
+		i18n.G("restart all services"),
+	)
+}

cli/app/restore.go

@@ -0,0 +1,142 @@
+package app
+
+import (
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app restore` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appRestoreAliases = i18n.G("rs")
+
+var AppRestoreCommand = &cobra.Command{
+	// translators: `app restore` command
+	Use:     i18n.G("restore <domain> [flags]"),
+	Aliases: strings.Split(appRestoreAliases, ","),
+	// translators: Short description for `app restore` command
+	Short: i18n.G("Restore a snapshot"),
+	Long: i18n.G(`Snapshots are restored while apps are deployed.
+
+Some restore scenarios may require service / app restarts.`),
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if snapshot != "" {
+			log.Debug(i18n.G("including SNAPSHOT=%s in backupbot exec invocation", snapshot))
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if targetPath != "" {
+			log.Debug(i18n.G("including TARGET=%s in backupbot exec invocation", targetPath))
+			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
+		}
+
+		if internal.NoInput {
+			log.Debug(i18n.G("including NONINTERACTIVE=%v in backupbot exec invocation", internal.NoInput))
+			execEnv = append(execEnv, fmt.Sprintf("NONINTERACTIVE=%v", internal.NoInput))
+		}
+
+		if len(volumes) > 0 {
+			allVolumes := strings.Join(volumes, ",")
+			log.Debug(i18n.G("including VOLUMES=%s in backupbot exec invocation", allVolumes))
+			execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%s", allVolumes))
+		}
+
+		if len(services) > 0 {
+			allServices := strings.Join(services, ",")
+			log.Debug(i18n.G("including CONTAINER=%s in backupbot exec invocation", allServices))
+			execEnv = append(execEnv, fmt.Sprintf("CONTAINER=%s", allServices))
+		}
+
+		if hooks {
+			log.Debug(i18n.G("including NO_COMMANDS=%v in backupbot exec invocation", false))
+			execEnv = append(execEnv, fmt.Sprintf("NO_COMMANDS=%v", false))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+var (
+	targetPath string
+	hooks      bool
+	services   []string
+	volumes    []string
+)
+
+func init() {
+	AppRestoreCommand.Flags().StringVarP(
+		&targetPath,
+		i18n.G("target"),
+		i18n.G("t"),
+		"/",
+		i18n.G("target path"),
+	)
+
+	AppRestoreCommand.Flags().StringArrayVarP(
+		&services,
+		i18n.G("services"),
+		i18n.G("s"),
+		[]string{},
+		i18n.G("restore specific services"),
+	)
+
+	AppRestoreCommand.Flags().StringArrayVarP(
+		&volumes,
+		i18n.G("volumes"),
+		i18n.G("v"),
+		[]string{},
+		i18n.G("restore specific volumes"),
+	)
+
+	AppRestoreCommand.Flags().BoolVarP(
+		&hooks,
+		i18n.G("hooks"),
+		i18n.G("H"),
+		false,
+		i18n.G("enable pre/post-hook command execution"),
+	)
+
+	AppRestoreCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/app/rollback.go

@@ -0,0 +1,381 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/deploy"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/lint"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app rollback` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appRollbackAliases = i18n.G("rl")
+
+var AppRollbackCommand = &cobra.Command{
+	// translators: `app rollback` command
+	Use:     i18n.G("rollback <domain> [version] [flags]"),
+	Aliases: strings.Split(appRollbackAliases, ","),
+	// translators: Short description for `app rollback` command
+	Short: i18n.G("Roll an app back to a previous version"),
+	Long: i18n.G(`This command rolls an app back to a previous version.
+
+Unlike "abra app deploy", chaos operations are not supported here. Only recipe
+versions are supported values for "[version]".
+
+It is possible to "--force/-f" an downgrade if you want to re-deploy a specific
+version.
+
+Only the deployed version is consulted when trying to determine what downgrades
+are available. The live deployment version is the "source of truth" in this
+case. The stored .env version is not consulted.
+
+A downgrade can be destructive, please ensure you have a copy of your app data
+beforehand. See "abra app backup" for more.`),
+	Example: i18n.G(` # standard rollback
+  abra app rollback 1312.net
+
+  # rollback to specific version
+  abra app rollback 1312.net 2.0.0+1.2.3`),
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				return []string{i18n.G("autocomplete failed: %s", err)}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.RecipeVersionComplete(app.Recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			downgradeWarnMessages []string
+			chosenDowngrade       string
+			availableDowngrades   []string
+		)
+
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := ensureDeployed(cl, app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(app.Recipe); err != nil {
+			log.Fatal(err)
+		}
+
+		versions, err := app.Recipe.Tags()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// NOTE(d1): we've no idea what the live deployment version is, so every
+		// possible downgrade can be shown. it's up to the user to make the choice
+		if deployMeta.Version == config.UNKNOWN_DEFAULT {
+			availableDowngrades = versions
+		}
+
+		if len(args) == 2 && args[1] != "" {
+			chosenDowngrade = args[1]
+
+			if err := validateDowngradeVersionArg(chosenDowngrade, app, deployMeta); err != nil {
+				log.Fatal(err)
+			}
+
+			availableDowngrades = append(availableDowngrades, chosenDowngrade)
+		}
+
+		if deployMeta.Version != config.UNKNOWN_DEFAULT && chosenDowngrade == "" {
+			downgradeAvailable, err := ensureDowngradesAvailable(versions, &availableDowngrades, deployMeta)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if !downgradeAvailable {
+				log.Info(i18n.G("no available downgrades"))
+				return
+			}
+		}
+
+		if internal.Force || internal.NoInput || chosenDowngrade != "" {
+			if len(availableDowngrades) > 0 {
+				chosenDowngrade = availableDowngrades[len(availableDowngrades)-1]
+			}
+		} else {
+			if err := chooseDowngrade(availableDowngrades, deployMeta, &chosenDowngrade); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if internal.Force &&
+			chosenDowngrade == "" &&
+			deployMeta.Version != config.UNKNOWN_DEFAULT {
+			chosenDowngrade = deployMeta.Version
+		}
+
+		if chosenDowngrade == "" {
+			log.Fatal(i18n.G("unknown deployed version, unable to downgrade"))
+		}
+
+		log.Debug(i18n.G("choosing %s as version to rollback", chosenDowngrade))
+
+		if _, err := app.Recipe.EnsureVersion(chosenDowngrade); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := deploy.MergeAbraShEnv(app.Recipe, app.Env); err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		stackName := app.StackName()
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
+		}
+
+		compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		newRecipeWithDowngradeVersion := fmt.Sprintf("%s:%s", app.Recipe.Name, chosenDowngrade)
+		appPkg.ExposeAllEnv(stackName, compose, app.Env, newRecipeWithDowngradeVersion)
+
+		appPkg.SetRecipeLabel(compose, stackName, app.Recipe.Name)
+		appPkg.SetChaosLabel(compose, stackName, internal.Chaos)
+		if internal.Chaos {
+			appPkg.SetChaosVersionLabel(compose, stackName, chosenDowngrade)
+		}
+
+		// Gather secrets
+		secretInfo, err := deploy.GatherSecretsForDeploy(cl, app, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Gather configs
+		configInfo, err := deploy.GatherConfigsForDeploy(cl, app, compose, app.Env, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Gather images
+		imageInfo, err := deploy.GatherImagesForDeploy(cl, app, compose, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployedVersion := deployMeta.Version
+		if deployMeta.IsChaos {
+			deployedVersion = deployMeta.ChaosVersion
+		}
+
+		// NOTE(d1): no release notes implemeneted for rolling back
+		if err := internal.DeployOverview(
+			app,
+			deployedVersion,
+			chosenDowngrade,
+			"",
+			downgradeWarnMessages,
+			secretInfo,
+			configInfo,
+			imageInfo,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		f, err := app.Filters(true, false, serviceNames...)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(
+			cl,
+			deployOpts,
+			compose,
+			stackName,
+			app.Server,
+			internal.DontWaitConverge,
+			internal.NoInput,
+			f,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := app.WriteRecipeVersion(chosenDowngrade, false); err != nil {
+			log.Fatal(i18n.G("writing recipe version failed: %s", err))
+		}
+	},
+}
+
+// chooseDowngrade prompts the user to choose an downgrade interactively.
+func chooseDowngrade(
+	availableDowngrades []string,
+	deployMeta stack.DeployMeta,
+	chosenDowngrade *string,
+) error {
+	msg := i18n.G("please select a downgrade (version: %s):", deployMeta.Version)
+
+	if deployMeta.IsChaos {
+		chaosVersion := formatter.BoldDirtyDefault(deployMeta.ChaosVersion)
+
+		msg = i18n.G(
+			"please select a downgrade (version: %s, chaos: %s):",
+			deployMeta.Version,
+			chaosVersion,
+		)
+	}
+
+	prompt := &survey.Select{
+		Message: msg,
+		Options: internal.SortVersionsDesc(availableDowngrades),
+	}
+
+	if err := survey.AskOne(prompt, chosenDowngrade); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// validateDownpgradeVersionArg validates the specific version.
+func validateDowngradeVersionArg(
+	specificVersion string,
+	app appPkg.App,
+	deployMeta stack.DeployMeta,
+) error {
+	parsedDeployedVersion, err := tagcmp.Parse(deployMeta.Version)
+	if err != nil {
+		return errors.New(i18n.G("current deployment '%s' is not a known version for %s", deployMeta.Version, app.Recipe.Name))
+	}
+
+	parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+	if err != nil {
+		return errors.New(i18n.G("'%s' is not a known version for %s", specificVersion, app.Recipe.Name))
+	}
+
+	if parsedSpecificVersion.IsGreaterThan(parsedDeployedVersion) &&
+		!parsedSpecificVersion.Equals(parsedDeployedVersion) {
+		return errors.New(i18n.G("%s is not a downgrade for %s?", deployMeta.Version, specificVersion))
+	}
+
+	if parsedSpecificVersion.Equals(parsedDeployedVersion) && !internal.Force {
+		return errors.New(i18n.G("%s is not a downgrade for %s?", deployMeta.Version, specificVersion))
+	}
+
+	return nil
+}
+
+// ensureDowngradesAvailable ensures that there are available downgrades.
+func ensureDowngradesAvailable(
+	versions []string,
+	availableDowngrades *[]string,
+	deployMeta stack.DeployMeta,
+) (bool, error) {
+	parsedDeployedVersion, err := tagcmp.Parse(deployMeta.Version)
+	if err != nil {
+		return false, err
+	}
+
+	for _, version := range versions {
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return false, err
+		}
+
+		if parsedVersion.IsLessThan(parsedDeployedVersion) &&
+			!(parsedVersion.Equals(parsedDeployedVersion)) {
+			*availableDowngrades = append(*availableDowngrades, version)
+		}
+	}
+
+	if len(*availableDowngrades) == 0 && !internal.Force {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func init() {
+	AppRollbackCommand.Flags().BoolVarP(
+		&internal.Force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("perform action without further prompt"),
+	)
+
+	AppRollbackCommand.Flags().BoolVarP(
+		&internal.NoDomainChecks,
+		i18n.G("no-domain-checks"),
+		i18n.G("D"),
+		false,
+		i18n.G("disable public DNS checks"),
+	)
+
+	AppRollbackCommand.Flags().BoolVarP(
+		&internal.DontWaitConverge,
+		i18n.G("no-converge-checks"),
+		i18n.G("c"),
+		false,
+		i18n.G("disable converge logic checks"),
+	)
+
+	AppRollbackCommand.Flags().BoolVarP(
+		&internal.ShowUnchanged,
+		i18n.G("show-unchanged"),
+		i18n.G("U"),
+		false,
+		i18n.G("show all configs & images, including unchanged ones"),
+	)
+}

cli/app/run.go

@@ -0,0 +1,122 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app run` aliases. use a comma separated list of aliases
+// with no spaces in between
+var appRunAliases = i18n.G("r")
+
+var AppRunCommand = &cobra.Command{
+	// translators: `app run` command
+	Use:     i18n.G("run <domain> <service> <cmd> [[args] [flags] | [flags] -- [args]]"),
+	Aliases: strings.Split(appRunAliases, ","),
+	// translators: Short description for `app run` command
+	Short: i18n.G("Run a command inside a service container"),
+	Example: i18n.G(`  # run <cmd> with args/flags
+  abra app run 1312.net app -- ls -lha
+
+  # run <cmd> without args/flags
+  abra app run 1312.net app bash --user nobody
+
+  # run <cmd> with both kinds of args/flags 
+  abra app run 1312.net app --user nobody -- ls -lha`),
+	Args: cobra.MinimumNArgs(3),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			return autocomplete.ServiceNameComplete(args[0])
+		case 2:
+			return autocomplete.CommandNameComplete(args[0])
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		serviceName := args[1]
+		stackAndServiceName := fmt.Sprintf("^%s_%s", app.StackName(), serviceName)
+
+		filters := filters.NewArgs()
+		filters.Add("name", stackAndServiceName)
+
+		targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		userCmd := args[2:]
+		execCreateOpts := containertypes.ExecOptions{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          userCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if runAsUser != "" {
+			execCreateOpts.User = runAsUser
+		}
+		if noTTY {
+			execCreateOpts.Tty = false
+		}
+
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+var (
+	noTTY     bool
+	runAsUser string
+)
+
+func init() {
+	AppRunCommand.Flags().BoolVarP(&noTTY,
+		i18n.G("no-tty"),
+		i18n.G("t"),
+		false,
+		i18n.G("do not request a TTY"),
+	)
+
+	AppRunCommand.Flags().StringVarP(
+		&runAsUser,
+		i18n.G("user"),
+		i18n.G("u"),
+		"",
+		i18n.G("run command as user"),
+	)
+}

cli/app/secret.go

@@ -0,0 +1,653 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app secret generate` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appSecretGenerateAliases = i18n.G("g")
+
+var AppSecretGenerateCommand = &cobra.Command{
+	// translators: `app secret generate` command
+	Use:     i18n.G("generate <domain> [[secret] [version] | --all] [flags]"),
+	Aliases: strings.Split(appSecretGenerateAliases, ","),
+	// translators: Short description for `app secret generate` command
+	Short: i18n.G("Generate secrets"),
+	Args:  cobra.RangeArgs(1, 3),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				return []string{i18n.G("autocomplete failed: %s", err)}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.SecretComplete(app.Recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		if len(args) <= 2 && !generateAllSecrets {
+			log.Fatal(i18n.G("missing arguments [secret]/[version] or '--all'"))
+		}
+
+		if len(args) > 2 && generateAllSecrets {
+			log.Fatal(i18n.G("cannot use '[secret] [version]' and '--all' together"))
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !generateAllSecrets {
+			secretName := args[1]
+			secretVersion := args[2]
+			s, ok := secrets[secretName]
+			if !ok {
+				log.Fatal(i18n.G("%s doesn't exist in the env config?", secretName))
+			}
+			s.Version = secretVersion
+			secrets = map[string]secret.Secret{
+				secretName: s,
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secretVals, err := secret.GenerateSecrets(cl, secrets, app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if storeInPass {
+			for name, data := range secretVals {
+				if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+					log.Fatal(err)
+				}
+			}
+		}
+
+		if len(secretVals) == 0 {
+			log.Warn(i18n.G("no secrets generated"))
+			os.Exit(1)
+		}
+
+		headers := []string{i18n.G("NAME"), i18n.G("VALUE")}
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table.Headers(headers...)
+
+		var rows [][]string
+		for name, val := range secretVals {
+			row := []string{name, val}
+			rows = append(rows, row)
+			table.Row(row...)
+		}
+
+		if internal.MachineReadable {
+			out, err := formatter.ToJSON(headers, rows)
+			if err != nil {
+				log.Fatal(i18n.G("unable to render to JSON: %s", err))
+			}
+			fmt.Println(out)
+			return
+		}
+
+		if err := formatter.PrintTable(table); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Warn(i18n.G(
+			"generated secrets %s shown again, please take note of them %s",
+			formatter.BoldStyle.Render(i18n.G("NOT")),
+			formatter.BoldStyle.Render(i18n.G("NOW")),
+		))
+	},
+}
+
+// translators: `abra app secret insert` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appSecretInsertAliases = i18n.G("i")
+
+var AppSecretInsertCommand = &cobra.Command{
+	// translators: `app secret insert` command
+	Use:     i18n.G("insert <domain> <secret> <version> [<data>] [flags]"),
+	Aliases: strings.Split(appSecretInsertAliases, ","),
+	// translators: Short description for `app secret insert` command
+	Short: i18n.G("Insert secret"),
+	Long: i18n.G(`This command inserts a secret into an app environment.
+
+Arbitrary secret insertion is not supported. Secrets that are inserted must
+match those configured in the recipe beforehand.
+
+This command can be useful when you want to manually generate secrets for an app
+environment. Typically, you can let Abra generate them for you on app creation
+(see "abra app new --secrets/-S" for more).`),
+	Example: i18n.G(`  # insert regular secret
+  abra app secret insert 1312.net my_secret v1 mySuperSecret
+
+  # insert secret as file
+  abra app secret insert 1312.net my_secret v1 secret.txt -f
+
+  # insert secret from stdin
+  echo "mmySuperSecret" | abra app secret insert 1312.net my_secret v1`),
+	Args: cobra.MinimumNArgs(3),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				return []string{i18n.G("autocomplete failed: %s", err)}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.SecretComplete(app.Recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		name := args[1]
+		version := args[2]
+		data, err := readSecretData(args)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var isRecipeSecret bool
+		for secretName := range secrets {
+			if secretName == name {
+				isRecipeSecret = true
+			}
+		}
+		if !isRecipeSecret {
+			log.Fatal(i18n.G("no secret %s available for recipe %s?", name, app.Recipe.Name))
+		}
+
+		if insertFromFile {
+			raw, err := os.ReadFile(data)
+			if err != nil {
+				log.Fatal(i18n.G("reading secret from file: %s", err))
+			}
+			data = string(raw)
+		}
+
+		if trimInput {
+			data = strings.TrimSpace(data)
+		}
+
+		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
+		if err := client.StoreSecret(cl, secretName, data); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info(i18n.G("%s successfully stored on server", secretName))
+
+		if storeInPass {
+			if err := secret.PassInsertSecret(data, name, app.Name, app.Server); err != nil {
+				log.Fatal(err)
+			}
+		}
+	},
+}
+
+func readSecretData(args []string) (string, error) {
+	if len(args) == 4 {
+		return args[3], nil
+	}
+
+	if len(args) != 3 {
+		return "", errors.New(i18n.G("need 3 or 4 arguments"))
+	}
+	// First check if data is provided by stdin
+	fi, err := os.Stdin.Stat()
+	if err != nil {
+		return "", err
+	}
+	if fi.Mode()&os.ModeNamedPipe != 0 {
+		// Can't insert from stdin and read from file
+		if insertFromFile {
+			return "", errors.New(i18n.G("can not insert from file and read from stdin"))
+		}
+
+		log.Debug(i18n.G("reading secret data from stdin"))
+		bytes, err := io.ReadAll(os.Stdin)
+		if err != nil {
+			return "", errors.New(i18n.G("reading data from stdin: %s", err))
+		}
+
+		return string(bytes), nil
+	}
+	if internal.NoInput {
+		return "", errors.New(i18n.G("must provide <data> argument if --no-input is passed"))
+	}
+
+	log.Debug(i18n.G("secret data not provided on command-line or stdin, prompting"))
+	var prompt survey.Prompt
+	if !insertFromFile {
+		prompt = &survey.Password{
+			Message: i18n.G("specify secret value"),
+		}
+	} else {
+		prompt = &survey.Input{
+			Message: i18n.G("specify secret file"),
+		}
+	}
+	var data string
+	if err := survey.AskOne(prompt, &data); err != nil {
+		return "", err
+	}
+	return data, nil
+}
+
+// secretRm removes a secret.
+func secretRm(cl *dockerClient.Client, app appPkg.App, secretName, parsed string) error {
+	if err := cl.SecretRemove(context.Background(), secretName); err != nil {
+		return err
+	}
+
+	log.Info(i18n.G("deleted %s successfully from server", secretName))
+
+	if removeFromPass {
+		if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+			return err
+		}
+
+		log.Info(i18n.G("deleted %s successfully from local pass store", secretName))
+	}
+
+	return nil
+}
+
+// translators: `abra app secret remove` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appSecretRemoveAliases = i18n.G("rm")
+
+var AppSecretRmCommand = &cobra.Command{
+	// translators: `app secret remove` command
+	Use:     i18n.G("remove <domain> [[secret] | --all] [flags]"),
+	Aliases: strings.Split(appSecretRemoveAliases, ","),
+	// translators: Short description for `app secret remove` command
+	Short: i18n.G("Remove a secret"),
+	Long: i18n.G(`This command removes a secret from an app environment.
+
+Arbitrary secret removal is not supported. Secrets that are removed must
+match those configured in the recipe beforehand.`),
+	Example: i18n.G("  abra app secret rm 1312.net oauth_key"),
+	Args:    cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			if !rmAllSecrets {
+				app, err := appPkg.Get(args[0])
+				if err != nil {
+					return []string{i18n.G("autocomplete failed: %s", err)}, cobra.ShellCompDirectiveError
+				}
+				return autocomplete.SecretComplete(app.Recipe.Name)
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if len(args) == 2 && rmAllSecrets {
+			log.Fatal(i18n.G("cannot use [secret] and --all/-a together"))
+		}
+
+		if len(args) != 2 && !rmAllSecrets {
+			log.Fatal(i18n.G("no secret(s) specified?"))
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		filters, err := app.Filters(false, false)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		remoteSecretNames := make(map[string]bool)
+		for _, cont := range secretList {
+			remoteSecretNames[cont.Spec.Annotations.Name] = true
+		}
+
+		var secretToRm string
+		if len(args) == 2 {
+			secretToRm = args[1]
+		}
+
+		match := false
+		for secretName, val := range secrets {
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+			if _, ok := remoteSecretNames[secretRemoteName]; ok {
+				if secretToRm != "" {
+					if secretName == secretToRm {
+						if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
+							log.Fatal(err)
+						}
+
+						return
+					}
+				} else {
+					match = true
+
+					if err := secretRm(cl, app, secretRemoteName, secretName); err != nil {
+						log.Fatal(err)
+					}
+				}
+			}
+		}
+
+		if !match && secretToRm != "" {
+			log.Fatal(i18n.G("%s doesn't exist on server?", secretToRm))
+		}
+
+		if !match {
+			log.Fatal(i18n.G("no secrets to remove?"))
+		}
+	},
+}
+
+// translators: `abra app secret ls` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appSecretLsAliases = i18n.G("ls")
+
+var AppSecretLsCommand = &cobra.Command{
+	// translators: `app secret list` command
+	Use:     i18n.G("list <domain>"),
+	Aliases: strings.Split(appSecretLsAliases, ","),
+	// translators: Short description for `app secret list` command
+	Short: i18n.G("List all secrets"),
+	Args:  cobra.MinimumNArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		headers := []string{i18n.G("NAME"), i18n.G("VERSION"), i18n.G("GENERATED NAME"), i18n.G("CREATED ON SERVER")}
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table.Headers(headers...)
+
+		secStats, err := secret.PollSecretsStatus(cl, app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Sort secrets to ensure reproducible output
+		sort.Slice(secStats, func(i, j int) bool {
+			return secStats[i].LocalName < secStats[j].LocalName
+		})
+		var rows [][]string
+		for _, secStat := range secStats {
+			row := []string{
+				secStat.LocalName,
+				secStat.Version,
+				secStat.RemoteName,
+				strconv.FormatBool(secStat.CreatedOnRemote),
+			}
+
+			rows = append(rows, row)
+			table.Row(row...)
+		}
+
+		if len(rows) > 0 {
+			if internal.MachineReadable {
+				out, err := formatter.ToJSON(headers, rows)
+				if err != nil {
+					log.Fatal(i18n.G("unable to render to JSON: %s", err))
+				}
+				fmt.Println(out)
+				return
+			}
+
+			if err := formatter.PrintTable(table); err != nil {
+				log.Fatal(err)
+			}
+
+			return
+		}
+
+		log.Warn(i18n.G("no secrets stored for %s", app.Name))
+	},
+}
+
+var AppSecretCommand = &cobra.Command{
+	// translators: `app secret` command group
+	Use:     i18n.G("secret [cmd] [args] [flags]"),
+	Aliases: []string{i18n.G("s")},
+	// translators: Short description for `app secret` command group
+	Short: i18n.G("Manage app secrets"),
+}
+
+var (
+	storeInPass        bool
+	insertFromFile     bool
+	trimInput          bool
+	rmAllSecrets       bool
+	generateAllSecrets bool
+	removeFromPass     bool
+)
+
+func init() {
+	AppSecretGenerateCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+
+	AppSecretGenerateCommand.Flags().BoolVarP(
+		&storeInPass,
+		i18n.G("pass"),
+		i18n.G("p"),
+		false,
+		i18n.G("store generated secrets in a local pass store"),
+	)
+
+	AppSecretGenerateCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	AppSecretGenerateCommand.Flags().BoolVarP(
+		&generateAllSecrets,
+		i18n.G("all"),
+		i18n.GC("a", "app secret generate"),
+		false,
+		i18n.G("generate all secrets"),
+	)
+
+	AppSecretInsertCommand.Flags().BoolVarP(
+		&storeInPass,
+		i18n.G("pass"),
+		i18n.G("p"),
+		false,
+		i18n.G("store generated secrets in a local pass store"),
+	)
+
+	AppSecretInsertCommand.Flags().BoolVarP(
+		&insertFromFile,
+		i18n.G("file"),
+		i18n.G("f"),
+		false,
+		i18n.G("treat input as a file"),
+	)
+
+	AppSecretInsertCommand.Flags().BoolVarP(
+		&trimInput,
+		i18n.G("trim"),
+		i18n.G("t"),
+		false,
+		i18n.G("trim input"),
+	)
+
+	AppSecretInsertCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	AppSecretRmCommand.Flags().BoolVarP(
+		&rmAllSecrets,
+		i18n.G("all"),
+		i18n.GC("a", "app secret rm"),
+		false,
+		i18n.G("remove all secrets"),
+	)
+
+	AppSecretRmCommand.Flags().BoolVarP(
+		&removeFromPass,
+		i18n.G("pass"),
+		i18n.G("p"),
+		false,
+		i18n.G("remove generated secrets from a local pass store"),
+	)
+
+	AppSecretRmCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	AppSecretLsCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	AppSecretLsCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+}

cli/app/services.go

@@ -0,0 +1,103 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/service"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app services` aliases. use a comma separated list of
+// aliases with no spaces in between
+var appServicesAliases = i18n.G("sr")
+
+var AppServicesCommand = &cobra.Command{
+	// translators: `app services` command
+	Use:     i18n.G("services <domain> [flags]"),
+	Aliases: strings.Split(appServicesAliases, ","),
+	// translators: Short description for `app services` command
+	Short: i18n.G("Display all services of an app"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is not deployed?", app.Name))
+		}
+
+		filters, err := app.Filters(true, true)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		headers := []string{i18n.G("SERVICE (SHORT)"), i18n.G("SERVICE (LONG)")}
+		table.Headers(headers...)
+
+		var rows [][]string
+		for _, container := range containers {
+			var containerNames []string
+			for _, containerName := range container.Names {
+				trimmed := strings.TrimPrefix(containerName, "/")
+				containerNames = append(containerNames, trimmed)
+			}
+
+			serviceShortName := service.ContainerToServiceName(container.Names, app.StackName())
+			serviceLongName := fmt.Sprintf("%s_%s", app.StackName(), serviceShortName)
+
+			row := []string{
+				serviceShortName,
+				serviceLongName,
+			}
+
+			rows = append(rows, row)
+		}
+
+		table.Rows(rows...)
+
+		if len(rows) > 0 {
+			if err := formatter.PrintTable(table); err != nil {
+				log.Fatal(err)
+			}
+		}
+	},
+}

cli/app/undeploy.go

@@ -0,0 +1,173 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app undeploy` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appUndeployAliases = i18n.G("un")
+
+var AppUndeployCommand = &cobra.Command{
+	// translators: `app undeploy` command
+	Use: i18n.G("undeploy <domain> [flags]"),
+	// translators: Short description for `app undeploy` command
+	Aliases: strings.Split(appUndeployAliases, ","),
+	Short:   i18n.G("Undeploy a deployed app"),
+	Long: i18n.G(`This does not destroy any application data.
+
+However, you should remain vigilant, as your swarm installation will consider
+any previously attached volumes as eligible for pruning once undeployed.
+
+Passing "--prune/-p" does not remove those volumes.`),
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+		stackName := app.StackName()
+
+		if err := app.Recipe.EnsureExists(); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Debug(i18n.G("checking whether %s is already deployed", stackName))
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, stackName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is not deployed?", app.Name))
+		}
+
+		version := deployMeta.Version
+		if deployMeta.IsChaos {
+			version = deployMeta.ChaosVersion
+		}
+
+		if err := internal.DeployOverview(
+			app,
+			version,
+			config.MISSING_DEFAULT,
+			"",
+			nil,
+			nil,
+			nil,
+			nil,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		opts := stack.Deploy{Composefiles: composeFiles, Namespace: stackName}
+		compose, err := appPkg.GetAppComposeConfig(app.Name, opts, app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		rmOpts := stack.Remove{
+			Namespaces: []string{stackName},
+			Detach:     false,
+		}
+		if err := stack.RunRemove(context.Background(), cl, rmOpts); err != nil {
+			log.Fatal(err)
+		}
+
+		if prune {
+			if err := pruneApp(cl, app); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		log.Info(i18n.G("undeploy succeeded 🟢"))
+
+		if err := app.WriteRecipeVersion(version, false); err != nil {
+			log.Fatal(i18n.G("writing recipe version failed: %s", err))
+		}
+	},
+}
+
+// pruneApp runs the equivalent of a "docker system prune" but only filtering
+// against resources connected with the app deployment. It is not a system wide
+// prune. Volumes are not pruned to avoid unwated data loss.
+func pruneApp(cl *dockerClient.Client, app appPkg.App) error {
+	stackName := app.StackName()
+	ctx := context.Background()
+
+	pruneFilters := filters.NewArgs()
+	stackSearch := fmt.Sprintf("%s*", stackName)
+	pruneFilters.Add("label", stackSearch)
+	cr, err := cl.ContainersPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	cntSpaceReclaimed := formatter.ByteCountSI(cr.SpaceReclaimed)
+	log.Info(i18n.G("containers pruned: %d; space reclaimed: %s", len(cr.ContainersDeleted), cntSpaceReclaimed))
+
+	nr, err := cl.NetworksPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	log.Info(i18n.G("networks pruned: %d", len(nr.NetworksDeleted)))
+
+	ir, err := cl.ImagesPrune(ctx, pruneFilters)
+	if err != nil {
+		return err
+	}
+
+	imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
+	log.Info(i18n.G("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed))
+
+	return nil
+}
+
+var (
+	prune bool
+)
+
+func init() {
+	AppUndeployCommand.Flags().BoolVarP(
+		&prune,
+		i18n.G("prune"),
+		i18n.G("p"),
+		false,
+		i18n.G("prune unused containers, networks, and dangling images"),
+	)
+}

cli/app/upgrade.go

@@ -0,0 +1,498 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/deploy"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app upgrade` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appUpgradeAliases = i18n.G("up")
+
+var AppUpgradeCommand = &cobra.Command{
+	// translators: `app upgrade` command
+	Use:     i18n.G("upgrade <domain> [version] [flags]"),
+	Aliases: strings.Split(appUpgradeAliases, ","),
+	// translators: Short description for `app upgrade` command
+	Short: i18n.G("Upgrade an app"),
+	Long: i18n.G(`Upgrade an app.
+
+Unlike "abra app deploy", chaos operations are not supported here. Only recipe
+versions are supported values for "[version]".
+
+It is possible to "--force/-f" an upgrade if you want to re-deploy a specific
+version.
+
+Only the deployed version is consulted when trying to determine what upgrades
+are available. The live deployment version is the "source of truth" in this
+case. The stored .env version is not consulted.
+
+An upgrade can be destructive, please ensure you have a copy of your app data
+beforehand. See "abra app backup" for more.`),
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				return []string{i18n.G("autocomplete failed: %s", err)}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.RecipeVersionComplete(app.Recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			upgradeWarnMessages []string
+			chosenUpgrade       string
+			availableUpgrades   []string
+			upgradeReleaseNotes string
+		)
+
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(recipe.EnsureContext{
+			Chaos:   internal.Chaos,
+			Offline: internal.Offline,
+			// Ignore the env version for now, to make sure we are at the latest commit.
+			// This enables us to get release notes, that were added after a release.
+			IgnoreEnvVersion: true,
+		}); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := ensureDeployed(cl, app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := lint.LintForErrors(app.Recipe); err != nil {
+			log.Fatal(err)
+		}
+
+		versions, err := app.Recipe.Tags()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// NOTE(d1): we've no idea what the live deployment version is, so every
+		// possible upgrade can be shown. it's up to the user to make the choice
+		if deployMeta.Version == config.UNKNOWN_DEFAULT {
+			availableUpgrades = versions
+		}
+
+		if len(args) == 2 && args[1] != "" {
+			chosenUpgrade = args[1]
+
+			if err := validateUpgradeVersionArg(chosenUpgrade, app, deployMeta); err != nil {
+				log.Fatal(err)
+			}
+
+			availableUpgrades = append(availableUpgrades, chosenUpgrade)
+		}
+
+		if deployMeta.Version != config.UNKNOWN_DEFAULT && chosenUpgrade == "" {
+			upgradeAvailable, err := ensureUpgradesAvailable(app, versions, &availableUpgrades, deployMeta)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if !upgradeAvailable {
+				log.Info(i18n.G("no available upgrades"))
+				return
+			}
+		}
+
+		if internal.Force || internal.NoInput || chosenUpgrade != "" {
+			if len(availableUpgrades) > 0 {
+				chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+			}
+		} else {
+			if err := chooseUpgrade(availableUpgrades, deployMeta, &chosenUpgrade); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if internal.Force &&
+			chosenUpgrade == "" &&
+			deployMeta.Version != config.UNKNOWN_DEFAULT {
+			chosenUpgrade = deployMeta.Version
+		}
+
+		if chosenUpgrade == "" {
+			log.Fatal(i18n.G("unknown deployed version, unable to upgrade"))
+		}
+
+		log.Debug(i18n.G("choosing %s as version to upgrade", chosenUpgrade))
+
+		// Get the release notes before checking out the new version in the
+		// recipe. This enables us to get release notes, that were added after
+		// a release.
+		if err := getReleaseNotes(app, versions, chosenUpgrade, deployMeta, &upgradeReleaseNotes); err != nil {
+			log.Fatal(err)
+		}
+
+		if _, err := app.Recipe.EnsureVersion(chosenUpgrade); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := deploy.MergeAbraShEnv(app.Recipe, app.Env); err != nil {
+			log.Fatal(err)
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		stackName := app.StackName()
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    stackName,
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
+		}
+
+		compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		newRecipeWithUpgradeVersion := fmt.Sprintf("%s:%s", app.Recipe.Name, chosenUpgrade)
+		appPkg.ExposeAllEnv(stackName, compose, app.Env, newRecipeWithUpgradeVersion)
+
+		appPkg.SetRecipeLabel(compose, stackName, app.Recipe.Name)
+		appPkg.SetChaosLabel(compose, stackName, internal.Chaos)
+		if internal.Chaos {
+			appPkg.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
+		}
+
+		envVars, err := appPkg.CheckEnv(app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				upgradeWarnMessages = append(upgradeWarnMessages,
+					i18n.G("%s missing from %s.env", envVar.Name, app.Domain),
+				)
+			}
+		}
+
+		// Gather secrets
+		secretInfo, err := deploy.GatherSecretsForDeploy(cl, app, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Gather configs
+		configInfo, err := deploy.GatherConfigsForDeploy(cl, app, compose, app.Env, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Gather images
+		imageInfo, err := deploy.GatherImagesForDeploy(cl, app, compose, internal.ShowUnchanged)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if showReleaseNotes {
+			fmt.Print(upgradeReleaseNotes)
+			return
+		}
+
+		if upgradeReleaseNotes == "" {
+			upgradeWarnMessages = append(
+				upgradeWarnMessages,
+				fmt.Sprintf("no release notes available for %s", chosenUpgrade),
+			)
+		}
+
+		deployedVersion := deployMeta.Version
+		if deployMeta.IsChaos {
+			deployedVersion = deployMeta.ChaosVersion
+		}
+
+		if err := internal.DeployOverview(
+			app,
+			deployedVersion,
+			chosenUpgrade,
+			upgradeReleaseNotes,
+			upgradeWarnMessages,
+			secretInfo,
+			configInfo,
+			imageInfo,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		f, err := app.Filters(true, false, serviceNames...)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(
+			cl,
+			deployOpts,
+			compose,
+			stackName,
+			app.Server,
+			internal.DontWaitConverge,
+			internal.NoInput,
+			f,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		postDeployCmds, ok := app.Env["POST_UPGRADE_CMDS"]
+		if ok && !internal.DontWaitConverge {
+			log.Debug(i18n.G("run the following post-deploy commands: %s", postDeployCmds))
+
+			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
+				log.Fatal(i18n.G("attempting to run post deploy commands, saw: %s", err))
+			}
+		}
+
+		if err := app.WriteRecipeVersion(chosenUpgrade, false); err != nil {
+			log.Fatal(i18n.G("writing recipe version failed: %s", err))
+		}
+	},
+}
+
+// chooseUpgrade prompts the user to choose an upgrade interactively.
+func chooseUpgrade(
+	availableUpgrades []string,
+	deployMeta stack.DeployMeta,
+	chosenUpgrade *string,
+) error {
+	msg := i18n.G("please select an upgrade (version: %s):", deployMeta.Version)
+
+	if deployMeta.IsChaos {
+		chaosVersion := formatter.BoldDirtyDefault(deployMeta.ChaosVersion)
+
+		msg = i18n.G(
+			"please select an upgrade (version: %s, chaos: %s):",
+			deployMeta.Version,
+			chaosVersion,
+		)
+	}
+
+	prompt := &survey.Select{
+		Message: msg,
+		Options: internal.SortVersionsDesc(availableUpgrades),
+	}
+
+	if err := survey.AskOne(prompt, chosenUpgrade); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getReleaseNotes(
+	app appPkg.App,
+	versions []string,
+	chosenUpgrade string,
+	deployMeta stack.DeployMeta,
+	upgradeReleaseNotes *string,
+) error {
+	parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
+	if err != nil {
+		return errors.New(i18n.G("parsing chosen upgrade version failed: %s", err))
+	}
+
+	parsedDeployedVersion, err := tagcmp.Parse(deployMeta.Version)
+	if err != nil {
+		return errors.New(i18n.G("parsing deployment version failed: %s", err))
+	}
+
+	for _, version := range internal.SortVersionsDesc(versions) {
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return errors.New(i18n.G("parsing recipe version failed: %s", err))
+		}
+
+		if parsedVersion.IsGreaterThan(parsedDeployedVersion) &&
+			parsedVersion.IsLessThan(parsedChosenUpgrade) {
+			note, err := app.Recipe.GetReleaseNotes(version, app.Domain)
+			if err != nil {
+				return err
+			}
+
+			if note != "" {
+				// NOTE(d1): trim any final newline on the end of the note itself before
+				//           we manually handle newlines (for multiple release notes and
+				//           ensuring space between the warning messages)
+				note = strings.TrimSuffix(note, "\n")
+
+				*upgradeReleaseNotes += fmt.Sprintf("%s\n", note)
+			}
+		}
+	}
+
+	return nil
+}
+
+// ensureUpgradesAvailable ensures that there are available upgrades.
+func ensureUpgradesAvailable(
+	app appPkg.App,
+	versions []string,
+	availableUpgrades *[]string,
+	deployMeta stack.DeployMeta,
+) (bool, error) {
+	parsedDeployedVersion, err := tagcmp.Parse(deployMeta.Version)
+	if err != nil {
+		return false, errors.New(i18n.G("parsing deployed version failed: %s", err))
+	}
+
+	for _, version := range versions {
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return false, errors.New(i18n.G("parsing recipe version failed: %s", err))
+		}
+
+		if parsedVersion.IsGreaterThan(parsedDeployedVersion) &&
+			!(parsedVersion.Equals(parsedDeployedVersion)) {
+			*availableUpgrades = append(*availableUpgrades, version)
+		}
+	}
+
+	if len(*availableUpgrades) == 0 && !internal.Force {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// validateUpgradeVersionArg validates the specific version.
+func validateUpgradeVersionArg(
+	specificVersion string,
+	app appPkg.App,
+	deployMeta stack.DeployMeta,
+) error {
+	parsedSpecificVersion, err := tagcmp.Parse(specificVersion)
+	if err != nil {
+		return errors.New(i18n.G("'%s' is not a known version for %s", specificVersion, app.Recipe.Name))
+	}
+
+	parsedDeployedVersion, err := tagcmp.Parse(deployMeta.Version)
+	if err != nil {
+		return errors.New(i18n.G("'%s' is not a known version", deployMeta.Version))
+	}
+
+	if parsedSpecificVersion.IsLessThan(parsedDeployedVersion) &&
+		!parsedSpecificVersion.Equals(parsedDeployedVersion) {
+		return errors.New(i18n.G("%s is not an upgrade for %s?", deployMeta.Version, specificVersion))
+	}
+
+	if parsedSpecificVersion.Equals(parsedDeployedVersion) && !internal.Force {
+		return errors.New(i18n.G("%s is not an upgrade for %s?", deployMeta.Version, specificVersion))
+	}
+
+	return nil
+}
+
+// ensureDeployed ensures the app is deployed and if so, returns deployment
+// meta info.
+func ensureDeployed(cl *dockerClient.Client, app appPkg.App) (stack.DeployMeta, error) {
+	log.Debug(i18n.G("checking whether %s is already deployed", app.StackName()))
+
+	deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+	if err != nil {
+		return stack.DeployMeta{}, err
+	}
+
+	if !deployMeta.IsDeployed {
+		return stack.DeployMeta{}, errors.New(i18n.G("%s is not deployed?", app.Name))
+	}
+
+	return deployMeta, nil
+}
+
+var showReleaseNotes bool
+
+func init() {
+	AppUpgradeCommand.Flags().BoolVarP(
+		&internal.Force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("perform action without further prompt"),
+	)
+
+	AppUpgradeCommand.Flags().BoolVarP(
+		&internal.NoDomainChecks,
+		i18n.G("no-domain-checks"),
+		i18n.G("D"),
+		false,
+		i18n.G("disable public DNS checks"),
+	)
+
+	AppUpgradeCommand.Flags().BoolVarP(
+		&internal.DontWaitConverge,
+		i18n.G("no-converge-checks"),
+		i18n.G("c"),
+		false,
+		i18n.G("disable converge logic checks"),
+	)
+
+	AppUpgradeCommand.Flags().BoolVarP(
+		&showReleaseNotes,
+		i18n.G("releasenotes"),
+		i18n.G("r"),
+		false,
+		i18n.G("only show release notes"),
+	)
+
+	AppUpgradeCommand.Flags().BoolVarP(
+		&internal.ShowUnchanged,
+		i18n.G("show-unchanged"),
+		i18n.G("U"),
+		false,
+		i18n.G("show all configs & images, including unchanged ones"),
+	)
+}

cli/app/volume.go

@@ -0,0 +1,221 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra app volume list` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appVolumeListAliases = i18n.G("ls")
+
+var AppVolumeListCommand = &cobra.Command{
+	// translators: `app volume list` command
+	Use:     i18n.G("list <domain> [flags]"),
+	Aliases: strings.Split(appVolumeListAliases, ","),
+	// translators: Short description for `app list` command
+	Short: i18n.G("List volumes associated with an app"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		filters, err := app.Filters(false, true)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		volumes, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		headers := []string{i18n.G("NAME"), i18n.G("ON SERVER")}
+
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table.Headers(headers...)
+
+		var rows [][]string
+		for _, volume := range volumes {
+			row := []string{volume.Name, volume.Mountpoint}
+			rows = append(rows, row)
+		}
+
+		table.Rows(rows...)
+
+		if len(rows) > 0 {
+			if err := formatter.PrintTable(table); err != nil {
+				log.Fatal(err)
+			}
+			return
+		}
+
+		log.Warn(i18n.G("no volumes created for %s", app.Name))
+	},
+}
+
+// translators: `abra app volume remove` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appVolumeRemoveAliases = i18n.G("rm")
+
+var AppVolumeRemoveCommand = &cobra.Command{
+	// translators: `app volume remove` command
+	Use: i18n.G("remove <domain> [volume] [flags]"),
+	// translators: Short description for `app volume remove` command
+	Short: i18n.G("Remove volume(s) associated with an app"),
+	Long: i18n.G(`Remove volumes associated with an app.
+
+The app in question must be undeployed before you try to remove volumes. See
+"abra app undeploy <domain>" for more.
+
+The command is interactive and will show a multiple select input which allows
+you to make a seclection. Use the "?" key to see more help on navigating this
+interface.
+
+Passing "--force/-f" will select all volumes for removal. Be careful.`),
+	Example: i18n.G(`  # delete volumes interactively
+  abra app volume rm 1312.net
+
+  # delete specific volume
+  abra app volume rm 1312.net my_volume`),
+	Aliases: strings.Split(appVolumeRemoveAliases, ","),
+	Args:    cobra.MinimumNArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		var volumeToDelete string
+		if len(args) == 2 {
+			volumeToDelete = args[1]
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if deployMeta.IsDeployed {
+			log.Fatal(i18n.G("%s is still deployed. Run \"abra app undeploy %s\"", app.Name, app.Name))
+		}
+
+		filters, err := app.Filters(false, true)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
+		if err != nil {
+			log.Fatal(err)
+		}
+		volumeNames := client.GetVolumeNames(volumeList)
+
+		if volumeToDelete != "" {
+			var exactMatch bool
+
+			fullVolumeToDeleteName := fmt.Sprintf("%s_%s", app.StackName(), volumeToDelete)
+			for _, volName := range volumeNames {
+				if volName == fullVolumeToDeleteName {
+					exactMatch = true
+				}
+			}
+
+			if !exactMatch {
+				log.Fatal(i18n.G("unable to remove volume: no volume with name '%s'?", volumeToDelete))
+			}
+
+			err := client.RemoveVolumes(cl, context.Background(), []string{fullVolumeToDeleteName}, internal.Force, 5)
+			if err != nil {
+				log.Fatal(i18n.G("removing volume %s failed: %s", volumeToDelete, err))
+			}
+
+			log.Info(i18n.G("volume %s removed successfully", volumeToDelete))
+
+			return
+		}
+
+		var volumesToRemove []string
+		if !internal.Force && !internal.NoInput {
+			volumesPrompt := &survey.MultiSelect{
+				Message: i18n.G("which volumes do you want to remove?"),
+				Help:    i18n.G("'x' indicates selected, enter / return to confirm, ctrl-c to exit, vim mode is enabled"),
+				VimMode: true,
+				Options: volumeNames,
+				Default: volumeNames,
+			}
+			if err := survey.AskOne(volumesPrompt, &volumesToRemove); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if internal.Force || internal.NoInput {
+			volumesToRemove = volumeNames
+		}
+
+		if len(volumesToRemove) > 0 {
+			err := client.RemoveVolumes(cl, context.Background(), volumesToRemove, internal.Force, 5)
+			if err != nil {
+				log.Fatal(i18n.G("removing volumes failed: %s", err))
+			}
+
+			log.Info(i18n.G("%d volumes removed successfully", len(volumesToRemove)))
+		} else {
+			log.Info(i18n.G("no volumes removed"))
+		}
+	},
+}
+
+// translators: `abra app volume` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appVolumeAliases = i18n.G("vl")
+
+var AppVolumeCommand = &cobra.Command{
+	// translators: `app volume` command group
+	Use:     i18n.G("volume [cmd] [args] [flags]"),
+	Aliases: strings.Split(appVolumeAliases, ","),
+	Short:   i18n.G("Manage app volumes"),
+}
+
+func init() {
+	AppVolumeRemoveCommand.Flags().BoolVarP(
+		&internal.Force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("perform action without further prompt"),
+	)
+}

cli/catalogue/catalogue.go

@@ -0,0 +1,322 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"slices"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/go-git/go-git/v5"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra catalogue sync` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appCatalogueSyncAliases = i18n.G("s")
+
+var CatalogueSyncCommand = &cobra.Command{
+	// translators: `catalogue sync` command
+	Use:     i18n.G("sync [flags]"),
+	Aliases: strings.Split(appCatalogueSyncAliases, ","),
+	// translators: Short description for `catalogue sync` command
+	Short: i18n.G("Sync recipe catalogue for latest changes"),
+	Args:  cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := catalogue.EnsureCatalogue(); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := catalogue.EnsureUpToDate(); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info(i18n.G("catalogue successfully synced"))
+	},
+}
+
+// translators: `abra catalogue` aliases. use a comma separated list of aliases with
+// no spaces in between
+var appCatalogueAliases = i18n.G("g")
+
+var CatalogueGenerateCommand = &cobra.Command{
+	// translators: `catalogue generate` command
+	Use:     i18n.G("generate [recipe] [flags]"),
+	Aliases: strings.Split(appCatalogueAliases, ","),
+	// translators: Short description for `catalogue generate` command
+	Short: i18n.G("Generate the recipe catalogue"),
+	Long: i18n.G(`Generate a new copy of the recipe catalogue.
+
+N.B. this command **will** wipe local unstaged changes from your local recipes
+if present. "--chaos/-C" on this command refers to the catalogue repository
+("$ABRA_DIR/catalogue") and not the recipes. Please take care not to lose your
+changes.
+
+It is possible to generate new metadata for a single recipe by passing
+[recipe]. The existing local catalogue will be updated, not overwritten.
+
+It is quite easy to get rate limited by Docker Hub when running this command.
+If you have a Hub account you can "docker login" and Abra will automatically
+use those details.
+
+Publish your new release to git.coopcloud.tech with "--publish/-p". This
+requires that you have permission to git push to these repositories and have
+your SSH keys configured on your account. Enable ssh-agent and make sure to add
+your private key and enter your passphrase beforehand.
+
+    eval ` + "`ssh-agent`" + `
+    ssh-add ~/.ssh/<my-ssh-private-key-for-git-coopcloud-tech>`),
+	Example: `  # publish catalogue
+  eval ` + "`ssh-agent`" + `
+  ssh-add ~/.ssh/id_ed25519
+  abra catalogue generate -p`,
+	Args: cobra.RangeArgs(0, 1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var recipeName string
+		if len(args) > 0 {
+			recipeName = args[0]
+		}
+
+		if os.Getenv("SSH_AUTH_SOCK") == "" {
+			log.Warn(i18n.G("ssh: SSH_AUTH_SOCK missing, --publish/-p will fail. see \"abra catalogue generate --help\""))
+		}
+
+		if recipeName != "" {
+			internal.ValidateRecipe(args, cmd.Name())
+		}
+
+		if err := catalogue.EnsureCatalogue(); err != nil {
+			log.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := catalogue.EnsureIsClean(); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		repos, err := recipe.ReadReposMetadata(internal.Debug)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		barLength := len(repos)
+		if recipeName != "" {
+			barLength = 1
+		}
+
+		if !skipUpdates {
+			if err := recipe.UpdateRepositories(repos, recipeName, internal.Debug); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		var warnings []string
+		catl := make(recipe.RecipeCatalogue)
+		catlBar := formatter.CreateProgressbar(barLength, i18n.G("collecting catalogue metadata"))
+		for _, recipeMeta := range repos {
+			if recipeName != "" && recipeName != recipeMeta.Name {
+				if !internal.Debug {
+					catlBar.Add(1)
+				}
+				continue
+			}
+
+			r := recipe.Get(recipeMeta.Name)
+			versions, warnMsgs, err := r.GetRecipeVersions()
+			if err != nil {
+				warnings = append(warnings, err.Error())
+			}
+			if len(warnMsgs) > 0 {
+				warnings = append(warnings, warnMsgs...)
+			}
+
+			features, category, warnMsgs, err := recipe.GetRecipeFeaturesAndCategory(r)
+			if err != nil {
+				warnings = append(warnings, err.Error())
+			}
+			if len(warnMsgs) > 0 {
+				warnings = append(warnings, warnMsgs...)
+			}
+
+			catl[recipeMeta.Name] = recipe.RecipeMeta{
+				Name:          recipeMeta.Name,
+				Repository:    recipeMeta.CloneURL,
+				SSHURL:        recipeMeta.SSHURL,
+				Icon:          recipeMeta.AvatarURL,
+				DefaultBranch: recipeMeta.DefaultBranch,
+				Description:   recipeMeta.Description,
+				Website:       recipeMeta.Website,
+				Versions:      versions,
+				Category:      category,
+				Features:      features,
+			}
+
+			if !internal.Debug {
+				catlBar.Add(1)
+			}
+		}
+
+		if err := catlBar.Close(); err != nil {
+			log.Fatal(err)
+		}
+
+		var uniqueWarnings []string
+		for _, w := range warnings {
+			if !slices.Contains(uniqueWarnings, w) {
+				uniqueWarnings = append(uniqueWarnings, w)
+			}
+		}
+
+		for _, warnMsg := range uniqueWarnings {
+			log.Warn(warnMsg)
+		}
+
+		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if recipeName == "" {
+			if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
+				log.Fatal(err)
+			}
+		} else {
+			catlFS, err := recipe.ReadRecipeCatalogue(internal.Offline)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			catlFS[recipeName] = catl[recipeName]
+
+			updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		log.Info(i18n.G("generated recipe catalogue: %s", config.RECIPES_JSON))
+
+		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
+		if publishChanges {
+
+			isClean, err := gitPkg.IsClean(cataloguePath)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if isClean {
+				if !internal.Dry {
+					log.Fatal(i18n.G("no changes discovered in %s, nothing to publish?", cataloguePath))
+				}
+			}
+
+			msg := i18n.G("chore: publish new catalogue release changes")
+			if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil {
+				log.Fatal(err)
+			}
+
+			repo, err := git.PlainOpen(cataloguePath)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			sshURL := fmt.Sprintf(config.TOOLSHED_SSH_URL_TEMPLATE, config.CATALOGUE_JSON_REPO_NAME)
+			if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil {
+				log.Fatal(err)
+			}
+
+			if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		repo, err := git.PlainOpen(cataloguePath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		head, err := repo.Head()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !internal.Dry && publishChanges {
+			url := fmt.Sprintf("%s/%s/commit/%s", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME, head.Hash())
+			log.Info(i18n.G("new changes published: %s", url))
+		}
+
+		if internal.Dry {
+			log.Info(i18n.G("dry run: no changes published"))
+		}
+	},
+}
+
+// CatalogueCommand defines the `abra catalogue` command and sub-commands.
+var CatalogueCommand = &cobra.Command{
+	// translators: `catalogue` command group
+	Use: i18n.G("catalogue [cmd] [args] [flags]"),
+	// translators: Short description for `catalogue` command group
+	Short:   i18n.G("Manage the recipe catalogue"),
+	Aliases: []string{"c"},
+}
+
+var (
+	publishChanges bool
+	skipUpdates    bool
+)
+
+func init() {
+	CatalogueGenerateCommand.Flags().BoolVarP(
+		&publishChanges,
+		i18n.G("publish"),
+		i18n.G("p"),
+		false,
+		i18n.G("publish changes to git.coopcloud.tech"),
+	)
+
+	CatalogueGenerateCommand.Flags().BoolVarP(
+		&internal.Dry,
+		i18n.G("dry-run"),
+		i18n.G("r"),
+		false,
+		i18n.G("report changes that would be made"),
+	)
+
+	CatalogueGenerateCommand.Flags().BoolVarP(
+		&skipUpdates,
+		i18n.G("skip-updates"),
+		i18n.G("s"),
+		false,
+		i18n.G("skip updating recipe repositories"),
+	)
+
+	CatalogueGenerateCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+}

cli/complete.go

@@ -0,0 +1,71 @@
+package cli
+
+import (
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra autocomplete` aliases. use a comma separated list of
+// aliases with no spaces in between
+var autocompleteAliases = i18n.G("ac")
+
+var AutocompleteCommand = &cobra.Command{
+	// translators: `autocomplete` command
+	Use:     i18n.G("autocomplete [bash|zsh|fish|powershell]"),
+	Aliases: strings.Split(autocompleteAliases, ","),
+	// translators: Short description for `autocomplete` command
+	Short: i18n.G("Generate autocompletion script"),
+	Long: i18n.G(`To load completions:
+
+Bash:
+  # Load autocompletion for the current Bash session
+  $ source <(abra autocomplete bash)
+
+  # To load autocompletion for each session, execute once:
+  # Linux:
+  $ abra autocomplete bash | sudo tee /etc/bash_completion.d/abra
+  # macOS:
+  $ abra autocomplete bash | sudo tee $(brew --prefix)/etc/bash_completion.d/abra
+
+Zsh:
+  # If shell autocompletion is not already enabled in your environment,
+  # you will need to enable it.  You can execute the following once:
+
+  $ echo "autoload -U compinit; compinit" >> ~/.zshrc
+
+  # To load autocompletions for each session, execute once:
+  $ abra autocomplete zsh > "${fpath[1]}/_abra"
+
+  # You will need to start a new shell for this setup to take effect.
+
+fish:
+  $ abra autocomplete fish | source
+
+  # To load autocompletions for each session, execute once:
+  $ abra autocomplete fish > ~/.config/fish/completions/abra.fish
+
+PowerShell:
+  PS> abra autocomplete powershell | Out-String | Invoke-Expression
+
+  # To load autocompletions for every new session, run:
+  PS> abra autocomplete powershell > abra.ps1
+  # and source this file from your PowerShell profile.`),
+	DisableFlagsInUseLine: true,
+	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
+	Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	Run: func(cmd *cobra.Command, args []string) {
+		switch args[0] {
+		case "bash":
+			cmd.Root().GenBashCompletion(os.Stdout)
+		case "zsh":
+			cmd.Root().GenZshCompletion(os.Stdout)
+		case "fish":
+			cmd.Root().GenFishCompletion(os.Stdout, true)
+		case "powershell":
+			cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout)
+		}
+	},
+}

cli/internal/backup.go

@@ -0,0 +1,76 @@
+package internal
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+)
+
+// RetrieveBackupBotContainer gets the deployed backupbot container.
+func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) {
+	ctx := context.Background()
+	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
+	if err != nil {
+		return types.Container{}, errors.New(i18n.G("no backupbot discovered, is it deployed?"))
+	}
+
+	log.Debug(i18n.G("retrieved %s as backup enabled service", chosenService.Spec.Name))
+
+	filters := filters.NewArgs()
+	filters.Add("name", chosenService.Spec.Name)
+	targetContainer, err := containerPkg.GetContainer(
+		ctx,
+		cl,
+		filters,
+		NoInput,
+	)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	return targetContainer, nil
+}
+
+// RunBackupCmdRemote runs a backup related command on a remote backupbot container.
+func RunBackupCmdRemote(
+	cl *dockerClient.Client,
+	backupCmd string,
+	containerID string,
+	execEnv []string) (io.Writer, error) {
+	execBackupListOpts := containertypes.ExecOptions{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          []string{"/usr/bin/backup", "--", backupCmd},
+		Detach:       false,
+		Env:          execEnv,
+		Tty:          true,
+	}
+
+	log.Debug(i18n.G("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts))
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}

cli/internal/cli.go

@@ -0,0 +1,23 @@
+package internal
+
+var (
+	// NOTE(d1): global
+	Debug   bool
+	NoInput bool
+	Offline bool
+	Help    bool
+	Version bool
+
+	// NOTE(d1): sub-command specific
+	Chaos            bool
+	DeployLatest     bool
+	DontWaitConverge bool
+	Dry              bool
+	Force            bool
+	MachineReadable  bool
+	Major            bool
+	Minor            bool
+	NoDomainChecks   bool
+	Patch            bool
+	ShowUnchanged    bool
+)

cli/internal/command.go

@@ -0,0 +1,146 @@
+package internal
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+)
+
+// RunCmdRemote executes an abra.sh command in the target service
+func RunCmdRemote(
+	cl *dockerClient.Client,
+	app appPkg.App,
+	disableTTY bool,
+	abraSh, serviceName, cmdName, cmdArgs, remoteUser string) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, false)
+	if err != nil {
+		return err
+	}
+
+	log.Debug(i18n.G("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server))
+
+	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(abraSh, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	copyOpts := containertypes.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
+		return err
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	shell := "/bin/bash"
+	findShell := []string{"test", "-e", shell}
+	execCreateOpts := containertypes.ExecOptions{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          findShell,
+		Detach:       false,
+		Tty:          false,
+	}
+
+	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		log.Info(i18n.G("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name))
+		shell = "/bin/sh"
+	}
+
+	var cmd []string
+	if cmdArgs != "" {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.Name, app.StackName(), cmdName, cmdArgs)}
+	} else {
+		cmd = []string{shell, "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; STACK_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.Name, app.StackName(), cmdName)}
+	}
+
+	log.Debug(i18n.G("running command: %s", strings.Join(cmd, " ")))
+
+	if remoteUser != "" {
+		log.Debug(i18n.G("running command with user %s", remoteUser))
+		execCreateOpts.User = remoteUser
+	}
+
+	execCreateOpts.Cmd = cmd
+
+	execCreateOpts.Tty = true
+	if disableTTY {
+		execCreateOpts.Tty = false
+		log.Debug(i18n.G("not requesting a remote TTY"))
+	}
+
+	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func EnsureCommand(abraSh, recipeName, execCmd string) error {
+	bytes, err := ioutil.ReadFile(abraSh)
+	if err != nil {
+		return err
+	}
+
+	if !strings.Contains(string(bytes), execCmd) {
+		return errors.New(i18n.G("%s doesn't have a %s function", recipeName, execCmd))
+	}
+
+	return nil
+}
+
+// RunCmd runs a shell command and streams stdout/stderr in real-time.
+func RunCmd(cmd *exec.Cmd) error {
+	r, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	cmd.Stderr = cmd.Stdout
+	done := make(chan struct{})
+	scanner := bufio.NewScanner(r)
+
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			fmt.Println(line)
+		}
+		done <- struct{}{}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	<-done
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/internal/deploy.go

@@ -0,0 +1,318 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/charmbracelet/lipgloss"
+	dockerClient "github.com/docker/docker/client"
+)
+
+var borderStyle = lipgloss.NewStyle().
+	BorderStyle(lipgloss.ThickBorder()).
+	Padding(0, 1, 0, 1).
+	MaxWidth(79).
+	BorderForeground(lipgloss.Color("63"))
+
+var headerStyle = lipgloss.NewStyle().
+	Underline(true).
+	Bold(true).
+	PaddingBottom(1)
+
+var leftStyle = lipgloss.NewStyle().
+	Bold(true)
+
+var rightStyle = lipgloss.NewStyle()
+
+// horizontal is a JoinHorizontal helper function.
+func horizontal(left, mid, right string) string {
+	return lipgloss.JoinHorizontal(lipgloss.Left, left, mid, right)
+}
+
+func formatComposeFiles(composeFiles string) string {
+	return strings.ReplaceAll(composeFiles, ":", "\n")
+}
+
+// DeployOverview shows a deployment overview
+func DeployOverview(
+	app appPkg.App,
+	deployedVersion string,
+	toDeployVersion string,
+	releaseNotes string,
+	warnMessages []string,
+	secrets []string,
+	configs []string,
+	images []string,
+) error {
+	deployConfig := "compose.yml"
+	if composeFiles, ok := app.Env["COMPOSE_FILE"]; ok {
+		deployConfig = formatComposeFiles(composeFiles)
+	}
+
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	domain := fmt.Sprintf("https://%s", app.Domain)
+	if domain == "" {
+		domain = config.MISSING_DEFAULT
+	}
+
+	envVersion := app.Recipe.EnvVersionRaw
+	if envVersion == "" {
+		envVersion = config.MISSING_DEFAULT
+	}
+
+	rows := [][]string{
+		{i18n.G("DOMAIN"), domain},
+		{i18n.G("RECIPE"), app.Recipe.Name},
+		{i18n.G("SERVER"), server},
+		{i18n.G("CONFIG"), deployConfig},
+		{"", ""},
+		{i18n.G("CURRENT DEPLOYMENT"), formatter.BoldDirtyDefault(deployedVersion)},
+		{i18n.G("ENV VERSION"), formatter.BoldDirtyDefault(envVersion)},
+		{i18n.G("NEW DEPLOYMENT"), formatter.BoldDirtyDefault(toDeployVersion)},
+	}
+
+	if len(images) > 0 {
+		imageRows := [][]string{
+			{"", ""},
+			{i18n.G("IMAGES"), strings.Join(images, "\n")},
+		}
+		rows = append(rows, imageRows...)
+	}
+
+	if len(secrets) > 0 {
+		secretsRows := [][]string{
+			{"", ""},
+			{i18n.G("SECRETS"), strings.Join(secrets, "\n")},
+		}
+		rows = append(rows, secretsRows...)
+	}
+
+	if len(configs) > 0 {
+		configsRows := [][]string{
+			{"", ""},
+			{i18n.G("CONFIGS"), strings.Join(configs, "\n")},
+		}
+		rows = append(rows, configsRows...)
+	}
+
+	deployType := getDeployType(deployedVersion, toDeployVersion)
+	overview := formatter.CreateOverview(i18n.G("%s OVERVIEW", deployType), rows)
+
+	fmt.Println(overview)
+
+	if releaseNotes != "" {
+		fmt.Print(releaseNotes)
+	}
+
+	for _, msg := range warnMessages {
+		log.Warn(msg)
+	}
+
+	if NoInput {
+		return nil
+	}
+
+	response := false
+	prompt := &survey.Confirm{Message: "proceed?"}
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		log.Fatal(i18n.G("deployment cancelled"))
+	}
+
+	return nil
+}
+
+func getDeployType(currentVersion, newVersion string) string {
+	if newVersion == config.MISSING_DEFAULT {
+		return i18n.G("UNDEPLOY")
+	}
+
+	if strings.Contains(newVersion, "+U") {
+		return i18n.G("CHAOS DEPLOY")
+	}
+
+	if strings.Contains(currentVersion, "+U") {
+		return i18n.G("UNCHAOS DEPLOY")
+	}
+
+	if currentVersion == newVersion {
+		return ("REDEPLOY")
+	}
+
+	if currentVersion == config.MISSING_DEFAULT {
+		return i18n.G("NEW DEPLOY")
+	}
+
+	currentParsed, err := tagcmp.Parse(currentVersion)
+	if err != nil {
+		return i18n.G("DEPLOY")
+	}
+
+	newParsed, err := tagcmp.Parse(newVersion)
+	if err != nil {
+		return i18n.G("DEPLOY")
+	}
+
+	if currentParsed.IsLessThan(newParsed) {
+		return i18n.G("UPGRADE")
+	}
+
+	return i18n.G("DOWNGRADE")
+}
+
+// MoveOverview shows a overview before moving an app to a different server
+func MoveOverview(
+	app appPkg.App,
+	newServer string,
+	secrets []string,
+	volumes []string,
+) {
+	server := app.Server
+	if app.Server == "default" {
+		server = "local"
+	}
+
+	domain := app.Domain
+	if domain == "" {
+		domain = config.MISSING_DEFAULT
+	}
+
+	secretsOverview := strings.Join(secrets, "\n")
+	if len(secrets) == 0 {
+		secretsOverview = config.MISSING_DEFAULT
+	}
+
+	volumesOverview := strings.Join(volumes, "\n")
+	if len(volumes) == 0 {
+		volumesOverview = config.MISSING_DEFAULT
+	}
+
+	rows := [][]string{
+		{i18n.G("DOMAIN"), domain},
+		{i18n.G("RECIPE"), app.Recipe.Name},
+		{i18n.G("OLD SERVER"), server},
+		{i18n.G("NEW SERVER"), newServer},
+		{i18n.G("SECRETS"), secretsOverview},
+		{i18n.G("VOLUMES"), volumesOverview},
+	}
+
+	overview := formatter.CreateOverview(i18n.G("MOVE OVERVIEW"), rows)
+
+	fmt.Println(overview)
+}
+
+func PromptProcced() error {
+	if NoInput {
+		return nil
+	}
+
+	if Dry {
+		return errors.New(i18n.G("dry run"))
+	}
+
+	response := false
+	prompt := &survey.Confirm{Message: i18n.G("proceed?")}
+	if err := survey.AskOne(prompt, &response); err != nil {
+		return err
+	}
+
+	if !response {
+		return errors.New(i18n.G("cancelled"))
+	}
+
+	return nil
+}
+
+// PostCmds parses a string of commands and executes them inside of the respective services
+// the commands string must have the following format:
+// "<service> <command> <arguments>|<service> <command> <arguments>|... "
+func PostCmds(cl *dockerClient.Client, app appPkg.App, commands string) error {
+	if _, err := os.Stat(app.Recipe.AbraShPath); err != nil {
+		if os.IsNotExist(err) {
+			return errors.New(i18n.G("%s does not exist for %s?", app.Recipe.AbraShPath, app.Name))
+		}
+		return err
+	}
+
+	for _, command := range strings.Split(commands, "|") {
+		commandParts := strings.Split(command, " ")
+		if len(commandParts) < 2 {
+			return errors.New(i18n.G("not enough arguments: %s", command))
+		}
+		targetServiceName := commandParts[0]
+		cmdName := commandParts[1]
+		parsedCmdArgs := ""
+		if len(commandParts) > 2 {
+			parsedCmdArgs = fmt.Sprintf("%s ", strings.Join(commandParts[2:], " "))
+		}
+		log.Info(i18n.G("running post-command '%s %s' in container %s", cmdName, parsedCmdArgs, targetServiceName))
+
+		if err := EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+			return err
+		}
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			return err
+		}
+
+		matchingServiceName := false
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
+			}
+		}
+
+		if !matchingServiceName {
+			return fmt.Errorf("no service %s for %s?", targetServiceName, app.Name)
+		}
+
+		log.Debug(i18n.G("running command %s %s within the context of %s_%s", cmdName, parsedCmdArgs, app.StackName(), targetServiceName))
+
+		requestTTY := true
+		if err := RunCmdRemote(
+			cl,
+			app,
+			requestTTY,
+			app.Recipe.AbraShPath, targetServiceName, cmdName, parsedCmdArgs, ""); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SortVersionsDesc sorts versions in descending order.
+func SortVersionsDesc(versions []string) []string {
+	var tags []tagcmp.Tag
+
+	for _, v := range versions {
+		parsed, _ := tagcmp.Parse(v) // skips unsupported tags
+		tags = append(tags, parsed)
+	}
+
+	sort.Sort(tagcmp.ByTagDesc(tags))
+
+	var desc []string
+	for _, t := range tags {
+		desc = append(desc, t.String())
+	}
+
+	return desc
+}

cli/internal/deploy_test.go

@@ -0,0 +1,17 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSortVersionsDesc(t *testing.T) {
+	versions := SortVersionsDesc([]string{
+		"0.2.3+1.2.2",
+		"1.0.0+2.2.2",
+	})
+
+	assert.Equal(t, "1.0.0+2.2.2", versions[0])
+	assert.Equal(t, "0.2.3+1.2.2", versions[1])
+}

cli/internal/ensure.go

@@ -0,0 +1,11 @@
+package internal
+
+import "coopcloud.tech/abra/pkg/recipe"
+
+func GetEnsureContext() recipe.EnsureContext {
+	return recipe.EnsureContext{
+		Chaos,
+		Offline,
+		DeployLatest,
+	}
+}

cli/internal/recipe.go

@@ -0,0 +1,118 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/distribution/reference"
+)
+
+// PromptBumpType prompts for version bump type
+func PromptBumpType(tagString, latestRelease, changeOverview string) error {
+	if (!Major && !Minor && !Patch) && tagString == "" {
+		fmt.Print(i18n.G(`
+You need to make a decision about what kind of an update this new recipe
+version is. If someone else performs this upgrade, do they have to do some
+migration work or take care of some breaking changes? This can be signaled in
+the version you specify on the recipe deploy label and is called a semantic
+version.
+
+The latest published version is %s.
+
+%s
+
+Here is a semver cheat sheet (more on https://semver.org):
+
+    major: new features/bug fixes, backwards incompatible (e.g 1.0.0 -> 2.0.0).
+           the upgrade won't work without some preparation work and others need
+           to take care when performing it. "it could go wrong".
+
+    minor: new features/bug fixes, backwards compatible (e.g. 0.1.0 -> 0.2.0).
+           the upgrade should Just Work and there are no breaking changes in
+           the app and the recipe config. "it should go fine".
+
+    patch: bug fixes, backwards compatible (e.g. 0.0.1 -> 0.0.2). this upgrade
+           should also Just Work and is mostly to do with minor bug fixes
+           and/or security patches. "nothing to worry about".
+
+`, latestRelease, changeOverview))
+
+		var chosenBumpType string
+		prompt := &survey.Select{
+			Message: fmt.Sprintf("select recipe version increment type"),
+			Options: []string{i18n.G("major"), i18n.G("minor"), i18n.G("patch")},
+		}
+
+		if err := survey.AskOne(prompt, &chosenBumpType); err != nil {
+			return err
+		}
+
+		SetBumpType(chosenBumpType)
+	}
+
+	return nil
+}
+
+// GetBumpType figures out which bump type is specified
+func GetBumpType() string {
+	var bumpType string
+
+	if Major {
+		bumpType = i18n.G("major")
+	} else if Minor {
+		bumpType = i18n.G("minor")
+	} else if Patch {
+		bumpType = i18n.G("patch")
+	} else {
+		log.Fatal(i18n.G("no version bump type specififed?"))
+	}
+
+	return bumpType
+}
+
+// SetBumpType figures out which bump type is specified
+func SetBumpType(bumpType string) {
+	if bumpType == i18n.G("major") {
+		Major = true
+	} else if bumpType == i18n.G("minor") {
+		Minor = true
+	} else if bumpType == i18n.G("patch") {
+		Patch = true
+	} else {
+		log.Fatal(i18n.G("no version bump type specififed?"))
+	}
+}
+
+// GetMainAppImage retrieves the main 'app' image name
+func GetMainAppImage(recipe recipe.Recipe) (string, error) {
+	var path string
+
+	config, err := recipe.GetComposeConfig(nil)
+	if err != nil {
+		return "", err
+	}
+	for _, service := range config.Services {
+		if service.Name == "app" {
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				return "", err
+			}
+
+			path = reference.Path(img)
+			path = formatter.StripTagMeta(path)
+
+			return path, nil
+		}
+	}
+
+	if path == "" {
+		return path, errors.New(i18n.G("%s has no main 'app' service?", recipe.Name))
+	}
+
+	return path, nil
+}

cli/internal/validate.go

@@ -0,0 +1,177 @@
+package internal
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/AlecAivazis/survey/v2"
+)
+
+// ValidateRecipe ensures the recipe arg is valid.
+func ValidateRecipe(args []string, cmdName string) recipe.Recipe {
+	var recipeName string
+	if len(args) > 0 {
+		recipeName = args[0]
+	}
+
+	var recipes []string
+
+	catl, err := recipe.ReadRecipeCatalogue(Offline)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	knownRecipes := make(map[string]bool)
+	for name := range catl {
+		knownRecipes[name] = true
+	}
+
+	localRecipes, err := recipe.GetRecipesLocal()
+	if err != nil {
+		log.Debug(i18n.G("can't read local recipes: %s", err))
+	} else {
+		for _, recipeLocal := range localRecipes {
+			if _, ok := knownRecipes[recipeLocal]; !ok {
+				knownRecipes[recipeLocal] = true
+			}
+		}
+	}
+
+	for recipeName := range knownRecipes {
+		recipes = append(recipes, recipeName)
+	}
+
+	if recipeName == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: i18n.G("Select recipe"),
+			Options: recipes,
+		}
+		if err := survey.AskOne(prompt, &recipeName); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if recipeName == "" {
+		log.Fatal(i18n.G("no recipe name provided"))
+	}
+
+	if _, ok := knownRecipes[recipeName]; !ok {
+		if !strings.Contains(recipeName, "/") {
+			log.Fatal(i18n.G("no recipe '%s' exists?", recipeName))
+		}
+	}
+
+	chosenRecipe := recipe.Get(recipeName)
+	if err := chosenRecipe.EnsureExists(); err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = chosenRecipe.GetComposeConfig(nil)
+	if err != nil {
+		if cmdName == i18n.G("generate") {
+			if strings.Contains(err.Error(), "missing a compose") {
+				log.Fatal(err)
+			}
+			log.Warn(err)
+		} else {
+			if strings.Contains(err.Error(), "template_driver is not allowed") {
+				log.Warn(i18n.G("ensure %s recipe compose.* files include \"version: '3.8'\"", recipeName))
+			}
+			log.Fatal(i18n.G("unable to validate recipe: %s", err))
+		}
+	}
+
+	log.Debug(i18n.G("validated %s as recipe argument", recipeName))
+
+	return chosenRecipe
+}
+
+// ValidateApp ensures the app name arg is valid.
+func ValidateApp(args []string) app.App {
+	if len(args) == 0 {
+		log.Fatal(i18n.G("no app provided"))
+	}
+
+	appName := args[0]
+
+	app, err := app.Get(appName)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Debug(i18n.G("validated %s as app argument", appName))
+
+	return app
+}
+
+// ValidateDomain ensures the domain name arg is valid.
+func ValidateDomain(args []string) string {
+	var domainName string
+	if len(args) > 0 {
+		domainName = args[0]
+	}
+
+	if domainName == "" && !NoInput {
+		prompt := &survey.Input{
+			Message: i18n.G("Specify a domain name"),
+			Default: "1312.net",
+		}
+		if err := survey.AskOne(prompt, &domainName); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if domainName == "" {
+		log.Fatal(i18n.G("no domain provided"))
+	}
+
+	log.Debug(i18n.G("validated %s as domain argument", domainName))
+
+	return domainName
+}
+
+// ValidateServer ensures the server name arg is valid.
+func ValidateServer(args []string) string {
+	var serverName string
+	if len(args) > 0 {
+		serverName = args[0]
+	}
+
+	serverNames, err := config.ReadServerNames()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if serverName == "" && !NoInput {
+		prompt := &survey.Select{
+			Message: i18n.G("Specify a server name"),
+			Options: serverNames,
+		}
+		if err := survey.AskOne(prompt, &serverName); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	matched := false
+	for _, name := range serverNames {
+		if name == serverName {
+			matched = true
+		}
+	}
+
+	if serverName == "" {
+		log.Fatal(i18n.G("no server provided"))
+	}
+
+	if !matched {
+		log.Fatal(i18n.G("server doesn't exist?"))
+	}
+
+	log.Debug(i18n.G("validated %s as server argument", serverName))
+
+	return serverName
+}

cli/recipe/diff.go

@@ -0,0 +1,38 @@
+package recipe
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe diff` aliases. use a comma separated list of aliases
+// with no spaces in between
+var recipeDiffAliases = i18n.G("d")
+
+var RecipeDiffCommand = &cobra.Command{
+	// translators: `recipe diff` command
+	Use:     i18n.G("diff <recipe> [flags]"),
+	Aliases: strings.Split(recipeDiffAliases, ","),
+	// translators: Short description for `recipe diff` command
+	Short: i18n.G("Show unstaged changes in recipe config"),
+	Long:  i18n.G("This command requires /usr/bin/git."),
+	Args:  cobra.MinimumNArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		r := internal.ValidateRecipe(args, cmd.Name())
+		if err := gitPkg.DiffUnstaged(r.Dir); err != nil {
+			log.Fatal(err)
+		}
+	},
+}

cli/recipe/fetch.go

@@ -0,0 +1,142 @@
+package recipe
+
+import (
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/go-git/go-git/v5"
+	gitCfg "github.com/go-git/go-git/v5/config"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe fetch` aliases. use a comma separated list of aliases
+// with no spaces in between
+var recipeFetchAliases = i18n.G("f")
+
+var RecipeFetchCommand = &cobra.Command{
+	// translators: `recipe fetch` command
+	Use:     i18n.G("fetch [recipe | --all] [flags]"),
+	Aliases: strings.Split(recipeFetchAliases, ","),
+	// translators: Short description for `recipe fetch` command
+	Short: i18n.G("Clone recipe(s) locally"),
+	Long:  i18n.G(`Using "--force/-f" Git syncs an existing recipe. It does not erase unstaged changes.`),
+	Args:  cobra.RangeArgs(0, 1),
+	Example: i18n.G(`  # fetch from recipe catalogue
+  abra recipe fetch gitea
+
+  # fetch from remote recipe
+  abra recipe fetch git.foo.org/recipes/myrecipe
+
+  # fetch with ssh remote for hacking
+  abra recipe fetch gitea --ssh`),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var recipeName string
+		if len(args) > 0 {
+			recipeName = args[0]
+		}
+
+		if recipeName == "" && !fetchAllRecipes {
+			log.Fatal(i18n.G("missing [recipe] or --all/-a"))
+		}
+
+		if recipeName != "" && fetchAllRecipes {
+			log.Fatal(i18n.G("cannot use [recipe] and --all/-a together"))
+		}
+
+		if recipeName != "" {
+			r := recipe.Get(recipeName)
+			if _, err := os.Stat(r.Dir); !os.IsNotExist(err) {
+				if !force {
+					log.Warn(i18n.G("%s is already fetched", r.Name))
+					return
+				}
+			}
+
+			r = internal.ValidateRecipe(args, cmd.Name())
+
+			if sshRemote {
+				if r.SSHURL == "" {
+					log.Warn(i18n.G("unable to discover SSH remote for %s", r.Name))
+					return
+				}
+
+				repo, err := git.PlainOpen(r.Dir)
+				if err != nil {
+					log.Fatal(i18n.G("unable to open %s: %s", r.Dir, err))
+				}
+
+				if err = repo.DeleteRemote("origin"); err != nil {
+					log.Fatal(i18n.G("unable to remove default remote in %s: %s", r.Dir, err))
+				}
+
+				if _, err := repo.CreateRemote(&gitCfg.RemoteConfig{
+					Name: "origin",
+					URLs: []string{r.SSHURL},
+				}); err != nil {
+					log.Fatal(i18n.G("unable to set SSH remote in %s: %s", r.Dir, err))
+				}
+			}
+
+			return
+		}
+
+		catalogue, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		catlBar := formatter.CreateProgressbar(len(catalogue), i18n.G("fetching latest recipes..."))
+		ensureCtx := internal.GetEnsureContext()
+		for recipeName := range catalogue {
+			r := recipe.Get(recipeName)
+			if err := r.Ensure(ensureCtx); err != nil {
+				log.Error(err)
+			}
+			catlBar.Add(1)
+		}
+	},
+}
+
+var (
+	fetchAllRecipes bool
+	sshRemote       bool
+	force           bool
+)
+
+func init() {
+	RecipeFetchCommand.Flags().BoolVarP(
+		&fetchAllRecipes,
+		i18n.G("all"),
+		i18n.GC("a", "recipe fetch"),
+		false,
+		i18n.G("fetch all recipes"),
+	)
+
+	RecipeFetchCommand.Flags().BoolVarP(
+		&sshRemote,
+		i18n.G("ssh"),
+		i18n.G("s"),
+		false,
+		i18n.G("automatically set ssh remote"),
+	)
+
+	RecipeFetchCommand.Flags().BoolVarP(
+		&force,
+		i18n.G("force"),
+		i18n.G("f"),
+		false,
+		i18n.G("force re-fetch"),
+	)
+}

cli/recipe/lint.go

@@ -0,0 +1,149 @@
+package recipe
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe lint` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeLintAliases = i18n.G("l")
+
+var RecipeLintCommand = &cobra.Command{
+	// translators: `recipe lint` command
+	Use: i18n.G("lint <recipe> [flags]"),
+	// translators: Short description for `recipe lint` command
+	Short:   i18n.G("Lint a recipe"),
+	Aliases: strings.Split(recipeLintAliases, ","),
+	Args:    cobra.MinimumNArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		recipe := internal.ValidateRecipe(args, cmd.Name())
+
+		if err := recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		headers := []string{
+			i18n.G("ref"),
+			i18n.G("rule"),
+			i18n.G("severity"),
+			i18n.G("satisfied"),
+			i18n.G("skipped"),
+			i18n.G("resolve"),
+		}
+
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table.Headers(headers...)
+
+		hasError := false
+		var rows [][]string
+		var warnMessages []string
+		for level := range lint.LintRules {
+			for _, rule := range lint.LintRules[level] {
+				if onlyError && rule.Level != "error" {
+					log.Debug(i18n.G("skipping %s, does not have level \"error\"", rule.Ref))
+					continue
+				}
+
+				skipped := false
+				if rule.Skip(recipe) {
+					skipped = true
+				}
+
+				skippedOutput := "-"
+				if skipped {
+					skippedOutput = "✅"
+				}
+
+				satisfied := false
+				if !skipped {
+					ok, err := rule.Function(recipe)
+					if err != nil {
+						warnMessages = append(warnMessages, err.Error())
+					}
+
+					if !ok && rule.Level == i18n.G("error") {
+						hasError = true
+					}
+
+					if ok {
+						satisfied = true
+					}
+				}
+
+				satisfiedOutput := "✅"
+				if !satisfied {
+					satisfiedOutput = "❌"
+					if skipped {
+						satisfiedOutput = "-"
+					}
+				}
+
+				row := []string{
+					rule.Ref,
+					rule.Description,
+					rule.Level,
+					satisfiedOutput,
+					skippedOutput,
+					rule.HowToResolve,
+				}
+
+				rows = append(rows, row)
+				table.Row(row...)
+			}
+		}
+
+		if len(rows) > 0 {
+			if err := formatter.PrintTable(table); err != nil {
+				log.Fatal(err)
+			}
+
+			for _, warnMsg := range warnMessages {
+				log.Warn(warnMsg)
+			}
+
+			if hasError {
+				log.Warn(i18n.G("critical errors present in %s config", recipe.Name))
+			}
+		}
+	},
+}
+
+var (
+	onlyError bool
+)
+
+func init() {
+	RecipeLintCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		i18n.G("chaos"),
+		i18n.G("C"),
+		false,
+		i18n.G("ignore uncommitted recipes changes"),
+	)
+
+	RecipeLintCommand.Flags().BoolVarP(
+		&onlyError,
+		i18n.G("error"),
+		i18n.G("e"),
+		false,
+		i18n.G("only show errors"),
+	)
+}

cli/recipe/list.go

@@ -0,0 +1,116 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe list` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeListAliases = i18n.G("ls")
+
+var RecipeListCommand = &cobra.Command{
+	// translators: `recipe list` command
+	Use: i18n.G("list"),
+	// translators: Short description for `recipe list` command
+	Short:   i18n.G("List recipes"),
+	Aliases: strings.Split(recipeListAliases, ","),
+	Args:    cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		recipes := catl.Flatten()
+		sort.Sort(recipe.ByRecipeName(recipes))
+
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		headers := []string{
+			i18n.G("name"),
+			i18n.G("category"),
+			i18n.G("status"),
+			i18n.G("healthcheck"),
+			i18n.G("backups"),
+			i18n.G("email"),
+			i18n.G("tests"),
+			i18n.G("SSO"),
+		}
+
+		table.Headers(headers...)
+
+		var rows [][]string
+		for _, recipe := range recipes {
+			row := []string{
+				recipe.Name,
+				recipe.Category,
+				strconv.Itoa(recipe.Features.Status),
+				recipe.Features.Healthcheck,
+				recipe.Features.Backups,
+				recipe.Features.Email,
+				recipe.Features.Tests,
+				recipe.Features.SSO,
+			}
+
+			if pattern != "" {
+				if strings.Contains(recipe.Name, pattern) {
+					table.Row(row...)
+					rows = append(rows, row)
+				}
+			} else {
+				table.Row(row...)
+				rows = append(rows, row)
+			}
+		}
+
+		if len(rows) > 0 {
+			if internal.MachineReadable {
+				out, err := formatter.ToJSON(headers, rows)
+				if err != nil {
+					log.Fatal(i18n.G("unable to render to JSON: %s", err))
+				}
+				fmt.Println(out)
+				return
+			}
+
+			if err := formatter.PrintTable(table); err != nil {
+				log.Fatal(err)
+			}
+		}
+	},
+}
+
+var (
+	pattern string
+)
+
+func init() {
+	RecipeListCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+
+	RecipeListCommand.Flags().StringVarP(
+		&pattern,
+		i18n.G("pattern"),
+		i18n.G("p"),
+		"",
+		i18n.G("filter by recipe"),
+	)
+}

cli/recipe/new.go

@@ -0,0 +1,135 @@
+package recipe
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"text/template"
+
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/spf13/cobra"
+)
+
+// recipeMetadata is the recipe metadata for the README.md
+type recipeMetadata struct {
+	Name        string
+	Description string
+	Category    string
+	Status      string
+	Image       string
+	Healthcheck string
+	Backups     string
+	Email       string
+	Tests       string
+	SSO         string
+}
+
+// translators: `abra recipe new` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeNewAliases = i18n.G("n")
+
+var RecipeNewCommand = &cobra.Command{
+	// translators: `recipe new` command
+	Use:     i18n.G("new <recipe> [flags]"),
+	Aliases: strings.Split(recipeNewAliases, ","),
+	// translators: Short description for `abra recipe new` command
+	Short: i18n.G("Create a new recipe"),
+	Long:  i18n.G(`A community managed recipe template is used.`),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		recipeName := args[0]
+
+		r := recipe.Get(recipeName)
+		if _, err := os.Stat(r.Dir); !os.IsNotExist(err) {
+			log.Fatal(i18n.G("%s recipe directory already exists?", r.Dir))
+		}
+
+		url := i18n.G("%s/example.git", config.REPOS_BASE_URL)
+		if err := git.Clone(r.Dir, url); err != nil {
+			log.Fatal(err)
+		}
+
+		gitRepo := path.Join(r.Dir, ".git")
+		if err := os.RemoveAll(gitRepo); err != nil {
+			log.Fatal(err)
+		}
+		log.Debug(i18n.G("removed .git repo in %s", gitRepo))
+
+		meta := newRecipeMeta(recipeName)
+
+		for _, path := range []string{r.ReadmePath, r.SampleEnvPath} {
+			tpl, err := template.ParseFiles(path)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			var templated bytes.Buffer
+			if err := tpl.Execute(&templated, meta); err != nil {
+				log.Fatal(err)
+			}
+
+			if err := os.WriteFile(path, templated.Bytes(), 0o644); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if err := git.Init(r.Dir, true, gitName, gitEmail); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info(i18n.G("new recipe '%s' created: %s", recipeName, path.Join(r.Dir)))
+		log.Info(i18n.G("happy hacking 🎉"))
+	},
+}
+
+// newRecipeMeta creates a new recipeMetadata instance with defaults
+func newRecipeMeta(recipeName string) recipeMetadata {
+	return recipeMetadata{
+		Name:        recipeName,
+		Description: "> One line description of the recipe",
+		Category:    "Apps",
+		Status:      "0",
+		Image:       fmt.Sprintf("[`%s`](https://hub.docker.com/r/%s), 4, upstream", recipeName, recipeName),
+		Healthcheck: "No",
+		Backups:     "No",
+		Email:       "No",
+		Tests:       "No",
+		SSO:         "No",
+	}
+}
+
+var (
+	gitName  string
+	gitEmail string
+)
+
+func init() {
+	RecipeNewCommand.Flags().StringVarP(
+		&gitName,
+		i18n.G("git-name"),
+		i18n.G("N"),
+		"",
+		i18n.G("Git (user) name to do commits with"),
+	)
+
+	RecipeNewCommand.Flags().StringVarP(
+		&gitEmail,
+		i18n.G("git-email"),
+		i18n.G("e"),
+		"",
+		i18n.G("Git email name to do commits with"),
+	)
+}

cli/recipe/recipe.go

@@ -0,0 +1,30 @@
+package recipe
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe` aliases. use a comma separated list of aliases
+// with no spaces in between
+var recipeAliases = i18n.G("r")
+
+// RecipeCommand defines all recipe related sub-commands.
+var RecipeCommand = &cobra.Command{
+	// translators: `recipe` command group
+	Use:     i18n.G("recipe [cmd] [args] [flags]"),
+	Aliases: strings.Split(recipeAliases, ","),
+	// translators: Short description for `recipe` command group
+	Short: i18n.G("Manage recipes"),
+	Long: i18n.G(`A recipe is a blueprint for an app.
+
+It is a bunch of config files which describe how to deploy and maintain an app.
+Recipes are maintained by the Co-op Cloud community and you can use Abra to
+read them, deploy them and create apps for you.
+
+Anyone who uses a recipe can become a maintainer. Maintainers typically make
+sure the recipe is in good working order and the config upgraded in a timely
+manner.`),
+}

cli/recipe/release.go

@@ -0,0 +1,685 @@
+package recipe
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/distribution/reference"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe release` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeReleaseAliases = i18n.G("rl")
+
+var RecipeReleaseCommand = &cobra.Command{
+	// translators: `recipe release` command
+	Use:     i18n.G("release <recipe> [version] [flags]"),
+	Aliases: strings.Split(recipeReleaseAliases, ","),
+	// translators: Short description for `recipe release` command
+	Short: i18n.G("Release a new recipe version"),
+	Long: i18n.G(`Create a new version of a recipe.
+
+These versions are then published on the Co-op Cloud recipe catalogue. These
+versions take the following form:
+
+    a.b.c+x.y.z
+
+Where the "a.b.c" part is a semantic version determined by the maintainer. The
+"x.y.z" part is the image tag of the recipe "app" service (the main container
+which contains the software to be used, by naming convention).
+
+We maintain a semantic versioning scheme ("a.b.c") alongside the recipe
+versioning scheme ("x.y.z") in order to maximise the chances that the nature of
+recipe updates are properly communicated. I.e. developers of an app might
+publish a minor version but that might lead to changes in the recipe which are
+major and therefore require intervention while doing the upgrade work.
+
+Publish your new release to git.coopcloud.tech with "--publish/-p". This
+requires that you have permission to git push to these repositories and have
+your SSH keys configured on your account. Enable ssh-agent and make sure to add
+your private key and enter your passphrase beforehand.
+
+    eval ` + "`ssh-agent`" + `
+    ssh-add ~/.ssh/<my-ssh-private-key-for-git-coopcloud-tech>`),
+	Example: `  # publish release
+  eval ` + "`ssh-agent`" + `
+  ssh-add ~/.ssh/id_ed25519
+  abra recipe release gitea -p`,
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.RecipeNameComplete()
+		case 1:
+			return autocomplete.RecipeVersionComplete(args[0])
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		recipe := internal.ValidateRecipe(args, cmd.Name())
+
+		imagesTmp, err := GetImageVersions(recipe)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		mainApp, err := internal.GetMainAppImage(recipe)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		mainAppVersion := imagesTmp[mainApp]
+		if mainAppVersion == "" {
+			log.Fatal(i18n.G("main app service version for %s is empty?", recipe.Name))
+		}
+
+		var tagString string
+		if len(args) == 2 {
+			tagString = args[1]
+		}
+
+		if tagString != "" {
+			if _, err := tagcmp.Parse(tagString); err != nil {
+				log.Fatal(i18n.G("cannot parse %s, invalid tag specified?", tagString))
+			}
+		}
+
+		if (internal.Major || internal.Minor || internal.Patch) && tagString != "" {
+			log.Fatal(i18n.G("cannot specify tag and bump type at the same time"))
+		}
+
+		repo, err := git.PlainOpen(recipe.Dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		preCommitHead, err := repo.Head()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if tagString != "" {
+			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+				if cleanErr := cleanTag(recipe, tagString); cleanErr != nil {
+					log.Fatal(cleanErr)
+				}
+				if cleanErr := cleanCommit(recipe, preCommitHead); cleanErr != nil {
+					log.Fatal(cleanErr)
+				}
+				log.Fatal(err)
+			}
+		}
+
+		tags, err := recipe.Tags()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		labelVersion, err := getLabelVersion(recipe, false)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, tag := range tags {
+			previousTagLeftHand := strings.Split(tag, "+")[0]
+			newTagStringLeftHand := strings.Split(labelVersion, "+")[0]
+			if previousTagLeftHand == newTagStringLeftHand {
+				log.Fatal(i18n.G("%s+... conflicts with a previous release: %s", newTagStringLeftHand, tag))
+			}
+		}
+
+		if tagString == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+			tagString = labelVersion
+		}
+
+		isClean, err := gitPkg.IsClean(recipe.Dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if !isClean {
+			log.Info(i18n.G("%s currently has these unstaged changes 👇", recipe.Name))
+			if err := gitPkg.DiffUnstaged(recipe.Dir); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if len(tags) > 0 {
+			log.Warn(i18n.G("previous git tags detected, assuming new semver release"))
+
+			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
+				if cleanErr := cleanTag(recipe, tagString); cleanErr != nil {
+					log.Fatal(cleanErr)
+				}
+				if cleanErr := cleanCommit(recipe, preCommitHead); cleanErr != nil {
+					log.Fatal(cleanErr)
+				}
+				log.Fatal(err)
+			}
+		} else {
+			log.Warn(i18n.G("no tag specified and no previous tag available for %s, assuming initial release", recipe.Name))
+
+			if err := createReleaseFromTag(recipe, tagString, mainAppVersion); err != nil {
+				if cleanErr := cleanTag(recipe, tagString); cleanErr != nil {
+					log.Fatal(cleanErr)
+				}
+				if cleanErr := cleanCommit(recipe, preCommitHead); cleanErr != nil {
+					log.Fatal(cleanErr)
+				}
+				log.Fatal(err)
+			}
+		}
+
+		return
+	},
+}
+
+// GetImageVersions retrieves image versions for a recipe
+func GetImageVersions(recipe recipe.Recipe) (map[string]string, error) {
+	services := make(map[string]string)
+
+	config, err := recipe.GetComposeConfig(nil)
+	if err != nil {
+		return nil, err
+	}
+	missingTag := false
+	for _, service := range config.Services {
+		if service.Image == "" {
+			continue
+		}
+
+		img, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			return services, err
+		}
+
+		path := reference.Path(img)
+
+		path = formatter.StripTagMeta(path)
+
+		var tag string
+		switch img.(type) {
+		case reference.NamedTagged:
+			tag = img.(reference.NamedTagged).Tag()
+		case reference.Named:
+			if service.Name == "app" {
+				missingTag = true
+			}
+			continue
+		}
+
+		services[path] = tag
+	}
+
+	if missingTag {
+		return services, errors.New(i18n.G("app service is missing image tag?"))
+	}
+
+	return services, nil
+}
+
+// createReleaseFromTag creates a new release based on a supplied recipe version string
+func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string) error {
+	var err error
+
+	repo, err := git.PlainOpen(recipe.Dir)
+	if err != nil {
+		return err
+	}
+
+	tag, err := tagcmp.Parse(tagString)
+	if err != nil {
+		return err
+	}
+
+	if tag.MissingMinor {
+		tag.Minor = "0"
+		tag.MissingMinor = false
+	}
+
+	if tag.MissingPatch {
+		tag.Patch = "0"
+		tag.MissingPatch = false
+	}
+
+	if tagString == "" {
+		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
+	}
+
+	if err := addReleaseNotes(recipe, tagString); err != nil {
+		return errors.New(i18n.G("failed to add release notes: %s", err.Error()))
+	}
+
+	if err := commitRelease(recipe, tagString); err != nil {
+		return errors.New(i18n.G("failed to commit changes: %s", err.Error()))
+	}
+
+	if err := tagRelease(tagString, repo); err != nil {
+		return errors.New(i18n.G("failed to tag release: %s", err.Error()))
+	}
+
+	if err := pushRelease(recipe, tagString); err != nil {
+		return errors.New(i18n.G("failed to publish new release: %s", err.Error()))
+	}
+
+	return nil
+}
+
+// btoi converts a boolean value into an integer
+func btoi(b bool) int {
+	if b {
+		return 1
+	}
+
+	return 0
+}
+
+// getTagCreateOptions constructs git tag create options
+func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
+	msg := i18n.G("chore: publish %s release", tag)
+	return git.CreateTagOptions{Message: msg}, nil
+}
+
+// addReleaseNotes checks if the release/next release note exists and moves the
+// file to release/<tag>.
+func addReleaseNotes(recipe recipe.Recipe, tag string) error {
+	releaseDir := path.Join(recipe.Dir, "release")
+	if _, err := os.Stat(releaseDir); errors.Is(err, os.ErrNotExist) {
+		if err := os.Mkdir(releaseDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	tagReleaseNotePath := path.Join(releaseDir, tag)
+	if _, err := os.Stat(tagReleaseNotePath); err == nil {
+		// Release note for current tag already exist exists.
+		return nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	var addNextAsReleaseNotes bool
+
+	nextReleaseNotePath := path.Join(releaseDir, "next")
+	if _, err := os.Stat(nextReleaseNotePath); err == nil {
+		// release/next note exists. Move it to release/<tag>
+		if internal.Dry {
+			log.Debug(i18n.G("dry run: move release note from 'next' to %s", tag))
+			return nil
+		}
+
+		if !internal.NoInput {
+			prompt := &survey.Confirm{
+				Message: i18n.G("use release note in release/next?"),
+			}
+
+			if err := survey.AskOne(prompt, &addNextAsReleaseNotes); err != nil {
+				return err
+			}
+
+			if !addNextAsReleaseNotes {
+				return nil
+			}
+		}
+
+		if err := os.Rename(nextReleaseNotePath, tagReleaseNotePath); err != nil {
+			return err
+		}
+
+		if err := gitPkg.Add(recipe.Dir, path.Join("release", "next"), internal.Dry); err != nil {
+			return err
+		}
+
+		if err := gitPkg.Add(recipe.Dir, path.Join("release", tag), internal.Dry); err != nil {
+			return err
+		}
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	// NOTE(d1): No release note exists for the current release. Or, we've
+	// already used release/next as the release note
+	if internal.NoInput || addNextAsReleaseNotes {
+		return nil
+	}
+
+	prompt := &survey.Input{
+		Message: i18n.G("add release note? (leave empty to skip)"),
+	}
+
+	var releaseNote string
+	if err := survey.AskOne(prompt, &releaseNote); err != nil {
+		return err
+	}
+
+	if releaseNote == "" {
+		return nil
+	}
+
+	if err := os.WriteFile(tagReleaseNotePath, []byte(releaseNote), 0o644); err != nil {
+		return err
+	}
+
+	if err := gitPkg.Add(recipe.Dir, path.Join("release", tag), internal.Dry); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func commitRelease(recipe recipe.Recipe, tag string) error {
+	if internal.Dry {
+		log.Debug(i18n.G("dry run: no changes committed"))
+		return nil
+	}
+
+	isClean, err := gitPkg.IsClean(recipe.Dir)
+	if err != nil {
+		return err
+	}
+
+	if isClean {
+		if !internal.Dry {
+			return errors.New(i18n.G("no changes discovered in %s, nothing to publish?", recipe.Dir))
+		}
+	}
+
+	msg := fmt.Sprintf("chore: publish %s release", tag)
+	if err := gitPkg.Commit(recipe.Dir, msg, internal.Dry); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func tagRelease(tagString string, repo *git.Repository) error {
+	if internal.Dry {
+		log.Debug(i18n.G("dry run: no git tag created (%s)", tagString))
+		return nil
+	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return err
+	}
+
+	createTagOptions, err := getTagCreateOptions(tagString)
+	if err != nil {
+		return err
+	}
+
+	_, err = repo.CreateTag(tagString, head.Hash(), &createTagOptions)
+	if err != nil {
+		return err
+	}
+
+	hash := formatter.SmallSHA(head.Hash().String())
+	log.Debug(i18n.G("created tag %s at %s", tagString, hash))
+
+	return nil
+}
+
+func pushRelease(recipe recipe.Recipe, tagString string) error {
+	if internal.Dry {
+		log.Info(i18n.G("dry run: no changes published"))
+		return nil
+	}
+
+	if !publish && !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: i18n.G("publish new release?"),
+		}
+
+		if err := survey.AskOne(prompt, &publish); err != nil {
+			return err
+		}
+	}
+
+	if publish {
+		if os.Getenv("SSH_AUTH_SOCK") == "" {
+			return errors.New(i18n.G("ssh-agent not found. see \"abra recipe release --help\" and try again"))
+		}
+
+		if err := recipe.Push(internal.Dry); err != nil {
+			return err
+		}
+
+		url := fmt.Sprintf("%s/src/tag/%s", recipe.GitURL, tagString)
+		log.Info(i18n.G("new release published: %s", url))
+	} else {
+		log.Info(i18n.G("no -p/--publish passed, not publishing"))
+	}
+
+	return nil
+}
+
+func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recipe.Recipe, tags []string) error {
+	repo, err := git.PlainOpen(recipe.Dir)
+	if err != nil {
+		return err
+	}
+
+	bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+	if bumpType != 0 {
+		if (bumpType & (bumpType - 1)) != 0 {
+			return errors.New(i18n.G("you can only use one of: --major, --minor, --patch"))
+		}
+	}
+
+	var lastGitTag tagcmp.Tag
+	for _, tag := range tags {
+		parsed, err := tagcmp.Parse(tag)
+		if err != nil {
+			return err
+		}
+
+		if (lastGitTag == tagcmp.Tag{}) {
+			lastGitTag = parsed
+		} else if parsed.IsGreaterThan(lastGitTag) {
+			lastGitTag = parsed
+		}
+	}
+
+	newTag := lastGitTag
+	if internal.Patch {
+		now, err := strconv.Atoi(newTag.Patch)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = strconv.Itoa(now + 1)
+	} else if internal.Minor {
+		now, err := strconv.Atoi(newTag.Minor)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = "0"
+		newTag.Minor = strconv.Itoa(now + 1)
+	} else if internal.Major {
+		now, err := strconv.Atoi(newTag.Major)
+		if err != nil {
+			return err
+		}
+
+		newTag.Patch = "0"
+		newTag.Minor = "0"
+		newTag.Major = strconv.Itoa(now + 1)
+	}
+
+	if internal.Major || internal.Minor || internal.Patch {
+		newTag.Metadata = mainAppVersion
+		tagString = newTag.String()
+	}
+
+	if lastGitTag.String() == tagString {
+		return errors.New(i18n.G("latest git tag (%s) and synced label (%s) are the same?", lastGitTag, tagString))
+	}
+
+	if !internal.NoInput {
+		prompt := &survey.Confirm{
+			Message: i18n.G("current: %s, new: %s, correct?", lastGitTag, tagString),
+		}
+
+		var ok bool
+		if err := survey.AskOne(prompt, &ok); err != nil {
+			return err
+		}
+
+		if !ok {
+			return errors.New(i18n.G("exiting as requested"))
+		}
+	}
+
+	if err := addReleaseNotes(recipe, tagString); err != nil {
+		return errors.New(i18n.G("failed to add release notes: %s", err.Error()))
+	}
+
+	if err := commitRelease(recipe, tagString); err != nil {
+		return errors.New(i18n.G("failed to commit changes: %s", err.Error()))
+	}
+
+	if err := tagRelease(tagString, repo); err != nil {
+		return errors.New(i18n.G("failed to tag release: %s", err.Error()))
+	}
+
+	if err := pushRelease(recipe, tagString); err != nil {
+		return errors.New(i18n.G("failed to publish new release: %s", err.Error()))
+	}
+
+	return nil
+}
+
+// cleanCommit soft removes the latest release commit. No change are lost the
+// the commit itself is removed. This is the equivalent of `git reset HEAD~1`.
+func cleanCommit(recipe recipe.Recipe, head *plumbing.Reference) error {
+	repo, err := git.PlainOpen(recipe.Dir)
+	if err != nil {
+		return errors.New(i18n.G("unable to open repo in %s: %s", recipe.Dir, err))
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return errors.New(i18n.G("unable to open work tree in %s: %s", recipe.Dir, err))
+	}
+
+	opts := &git.ResetOptions{Commit: head.Hash(), Mode: git.MixedReset}
+	if err := worktree.Reset(opts); err != nil {
+		return errors.New(i18n.G("unable to soft reset %s: %s", recipe.Dir, err))
+	}
+
+	log.Debug(i18n.G("removed freshly created commit"))
+
+	return nil
+}
+
+// cleanTag removes a freshly created tag
+func cleanTag(recipe recipe.Recipe, tag string) error {
+	repo, err := git.PlainOpen(recipe.Dir)
+	if err != nil {
+		return errors.New(i18n.G("unable to open repo in %s: %s", recipe.Dir, err))
+	}
+
+	if err := repo.DeleteTag(tag); err != nil {
+		if !strings.Contains(err.Error(), "not found") {
+			return errors.New(i18n.G("unable to delete tag %s: %s", tag, err))
+		}
+	}
+
+	log.Debug(i18n.G("removed freshly created tag %s", tag))
+
+	return nil
+}
+
+func getLabelVersion(recipe recipe.Recipe, prompt bool) (string, error) {
+	initTag, err := recipe.GetVersionLabelLocal()
+	if err != nil {
+		return "", err
+	}
+
+	if initTag == "" {
+		return "", errors.New(i18n.G("unable to read version for %s from synced label. Did you try running \"abra recipe sync %s\" already?", recipe.Name, recipe.Name))
+	}
+
+	log.Warn(i18n.G("discovered %s as currently synced recipe label", initTag))
+
+	if prompt && !internal.NoInput {
+		var response bool
+		prompt := &survey.Confirm{Message: i18n.G("use %s as the new version?", initTag)}
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return "", err
+		}
+
+		if !response {
+			return "", errors.New(i18n.G("please fix your synced label for %s and re-run this command", recipe.Name))
+		}
+	}
+
+	return initTag, nil
+}
+
+var (
+	publish bool
+)
+
+func init() {
+	RecipeReleaseCommand.Flags().BoolVarP(
+		&internal.Dry,
+		i18n.G("dry-run"),
+		i18n.G("r"),
+		false,
+		i18n.G("report changes that would be made"),
+	)
+
+	RecipeReleaseCommand.Flags().BoolVarP(
+		&internal.Major,
+		i18n.G("major"),
+		i18n.G("x"),
+		false,
+		i18n.G("increase the major part of the version"),
+	)
+
+	RecipeReleaseCommand.Flags().BoolVarP(
+		&internal.Minor,
+		i18n.G("minor"),
+		i18n.G("y"),
+		false,
+		i18n.G("increase the minor part of the version"),
+	)
+
+	RecipeReleaseCommand.Flags().BoolVarP(
+		&internal.Patch,
+		i18n.G("patch"),
+		i18n.G("z"),
+		false,
+		i18n.G("increase the patch part of the version"),
+	)
+
+	RecipeReleaseCommand.Flags().BoolVarP(
+		&publish,
+		i18n.G("publish"),
+		i18n.G("p"),
+		false,
+		i18n.G("publish changes to git.coopcloud.tech"),
+	)
+}

cli/recipe/reset.go

@@ -0,0 +1,55 @@
+package recipe
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/go-git/go-git/v5"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe reset` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeResetAliases = i18n.G("rs")
+
+var RecipeResetCommand = &cobra.Command{
+	// translators: `recipe reset` command
+	Use:     i18n.G("reset <recipe> [flags]"),
+	Aliases: strings.Split(recipeResetAliases, ","),
+	// translators: Short description for `recipe reset` command
+	Short: i18n.G("Remove all unstaged changes from recipe config"),
+	Long:  i18n.G("WARNING: this will delete your changes. Be Careful."),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		r := internal.ValidateRecipe(args, cmd.Name())
+
+		repo, err := git.PlainOpen(r.Dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		ref, err := repo.Head()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		worktree, err := repo.Worktree()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}
+		if err := worktree.Reset(opts); err != nil {
+			log.Fatal(err)
+		}
+	},
+}

cli/recipe/sync.go

@@ -0,0 +1,313 @@
+package recipe
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/spf13/cobra"
+)
+
+// Errors
+var emptyVersionsInCatalogue = errors.New(i18n.G("catalogue versions list is unexpectedly empty"))
+
+// translators: `abra recipe reset` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeSyncAliases = i18n.G("s")
+
+var RecipeSyncCommand = &cobra.Command{
+	// translators: `recipe sync` command
+	Use:     i18n.G("sync <recipe> [version] [flags]"),
+	Aliases: strings.Split(recipeSyncAliases, ","),
+	// translators: Short description for `recipe sync` command
+	Short: i18n.G("Sync recipe version label"),
+	Long: i18n.G(`Generate labels for the main recipe service.
+
+By convention, the service named "app" using the following format:
+
+    coop-cloud.${STACK_NAME}.version=<version>
+
+Where [version] can be specifed on the command-line or Abra can attempt to
+auto-generate it for you. The <recipe> configuration will be updated on the
+local file system.`),
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.RecipeNameComplete()
+		case 1:
+			return autocomplete.RecipeVersionComplete(args[0])
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		recipe := internal.ValidateRecipe(args, cmd.Name())
+
+		mainApp, err := internal.GetMainAppImage(recipe)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		imagesTmp, err := GetImageVersions(recipe)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		mainAppVersion := imagesTmp[mainApp]
+
+		tags, err := recipe.Tags()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var nextTag string
+		if len(args) == 2 {
+			nextTag = args[1]
+		}
+
+		if len(tags) == 0 && nextTag == "" {
+			log.Warn(i18n.G("no git tags found for %s", recipe.Name))
+			if internal.NoInput {
+				log.Fatal(i18n.G("unable to continue, input required for initial version"))
+			}
+			fmt.Println(i18n.G(`
+The following options are two types of initial semantic version that you can
+pick for %s that will be published in the recipe catalogue. This follows the
+semver convention (more on https://semver.org), here is a short cheatsheet
+
+    0.1.0: development release, still hacking. when you make a major upgrade
+           you increment the "y" part (i.e. 0.1.0 -> 0.2.0) and only move to
+           using the "x" part when things are stable.
+
+    1.0.0: public release, assumed to be working. you already have a stable
+           and reliable deployment of this app and feel relatively confident
+           about it.
+
+If you want people to be able alpha test your current config for %s but don't
+think it is quite reliable, go with 0.1.0 and people will know that things are
+likely to change.
+
+`, recipe.Name, recipe.Name))
+			var chosenVersion string
+			edPrompt := &survey.Select{
+				Message: i18n.G("which version do you want to begin with?"),
+				Options: []string{"0.1.0", "1.0.0"},
+			}
+
+			if err := survey.AskOne(edPrompt, &chosenVersion); err != nil {
+				log.Fatal(err)
+			}
+
+			nextTag = fmt.Sprintf("%s+%s", chosenVersion, mainAppVersion)
+		}
+
+		if nextTag == "" && (!internal.Major && !internal.Minor && !internal.Patch) {
+			var changeOverview string
+
+			catl, err := recipePkg.ReadRecipeCatalogue(false)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			changesTable, err := formatter.CreateTable()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			latestRelease := tags[len(tags)-1]
+			latestRecipeVersion, err := getLatestVersion(recipe, catl)
+			if err != nil && err != emptyVersionsInCatalogue {
+				log.Fatal(err)
+			}
+			changesTable.Headers(i18n.G("SERVICE"), latestRelease, i18n.G("PROPOSED CHANGES"))
+
+			allRecipeVersions := catl[recipe.Name].Versions
+			for _, recipeVersion := range allRecipeVersions {
+				if serviceVersions, ok := recipeVersion[latestRecipeVersion]; ok {
+					for serviceName := range serviceVersions {
+						serviceMeta := serviceVersions[serviceName]
+
+						existingImageTag := fmt.Sprintf("%s:%s", serviceMeta.Image, serviceMeta.Tag)
+						newImageTag := fmt.Sprintf("%s:%s", serviceMeta.Image, imagesTmp[serviceMeta.Image])
+
+						if existingImageTag == newImageTag {
+							continue
+						}
+
+						changesTable.Row([]string{serviceName, existingImageTag, newImageTag}...)
+					}
+				}
+			}
+
+			changeOverview = changesTable.Render()
+
+			if err := internal.PromptBumpType("", latestRelease, changeOverview); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if nextTag == "" {
+			repo, err := git.PlainOpen(recipe.Dir)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			var lastGitTag tagcmp.Tag
+			iter, err := repo.Tags()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if err := iter.ForEach(func(ref *plumbing.Reference) error {
+				obj, err := repo.TagObject(ref.Hash())
+				if err != nil {
+					log.Fatal(i18n.G("tag at commit %s is unannotated or otherwise broken", ref.Hash()))
+					return err
+				}
+
+				tagcmpTag, err := tagcmp.Parse(obj.Name)
+				if err != nil {
+					return err
+				}
+
+				if (lastGitTag == tagcmp.Tag{}) {
+					lastGitTag = tagcmpTag
+				} else if tagcmpTag.IsGreaterThan(lastGitTag) {
+					lastGitTag = tagcmpTag
+				}
+
+				return nil
+			}); err != nil {
+				log.Fatal(err)
+			}
+
+			// bumpType is used to decide what part of the tag should be incremented
+			bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+			if bumpType != 0 {
+				// a bitwise check if the number is a power of 2
+				if (bumpType & (bumpType - 1)) != 0 {
+					log.Fatal(i18n.G("you can only use one version flag: --major, --minor or --patch"))
+				}
+			}
+
+			newTag := lastGitTag
+			if bumpType > 0 {
+				if internal.Patch {
+					now, err := strconv.Atoi(newTag.Patch)
+					if err != nil {
+						log.Fatal(err)
+					}
+
+					newTag.Patch = strconv.Itoa(now + 1)
+				} else if internal.Minor {
+					now, err := strconv.Atoi(newTag.Minor)
+					if err != nil {
+						log.Fatal(err)
+					}
+
+					newTag.Patch = "0"
+					newTag.Minor = strconv.Itoa(now + 1)
+				} else if internal.Major {
+					now, err := strconv.Atoi(newTag.Major)
+					if err != nil {
+						log.Fatal(err)
+					}
+
+					newTag.Patch = "0"
+					newTag.Minor = "0"
+					newTag.Major = strconv.Itoa(now + 1)
+				}
+			}
+
+			newTag.Metadata = mainAppVersion
+			log.Debug(i18n.G("choosing %s as new version for %s", newTag.String(), recipe.Name))
+			nextTag = newTag.String()
+		}
+
+		if _, err := tagcmp.Parse(nextTag); err != nil {
+			log.Fatal(i18n.G("invalid version %s specified", nextTag))
+		}
+
+		mainService := "app"
+		label := i18n.G("coop-cloud.${STACK_NAME}.version=%s", nextTag)
+		if !internal.Dry {
+			if err := recipe.UpdateLabel("compose.y*ml", mainService, label); err != nil {
+				log.Fatal(err)
+			}
+		} else {
+			log.Info(i18n.G("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name))
+		}
+
+		isClean, err := gitPkg.IsClean(recipe.Dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if !isClean {
+			log.Info(i18n.G("%s currently has these unstaged changes 👇", recipe.Name))
+			if err := gitPkg.DiffUnstaged(recipe.Dir); err != nil {
+				log.Fatal(err)
+			}
+		}
+	},
+}
+
+func init() {
+	RecipeSyncCommand.Flags().BoolVarP(
+		&internal.Dry,
+		i18n.G("dry-run"),
+		i18n.G("r"),
+		false,
+		i18n.G("report changes that would be made"),
+	)
+
+	RecipeSyncCommand.Flags().BoolVarP(
+		&internal.Major,
+		i18n.G("major"),
+		i18n.G("x"),
+		false,
+		i18n.G("increase the major part of the version"),
+	)
+
+	RecipeSyncCommand.Flags().BoolVarP(
+		&internal.Minor,
+		i18n.G("minor"),
+		i18n.G("y"),
+		false,
+		i18n.G("increase the minor part of the version"),
+	)
+
+	RecipeSyncCommand.Flags().BoolVarP(
+		&internal.Patch,
+		i18n.G("patch"),
+		i18n.G("z"),
+		false,
+		i18n.G("increase the patch part of the version"),
+	)
+}
+
+func getLatestVersion(recipe recipePkg.Recipe, catl recipePkg.RecipeCatalogue) (string, error) {
+	versions, err := recipePkg.GetRecipeCatalogueVersions(recipe.Name, catl)
+	if err != nil {
+		return "", err
+	}
+	if len(versions) > 0 {
+		return versions[len(versions)-1], nil
+	}
+	return "", emptyVersionsInCatalogue
+}

cli/recipe/sync_test.go

@@ -0,0 +1,33 @@
+package recipe
+
+import (
+	"testing"
+
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetLatestVersionReturnsErrorWhenVersionsIsEmpty(t *testing.T) {
+	recipe := recipePkg.Recipe{}
+	catalogue := recipePkg.RecipeCatalogue{}
+	_, err := getLatestVersion(recipe, catalogue)
+	assert.Equal(t, err, emptyVersionsInCatalogue)
+}
+
+func TestGetLatestVersionReturnsLastVersion(t *testing.T) {
+	recipe := recipePkg.Recipe{
+		Name: "test",
+	}
+	versions := []map[string]map[string]recipePkg.ServiceMeta{
+		make(map[string]map[string]recipePkg.ServiceMeta),
+		make(map[string]map[string]recipePkg.ServiceMeta),
+	}
+	versions[0]["0.0.3"] = make(map[string]recipePkg.ServiceMeta)
+	versions[1]["0.0.2"] = make(map[string]recipePkg.ServiceMeta)
+	catalogue := make(recipePkg.RecipeCatalogue)
+	catalogue["test"] = recipePkg.RecipeMeta{
+		Versions: versions,
+	}
+	version, _ := getLatestVersion(recipe, catalogue)
+	assert.Equal(t, version, "0.0.3")
+}

cli/recipe/upgrade.go

@@ -0,0 +1,386 @@
+package recipe
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/distribution/reference"
+	"github.com/spf13/cobra"
+)
+
+type imgPin struct {
+	image   string
+	version tagcmp.Tag
+}
+
+// anUpgrade represents a single service upgrade (as within a recipe), and the
+// list of tags that it can be upgraded to, for serialization purposes.
+type anUpgrade struct {
+	Service     string   `json:"service"`
+	Image       string   `json:"image"`
+	Tag         string   `json:"tag"`
+	UpgradeTags []string `json:"upgrades"`
+}
+
+// translators: `abra recipe upgrade` aliases. use a comma separated list of
+// aliases with no spaces in between
+var recipeUpgradeAliases = i18n.G("u")
+
+var RecipeUpgradeCommand = &cobra.Command{
+	// translators: `recipe upgrade` command
+	Use:     i18n.G("upgrade <recipe> [flags]"),
+	Aliases: strings.Split(recipeUpgradeAliases, ","),
+	// translators: Short description for `recipe upgrade` command
+	Short: i18n.G("Upgrade recipe image tags"),
+	Long: i18n.G(`Upgrade a given <recipe> configuration.
+
+It will update the relevant compose file tags on the local file system.
+
+Some image tags cannot be parsed because they do not follow some sort of
+semver-like convention. In this case, all possible tags will be listed and it
+is up to the end-user to decide.
+
+The command is interactive and will show a select input which allows you to
+make a seclection. Use the "?" key to see more help on navigating this
+interface.`),
+	Args: cobra.RangeArgs(0, 1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		recipe := internal.ValidateRecipe(args, cmd.Name())
+
+		if err := recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		bumpType := btoi(internal.Major)*4 + btoi(internal.Minor)*2 + btoi(internal.Patch)
+		if bumpType != 0 {
+			// a bitwise check if the number is a power of 2
+			if (bumpType & (bumpType - 1)) != 0 {
+				log.Fatal(i18n.G("you can only use one of: --major, --minor, --patch."))
+			}
+		}
+
+		if internal.MachineReadable {
+			// -m implies -n in this case
+			internal.NoInput = true
+		}
+
+		upgradeList := make(map[string]anUpgrade)
+
+		// check for versions file and load pinned versions
+		versionsPresent := false
+		versionsPath := path.Join(recipe.Dir, "versions")
+		servicePins := make(map[string]imgPin)
+		if _, err := os.Stat(versionsPath); err == nil {
+			log.Debug(i18n.G("found versions file for %s", recipe.Name))
+			file, err := os.Open(versionsPath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := scanner.Text()
+				splitLine := strings.Split(line, " ")
+				if splitLine[0] != "pin" || len(splitLine) != 3 {
+					log.Fatal(i18n.G("malformed version pin specification: %s", line))
+				}
+				pinSlice := strings.Split(splitLine[2], ":")
+				pinTag, err := tagcmp.Parse(pinSlice[1])
+				if err != nil {
+					log.Fatal(err)
+				}
+				pin := imgPin{
+					image:   pinSlice[0],
+					version: pinTag,
+				}
+				servicePins[splitLine[1]] = pin
+			}
+			if err := scanner.Err(); err != nil {
+				log.Error(err)
+			}
+			versionsPresent = true
+		} else {
+			log.Debug(i18n.G("did not find versions file for %s", recipe.Name))
+		}
+
+		config, err := recipe.GetComposeConfig(nil)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, service := range config.Services {
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			regVersions, err := client.GetRegistryTags(img)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			image := reference.Path(img)
+			log.Debug(i18n.G("retrieved %s from remote registry for %s", regVersions, image))
+			image = formatter.StripTagMeta(image)
+
+			switch img.(type) {
+			case reference.NamedTagged:
+				if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+					log.Debug(i18n.G("%s not considered semver-like", img.(reference.NamedTagged).Tag()))
+				}
+			default:
+				log.Warn(i18n.G("unable to read tag for image %s, is it missing? skipping upgrade for %s", image, service.Name))
+				continue
+			}
+
+			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
+			if err != nil {
+				log.Warn(i18n.G("unable to parse %s, error was: %s, skipping upgrade for %s", image, err.Error(), service.Name))
+				continue
+			}
+
+			log.Debug(i18n.G("parsed %s for %s", tag, service.Name))
+
+			var compatible []tagcmp.Tag
+			for _, regVersion := range regVersions {
+				other, err := tagcmp.Parse(regVersion)
+				if err != nil {
+					continue // skip tags that cannot be parsed
+				}
+
+				if tag.IsCompatible(other) && tag.IsLessThan(other) && !tag.Equals(other) {
+					compatible = append(compatible, other)
+				}
+			}
+
+			log.Debug(i18n.G("detected potential upgradable tags %s for %s", compatible, service.Name))
+
+			sort.Sort(tagcmp.ByTagDesc(compatible))
+
+			if len(compatible) == 0 && !allTags {
+				log.Info(i18n.G("no new versions available for %s, assuming %s is the latest (use -a/--all-tags to see all anyway)", image, tag))
+				continue // skip on to the next tag and don't update any compose files
+			}
+
+			catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name, internal.Offline)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			compatibleStrings := []string{"skip"}
+			for _, compat := range compatible {
+				skip := false
+				for _, catlVersion := range catlVersions {
+					if compat.String() == catlVersion {
+						skip = true
+					}
+				}
+				if !skip {
+					compatibleStrings = append(compatibleStrings, compat.String())
+				}
+			}
+
+			log.Debug(i18n.G("detected compatible upgradable tags %s for %s", compatibleStrings, service.Name))
+
+			var upgradeTag string
+			_, ok := servicePins[service.Name]
+			if versionsPresent && ok {
+				pinnedTag := servicePins[service.Name].version
+				if tag.IsLessThan(pinnedTag) {
+					pinnedTagString := pinnedTag.String()
+					contains := false
+					for _, v := range compatible {
+						if pinnedTag.IsUpgradeCompatible(v) {
+							contains = true
+							upgradeTag = v.String()
+							break
+						}
+					}
+					if contains {
+						log.Info(i18n.G("upgrading service %s from %s to %s (pinned tag: %s)", service.Name, tag.String(), upgradeTag, pinnedTagString))
+					} else {
+						log.Info(i18n.G("service %s, image %s pinned to %s, no compatible upgrade found", service.Name, servicePins[service.Name].image, pinnedTagString))
+						continue
+					}
+				} else {
+					log.Fatal(i18n.G("service %s is at version %s, but pinned to %s, please correct your compose.yml file manually!", service.Name, tag.String(), pinnedTag.String()))
+					continue
+				}
+			} else {
+				if bumpType != 0 {
+					for _, upTag := range compatible {
+						upElement, err := tag.UpgradeDelta(upTag)
+						if err != nil {
+							return
+						}
+						delta := upElement.UpgradeType()
+						if delta <= bumpType {
+							upgradeTag = upTag.String()
+							break
+						}
+					}
+					if upgradeTag == "" {
+						log.Warn(i18n.G("not upgrading from %s to %s for %s, because the upgrade type is more serious than what user wants", tag.String(), compatible[0].String(), image))
+						continue
+					}
+				} else {
+					msg := i18n.G("upgrade to which tag? (service: %s, image: %s, tag: %s)", service.Name, image, tag)
+					if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) || allTags {
+						tag := img.(reference.NamedTagged).Tag()
+						if !allTags {
+							log.Warn(i18n.G("unable to determine versioning semantics of %s, listing all tags", tag))
+						}
+						msg = i18n.G("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
+						compatibleStrings = []string{"skip"}
+						for _, regVersion := range regVersions {
+							compatibleStrings = append(compatibleStrings, regVersion)
+						}
+					}
+
+					// there is always at least the item "skip" in compatibleStrings (a list of
+					// possible upgradable tags) and at least one other tag.
+					upgradableTags := compatibleStrings[1:]
+					upgrade := anUpgrade{
+						Service:     service.Name,
+						Image:       image,
+						Tag:         tag.String(),
+						UpgradeTags: make([]string, len(upgradableTags)),
+					}
+
+					for n, s := range upgradableTags {
+						var sb strings.Builder
+						if _, err := sb.WriteString(s); err != nil {
+						}
+						upgrade.UpgradeTags[n] = sb.String()
+					}
+
+					upgradeList[upgrade.Service] = upgrade
+
+					if internal.NoInput {
+						upgradeTag = "skip"
+					} else {
+						prompt := &survey.Select{
+							Message: msg,
+							Help:    i18n.G("enter / return to confirm, choose 'skip' to not upgrade this tag, vim mode is enabled"),
+							VimMode: true,
+							Options: compatibleStrings,
+						}
+						if err := survey.AskOne(prompt, &upgradeTag); err != nil {
+							log.Fatal(err)
+						}
+					}
+				}
+			}
+			if upgradeTag != "skip" {
+				ok, err := recipe.UpdateTag(image, upgradeTag)
+				if err != nil {
+					log.Fatal(err)
+				}
+				if ok {
+					log.Info(i18n.G("tag upgraded from %s to %s for %s", tag.String(), upgradeTag, image))
+				}
+			} else {
+				if !internal.NoInput {
+					log.Warn(i18n.G("not upgrading %s, skipping as requested", image))
+				}
+			}
+		}
+
+		if internal.NoInput {
+			if internal.MachineReadable {
+				jsonstring, err := json.Marshal(upgradeList)
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				fmt.Println(string(jsonstring))
+
+				return
+			}
+
+			for _, upgrade := range upgradeList {
+				log.Info(i18n.G("can upgrade service: %s, image: %s, tag: %s ::", upgrade.Service, upgrade.Image, upgrade.Tag))
+				for _, utag := range upgrade.UpgradeTags {
+					log.Infof("    %s", utag)
+				}
+			}
+		}
+
+		isClean, err := gitPkg.IsClean(recipe.Dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if !isClean {
+			log.Info(i18n.G("%s currently has these unstaged changes 👇", recipe.Name))
+			if err := gitPkg.DiffUnstaged(recipe.Dir); err != nil {
+				log.Fatal(err)
+			}
+		}
+	},
+}
+
+var (
+	allTags bool
+)
+
+func init() {
+	RecipeUpgradeCommand.Flags().BoolVarP(
+		&internal.Major,
+		i18n.G("major"),
+		i18n.G("x"),
+		false,
+		i18n.G("increase the major part of the version"),
+	)
+
+	RecipeUpgradeCommand.Flags().BoolVarP(
+		&internal.Minor,
+		i18n.G("minor"),
+		i18n.G("y"),
+		false,
+		i18n.G("increase the minor part of the version"),
+	)
+
+	RecipeUpgradeCommand.Flags().BoolVarP(
+		&internal.Patch,
+		i18n.G("patch"),
+		i18n.G("z"),
+		false,
+		i18n.G("increase the patch part of the version"),
+	)
+
+	RecipeUpgradeCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+
+	RecipeUpgradeCommand.Flags().BoolVarP(
+		&allTags,
+		i18n.G("all-tags"),
+		i18n.GC("a", "recipe upgrade"),
+		false,
+		i18n.G("list all tags, not just upgrades"),
+	)
+}

cli/recipe/version.go

@@ -0,0 +1,143 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra recipe versions` aliases. use a comma separated list of aliases
+// with no spaces in between
+var recipeVersionsAliases = i18n.G("v")
+
+var RecipeVersionCommand = &cobra.Command{
+	// translators: `recipe versions` command
+	Use:     i18n.G("versions <recipe> [flags]"),
+	Aliases: strings.Split(recipeVersionsAliases, ","),
+	// translators: Short description for `recipe versions` command
+	Short: i18n.G("List recipe versions"),
+	Args:  cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.RecipeNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var warnMessages []string
+
+		recipe := internal.ValidateRecipe(args, cmd.Name())
+
+		catl, err := recipePkg.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		recipeMeta, ok := catl[recipe.Name]
+		if !ok {
+			warnMessages = append(warnMessages, i18n.G("retrieved versions from local recipe repository"))
+
+			recipeVersions, warnMsg, err := recipe.GetRecipeVersions()
+			if err != nil {
+				warnMessages = append(warnMessages, err.Error())
+			}
+			if len(warnMsg) > 0 {
+				warnMessages = append(warnMessages, warnMsg...)
+			}
+
+			recipeMeta = recipePkg.RecipeMeta{Versions: recipeVersions}
+		}
+
+		if len(recipeMeta.Versions) == 0 {
+			log.Fatal(i18n.G("%s has no published versions?", recipe.Name))
+		}
+
+		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
+			table, err := formatter.CreateTable()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			table.Headers(i18n.G("SERVICE"), i18n.G("IMAGE"), i18n.G("TAG"), i18n.G("VERSION"))
+
+			for version, meta := range recipeMeta.Versions[i] {
+				var allRows [][]string
+				var rows [][]string
+
+				for service, serviceMeta := range meta {
+					recipeVersion := version
+					if service != "app" {
+						recipeVersion = ""
+					}
+
+					rows = append(rows, []string{
+						service,
+						serviceMeta.Image,
+						serviceMeta.Tag,
+						recipeVersion,
+					})
+
+					allRows = append(allRows, []string{
+						version,
+						service,
+						serviceMeta.Image,
+						serviceMeta.Tag,
+						recipeVersion,
+					})
+				}
+
+				sort.Slice(rows, sortServiceByName(rows))
+
+				table.Rows(rows...)
+
+				if !internal.MachineReadable {
+					if err := formatter.PrintTable(table); err != nil {
+						log.Fatal(err)
+					}
+					continue
+				}
+
+				if internal.MachineReadable {
+					sort.Slice(allRows, sortServiceByName(allRows))
+					headers := []string{i18n.G("VERSION"), i18n.G("SERVICE"), i18n.G("NAME"), i18n.G("TAG")}
+					out, err := formatter.ToJSON(headers, allRows)
+					if err != nil {
+						log.Fatal(i18n.G("unable to render to JSON: %s", err))
+					}
+					fmt.Println(out)
+					continue
+				}
+			}
+		}
+
+		if !internal.MachineReadable {
+			for _, warnMsg := range warnMessages {
+				log.Warn(warnMsg)
+			}
+		}
+	},
+}
+
+func sortServiceByName(versions [][]string) func(i, j int) bool {
+	return func(i, j int) bool {
+		return versions[i][0] < versions[j][0]
+	}
+}
+
+func init() {
+	RecipeVersionCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+}

cli/run.go

@@ -0,0 +1,320 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/cli/app"
+	"coopcloud.tech/abra/cli/catalogue"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/cli/recipe"
+	"coopcloud.tech/abra/cli/server"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	charmLog "github.com/charmbracelet/log"
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
+)
+
+var (
+	// translators: `abra` usage template. please translate only words like
+	// "Aliases" and "Example" and nothing inside the {{ ... }}
+	usageTemplate = i18n.G(`Usage:{{if .Runnable}}
+  {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}
+  {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}
+
+Aliases:
+  {{.NameAndAliases}}{{end}}{{if .HasExample}}
+
+Examples:
+{{.Example}}{{end}}{{if .HasAvailableSubCommands}}
+
+Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
+  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}
+
+Flags:
+{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}
+
+Global Flags:
+{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}}
+
+Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}}
+  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
+
+Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
+`)
+
+	helpCmd = &cobra.Command{
+		Use: i18n.G("help [command]"),
+		// translators: Short description for `help` command
+		Short: i18n.G("Help about any command"),
+		Long: i18n.G(`Help provides help for any command in the application.
+Simply type abra help [path to command] for full details.`),
+		Run: func(c *cobra.Command, args []string) {
+			cmd, _, e := c.Root().Find(args)
+			if cmd == nil || e != nil {
+				c.Print(i18n.G("unknown help topic %#q\n", args))
+				if err := c.Root().Usage(); err != nil {
+					log.Fatal(err)
+				}
+			} else {
+				cmd.InitDefaultHelpFlag()
+				cmd.InitDefaultVersionFlag()
+				if err := cmd.Help(); err != nil {
+					log.Fatal(err)
+				}
+			}
+		},
+	}
+)
+
+func Run(version, commit string) {
+	rootCmd := &cobra.Command{
+		// translators: `abra` binary name
+		Use: i18n.G("abra [cmd] [args] [flags]"),
+		// translators: Short description for `abra` binary
+		Short: i18n.G("The Co-op Cloud command-line utility belt 🎩🐇"),
+		// translators: Long description for `abra` binary. This needs to be
+		// translated in the same way as the Short description so that everything
+		// matches up
+		Long: i18n.G(`The Co-op Cloud command-line utility belt 🎩🐇
+
+Config:
+  $ABRA_DIR: %s`, config.ABRA_DIR),
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		ValidArgs: []string{
+			// translators: `abra app` command for autocompletion
+			i18n.G("app"),
+			// translators: `abra autocomplete` command for autocompletion
+			i18n.G("autocomplete"),
+			// translators: `abra catalogue` command for autocompletion
+			i18n.G("catalogue"),
+			// translators: `abra man` command for autocompletion
+			i18n.G("man"),
+			// translators: `abra recipe` command for autocompletion
+			i18n.G("recipe"),
+			// translators: `abra server` command for autocompletion
+			i18n.G("server"),
+			// translators: `abra upgrade` command for autocompletion
+			i18n.G("upgrade"),
+		},
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			dirs := []map[string]os.FileMode{
+				{config.ABRA_DIR: 0764},
+				{config.SERVERS_DIR: 0700},
+				{config.RECIPES_DIR: 0764},
+				{config.LOGS_DIR: 0764},
+			}
+
+			for _, dir := range dirs {
+				for path, perm := range dir {
+					if err := os.Mkdir(path, perm); err != nil {
+						if !os.IsExist(err) {
+							return errors.New(i18n.G("unable to create %s: %s", path, err))
+						}
+						continue
+					}
+				}
+			}
+
+			log.Logger.SetStyles(charmLog.DefaultStyles())
+			charmLog.SetDefault(log.Logger)
+
+			if internal.MachineReadable {
+				log.SetOutput(os.Stderr)
+			}
+
+			if internal.Debug {
+				log.SetLevel(log.DebugLevel)
+				log.SetOutput(os.Stderr)
+				log.SetReportCaller(true)
+			}
+
+			log.Debug(i18n.G(
+				"abra version: %s, commit: %s, lang: %s",
+				version, formatter.SmallSHA(commit), i18n.Locale,
+			))
+
+			return nil
+		},
+	}
+
+	rootCmd.CompletionOptions.DisableDefaultCmd = true
+	rootCmd.SetUsageTemplate(usageTemplate)
+	rootCmd.SetHelpCommand(helpCmd)
+
+	// translators: `abra man` aliases. use a comma separated list of aliases
+	// with no spaces in between
+	manAliases := i18n.G("m")
+
+	manCommand := &cobra.Command{
+		// translators: `man` command
+		Use:     i18n.G("man [flags]"),
+		Aliases: strings.Split(manAliases, ","),
+		// translators: Short description for `man` command
+		Short: i18n.G("Generate manpage"),
+		Example: i18n.G(`  # generate the man pages into /usr/local/share/man/man1
+  abra_path=$(which abra)  # pass abra absolute path to sudo below
+  sudo $abra_path man
+  sudo mandb
+
+  # read the man pages
+  man abra
+  man abra-app-deploy`),
+		Run: func(cmd *cobra.Command, args []string) {
+			header := &doc.GenManHeader{
+				Title:   "ABRA",
+				Section: "1",
+			}
+
+			manDir := "/usr/local/share/man/man1"
+			if _, err := os.Stat(manDir); os.IsNotExist(err) {
+				log.Fatal(i18n.G("unable to proceed, %s does not exist?", manDir))
+			}
+
+			err := doc.GenManTree(rootCmd, header, manDir)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			log.Info(i18n.G("don't forget to run 'sudo mandb'"))
+		},
+	}
+
+	rootCmd.PersistentFlags().BoolVarP(
+		&internal.Debug,
+		"debug",
+		"d",
+		false,
+		i18n.G("show debug messages"),
+	)
+
+	rootCmd.PersistentFlags().BoolVarP(
+		&internal.NoInput,
+		"no-input",
+		"n",
+		false,
+		i18n.G("toggle non-interactive mode"),
+	)
+
+	rootCmd.PersistentFlags().BoolVarP(
+		&internal.Offline,
+		"offline",
+		"o",
+		false,
+		i18n.G("prefer offline & filesystem access"),
+	)
+
+	rootCmd.PersistentFlags().BoolVarP(
+		&internal.Help,
+		i18n.G("help"),
+		i18n.G("h"),
+		false,
+		i18n.G("help for abra"),
+	)
+
+	rootCmd.Flags().BoolVarP(
+		&internal.Version,
+		i18n.G("version"),
+		i18n.G("v"),
+		false,
+		i18n.G("version for abra"),
+	)
+
+	catalogue.CatalogueCommand.AddCommand(
+		catalogue.CatalogueGenerateCommand,
+		catalogue.CatalogueSyncCommand,
+	)
+
+	server.ServerCommand.AddCommand(
+		server.ServerAddCommand,
+		server.ServerListCommand,
+		server.ServerPruneCommand,
+		server.ServerRemoveCommand,
+	)
+
+	recipe.RecipeCommand.AddCommand(
+		recipe.RecipeDiffCommand,
+		recipe.RecipeFetchCommand,
+		recipe.RecipeLintCommand,
+		recipe.RecipeListCommand,
+		recipe.RecipeNewCommand,
+		recipe.RecipeReleaseCommand,
+		recipe.RecipeResetCommand,
+		recipe.RecipeSyncCommand,
+		recipe.RecipeUpgradeCommand,
+		recipe.RecipeVersionCommand,
+	)
+
+	rootCmd.AddCommand(
+		UpgradeCommand,
+		AutocompleteCommand,
+		manCommand,
+		app.AppCommand,
+		catalogue.CatalogueCommand,
+		server.ServerCommand,
+		recipe.RecipeCommand,
+	)
+
+	app.AppCmdCommand.AddCommand(
+		app.AppCmdListCommand,
+	)
+
+	app.AppSecretCommand.AddCommand(
+		app.AppSecretGenerateCommand,
+		app.AppSecretInsertCommand,
+		app.AppSecretRmCommand,
+		app.AppSecretLsCommand,
+	)
+
+	app.AppVolumeCommand.AddCommand(
+		app.AppVolumeListCommand,
+		app.AppVolumeRemoveCommand,
+	)
+
+	app.AppBackupCommand.AddCommand(
+		app.AppBackupListCommand,
+		app.AppBackupDownloadCommand,
+		app.AppBackupCreateCommand,
+		app.AppBackupSnapshotsCommand,
+	)
+
+	app.AppEnvCommand.AddCommand(
+		app.AppEnvListCommand,
+		app.AppEnvPullCommand,
+	)
+
+	app.AppCommand.AddCommand(
+		app.AppBackupCommand,
+		app.AppCheckCommand,
+		app.AppCmdCommand,
+		app.AppConfigCommand,
+		app.AppCpCommand,
+		app.AppDeployCommand,
+		app.AppListCommand,
+		app.AppLogsCommand,
+		app.AppNewCommand,
+		app.AppPsCommand,
+		app.AppRemoveCommand,
+		app.AppRestartCommand,
+		app.AppRestoreCommand,
+		app.AppRollbackCommand,
+		app.AppMoveCommand,
+		app.AppRunCommand,
+		app.AppSecretCommand,
+		app.AppServicesCommand,
+		app.AppUndeployCommand,
+		app.AppUpgradeCommand,
+		app.AppVolumeCommand,
+		app.AppLabelsCommand,
+		app.AppEnvCommand,
+	)
+
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

cli/server/add.go

@@ -0,0 +1,211 @@
+package server
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/dns"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/server"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra server add` aliases. use a comma separated list of
+// aliases with no spaces in between
+var serverAddAliases = i18n.GC("a", "server add")
+
+var ServerAddCommand = &cobra.Command{
+	// translators: `server add` command
+	Use:     i18n.G("add [[server] | --local] [flags]"),
+	Aliases: strings.Split(serverAddAliases, ","),
+	// translators: Short description for `server add` command
+	Short: i18n.G("Add a new server"),
+	Long: i18n.G(`Add a new server to your configuration so that it can be managed by Abra.
+
+Abra relies on the standard SSH command-line and ~/.ssh/config for client
+connection details. You must configure an entry per-host in your ~/.ssh/config
+for each server:
+
+  Host 1312.net 1312
+    Hostname 1312.net
+    User antifa
+    Port 12345
+    IdentityFile ~/.ssh/antifa@somewhere
+
+If "--local" is passed, then Abra assumes that the current local server is
+intended as the target server. This is useful when you want to have your entire
+Co-op Cloud config located on the server itself, and not on your local
+developer machine. The domain is then set to "default".`),
+	Example: i18n.G("  abra server add 1312.net"),
+	Args:    cobra.RangeArgs(0, 1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		if !local {
+			return autocomplete.ServerNameComplete()
+		}
+		return nil, cobra.ShellCompDirectiveDefault
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) > 0 && local {
+			log.Fatal(i18n.G("cannot use [server] and --local together"))
+		}
+
+		if len(args) == 0 && !local {
+			log.Fatal(i18n.G("missing argument or --local/-l flag"))
+		}
+
+		name := "default"
+		if !local {
+			name = internal.ValidateDomain(args)
+		}
+
+		// NOTE(d1): reasonable 5 second timeout for connections which can't
+		// succeed. The connection is attempted twice, so this results in 10
+		// seconds.
+		timeout := client.WithTimeout(5)
+
+		if local {
+			created, err := createServerDir(name)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			log.Debug(i18n.G("attempting to create client for %s", name))
+
+			if _, err := client.New(name, timeout); err != nil {
+				cleanUp(name)
+				log.Fatal(err)
+			}
+
+			if created {
+				log.Info(i18n.G("local server successfully added"))
+			} else {
+				log.Warn(i18n.G("local server already exists"))
+			}
+
+			return
+		}
+
+		_, err := createServerDir(name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		created, err := newContext(name)
+		if err != nil {
+			cleanUp(name)
+			log.Fatal(i18n.G("unable to create local context: %s", err))
+		}
+
+		log.Debug(i18n.G("attempting to create client for %s", name))
+
+		if _, err := client.New(name, timeout); err != nil {
+			cleanUp(name)
+			log.Fatal(i18n.G("ssh %s error: %s", name, sshPkg.Fatal(name, err)))
+		}
+
+		if created {
+			log.Info(i18n.G("%s successfully added", name))
+
+			if _, err := dns.EnsureIPv4(name); err != nil {
+				log.Warn(i18n.G("unable to resolve IPv4 for %s", name))
+			}
+
+			return
+		}
+
+		log.Warn(i18n.G("%s already exists", name))
+	},
+}
+
+// cleanUp cleans up the partially created context/client details for a failed
+// "server add" attempt.
+func cleanUp(name string) {
+	if name != "default" {
+		log.Debug(i18n.G("serverAdd: cleanUp: cleaning up context for %s", name))
+		if err := client.DeleteContext(name); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	serverDir := filepath.Join(config.SERVERS_DIR, name)
+	files, err := config.GetAllFilesInDirectory(serverDir)
+	if err != nil {
+		log.Fatal(i18n.G("serverAdd: cleanUp: unable to list files in %s: %s", serverDir, err))
+	}
+
+	if len(files) > 0 {
+		log.Debug(i18n.G("serverAdd: cleanUp: %s is not empty, aborting cleanup", serverDir))
+		return
+	}
+
+	if err := os.RemoveAll(serverDir); err != nil {
+		log.Fatal(i18n.G("serverAdd: cleanUp: failed to remove %s: %s", serverDir, err))
+	}
+}
+
+// newContext creates a new internal Docker context for a server. This is how
+// Docker manages SSH connection details. These are stored to disk in
+// ~/.docker. Abra can manage this completely for the user, so it's an
+// implementation detail.
+func newContext(name string) (bool, error) {
+	store := contextPkg.NewDefaultDockerContextStore()
+	contexts, err := store.Store.List()
+	if err != nil {
+		return false, err
+	}
+
+	for _, context := range contexts {
+		if context.Name == name {
+			log.Debug(i18n.G("context for %s already exists", name))
+			return false, nil
+		}
+	}
+
+	log.Debugf(i18n.G("creating context with domain %s", name))
+
+	if err := client.CreateContext(name); err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// createServerDir creates the ~/.abra/servers/... directory for a new server.
+func createServerDir(name string) (bool, error) {
+	if err := server.CreateServerDir(name); err != nil {
+		if !os.IsExist(err) {
+			return false, err
+		}
+
+		log.Debug(i18n.G("server dir for %s already created", name))
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+var (
+	local bool
+)
+
+func init() {
+	ServerAddCommand.Flags().BoolVarP(
+		&local,
+		i18n.G("local"),
+		i18n.G("l"),
+		false,
+		i18n.G("use local server"),
+	)
+}

cli/server/list.go

@@ -0,0 +1,110 @@
+package server
+
+import (
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/docker/cli/cli/connhelper/ssh"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra server list` aliases. use a comma separated list of
+// aliases with no spaces in between
+var serverListAliases = i18n.G("ls")
+
+var ServerListCommand = &cobra.Command{
+	// translators: `server list` command
+	Use:     i18n.G("list [flags]"),
+	Aliases: strings.Split(serverListAliases, ","),
+	// translators: Short description for `server list` command
+	Short: i18n.G("List managed servers"),
+	Args:  cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		dockerContextStore := contextPkg.NewDefaultDockerContextStore()
+		contexts, err := dockerContextStore.Store.List()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		table, err := formatter.CreateTable()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		headers := []string{i18n.G("NAME"), i18n.G("HOST")}
+		table.Headers(headers...)
+
+		serverNames, err := config.ReadServerNames()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var rows [][]string
+		for _, serverName := range serverNames {
+			var row []string
+			for _, dockerCtx := range contexts {
+				endpoint, err := contextPkg.GetContextEndpoint(dockerCtx)
+				if err != nil && strings.Contains(err.Error(), "does not exist") {
+					// No local context found, we can continue safely
+					continue
+				}
+
+				if dockerCtx.Name == serverName {
+					sp, err := ssh.ParseURL(endpoint)
+					if err != nil {
+						log.Fatal(err)
+					}
+
+					if sp.Host == "" {
+						sp.Host = i18n.G("unknown")
+					}
+
+					row = []string{serverName, sp.Host}
+					rows = append(rows, row)
+				}
+			}
+
+			if len(row) == 0 {
+				if serverName == "default" {
+					row = []string{serverName, i18n.G("local")}
+				} else {
+					row = []string{serverName, i18n.G("unknown")}
+				}
+				rows = append(rows, row)
+			}
+
+			table.Row(row...)
+		}
+
+		if internal.MachineReadable {
+			out, err := formatter.ToJSON(headers, rows)
+			if err != nil {
+				log.Fatal(i18n.G("unable to render to JSON: %s", err))
+			}
+
+			fmt.Println(out)
+
+			return
+		}
+
+		if err := formatter.PrintTable(table); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+func init() {
+	ServerListCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		i18n.G("machine"),
+		i18n.G("m"),
+		false,
+		i18n.G("print machine-readable output"),
+	)
+}

cli/server/prune.go

@@ -0,0 +1,111 @@
+package server
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra server prune` aliases. use a comma separated list of
+// aliases with no spaces in between
+var serverPruneliases = i18n.G("p")
+
+var ServerPruneCommand = &cobra.Command{
+	// translators: `server prune` command
+	Use:     i18n.G("prune <server> [flags]"),
+	Aliases: strings.Split(serverPruneliases, ","),
+	// translators: Short description for `server prune` command
+	Short: i18n.G("Prune resources on a server"),
+	Long: i18n.G(`Prunes unused containers, networks, and dangling images.
+
+Use "--volumes/-v" to remove volumes that are not associated with a deployed
+app. This can result in unwanted data loss if not used carefully.`),
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.ServerNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		serverName := internal.ValidateServer(args)
+
+		cl, err := client.New(serverName)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var filterArgs filters.Args
+
+		cr, err := cl.ContainersPrune(cmd.Context(), filterArgs)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		cntSpaceReclaimed := formatter.ByteCountSI(cr.SpaceReclaimed)
+		log.Info(i18n.G("containers pruned: %d; space reclaimed: %s", len(cr.ContainersDeleted), cntSpaceReclaimed))
+
+		nr, err := cl.NetworksPrune(cmd.Context(), filterArgs)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info(i18n.G("networks pruned: %d", len(nr.NetworksDeleted)))
+
+		pruneFilters := filters.NewArgs()
+		if allFilter {
+			log.Debug(i18n.G("removing all images, not only dangling ones"))
+			pruneFilters.Add("dangling", "false")
+		}
+
+		ir, err := cl.ImagesPrune(cmd.Context(), pruneFilters)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		imgSpaceReclaimed := formatter.ByteCountSI(ir.SpaceReclaimed)
+		log.Info(i18n.G("images pruned: %d; space reclaimed: %s", len(ir.ImagesDeleted), imgSpaceReclaimed))
+
+		if volumesFilter {
+			vr, err := cl.VolumesPrune(cmd.Context(), filterArgs)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			volSpaceReclaimed := formatter.ByteCountSI(vr.SpaceReclaimed)
+			log.Info(i18n.G("volumes pruned: %d; space reclaimed: %s", len(vr.VolumesDeleted), volSpaceReclaimed))
+		}
+
+		return
+	},
+}
+
+var (
+	allFilter     bool
+	volumesFilter bool
+)
+
+func init() {
+	ServerPruneCommand.Flags().BoolVarP(
+		&allFilter,
+		i18n.G("all"),
+		i18n.GC("a", "server prune"),
+		false,
+		i18n.G("remove all unused images"),
+	)
+
+	ServerPruneCommand.Flags().BoolVarP(
+		&volumesFilter,
+		i18n.G("volumes"),
+		i18n.G("v"),
+		false,
+		i18n.G("remove volumes"),
+	)
+}

cli/server/remove.go

@@ -0,0 +1,54 @@
+package server
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra server remove` aliases. use a comma separated list of
+// aliases with no spaces in between
+var serverRemoveAliases = i18n.G("rm")
+
+var ServerRemoveCommand = &cobra.Command{
+	// translators: `server remove` command
+	Use:     i18n.G("remove <server> [flags]"),
+	Aliases: strings.Split(serverRemoveAliases, ","),
+	// translators: Short description for `server remove` command
+	Short: i18n.G("Remove a managed server"),
+	Long: i18n.G(`Remove a managed server.
+
+Abra will remove the internal bookkeeping ($ABRA_DIR/servers/...) and
+underlying client connection context. This server will then be lost in time,
+like tears in rain.`),
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.ServerNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		serverName := internal.ValidateServer(args)
+
+		if err := client.DeleteContext(serverName); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := os.RemoveAll(filepath.Join(config.SERVERS_DIR, serverName)); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info(i18n.G("%s is now lost in time, like tears in rain", serverName))
+
+		return
+	},
+}

cli/server/server.go

@@ -0,0 +1,21 @@
+package server
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra server` aliases. use a comma separated list of aliases
+// with no spaces in between
+var serverAliases = i18n.G("s")
+
+// ServerCommand defines the `abra server` command and its subcommands
+var ServerCommand = &cobra.Command{
+	// translators: `server` command group
+	Use:     i18n.G("server [cmd] [args] [flags]"),
+	Aliases: strings.Split(serverAliases, ","),
+	// translators: Short description for `server` command group
+	Short: i18n.G("Manage servers"),
+}

cli/upgrade.go

@@ -0,0 +1,64 @@
+// Package cli provides the interface for the command-line.
+package cli
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
+)
+
+// translators: `abra upgrade` aliases. use a comma separated list of aliases with
+// no spaces in between
+var upgradeAliases = i18n.G("u")
+
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = &cobra.Command{
+	// translators: `upgrade` command
+	Use:     i18n.G("upgrade [flags]"),
+	Aliases: strings.Split(upgradeAliases, ","),
+	// translators: Short description for `upgrade` command
+	Short: i18n.G("Upgrade abra"),
+	Long: i18n.G(`Upgrade abra in-place with the latest stable or release candidate.
+
+By default, the latest stable release is downloaded.
+
+Use "--rc/-r" to install the latest release candidate. Please bear in mind that
+it may contain absolutely catastrophic deal-breaker bugs. Thank you very much
+for the testing efforts 💗`),
+	Example: i18n.G("  abra upgrade --rc"),
+	Args:    cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		mainURL := "https://install.abra.coopcloud.tech"
+		c := exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash", mainURL))
+
+		if releaseCandidate {
+			releaseCandidateURL := "https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer"
+			c = exec.Command("bash", "-c", fmt.Sprintf("wget -q -O- %s | bash -s -- --rc", releaseCandidateURL))
+		}
+
+		log.Debugf(i18n.G("attempting to run %s", c))
+
+		if err := internal.RunCmd(c); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+var (
+	releaseCandidate bool
+)
+
+func init() {
+	UpgradeCommand.Flags().BoolVarP(
+		&releaseCandidate,
+		"rc",
+		"r",
+		false,
+		i18n.G("install release candidate (may contain bugs)"),
+	)
+}

cmd/abra/main.go

@@ -0,0 +1,23 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli"
+)
+
+// Version is the current version of Abra.
+var Version string
+
+// Commit is the current git commit of Abra.
+var Commit string
+
+func main() {
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	cli.Run(Version, Commit)
+}

go.mod

@@ -0,0 +1,160 @@
+module coopcloud.tech/abra
+
+go 1.24.2
+
+require (
+	coopcloud.tech/tagcmp v0.0.0-20250818180036-0ec1b205b5ca
+	git.coopcloud.tech/toolshed/godotenv v1.5.2-0.20250103171850-4d0ca41daa5c
+	github.com/AlecAivazis/survey/v2 v2.3.7
+	github.com/charmbracelet/bubbletea v1.3.10
+	github.com/charmbracelet/lipgloss v1.1.0
+	github.com/charmbracelet/log v0.4.2
+	github.com/distribution/reference v0.6.0
+	github.com/docker/cli v29.1.5+incompatible
+	github.com/docker/docker v28.5.2+incompatible
+	github.com/docker/go-units v0.5.0
+	github.com/go-git/go-git/v5 v5.16.4
+	github.com/google/go-cmp v0.7.0
+	github.com/leonelquinteros/gotext v1.7.2
+	github.com/moby/moby/client v0.2.1
+	github.com/moby/sys/signal v0.7.1
+	github.com/moby/term v0.5.2
+	github.com/pkg/errors v0.9.1
+	github.com/schollz/progressbar/v3 v3.19.0
+	golang.org/x/term v0.39.0
+	gopkg.in/yaml.v3 v3.0.1
+	gotest.tools/v3 v3.5.2
+)
+
+require (
+	dario.cat/mergo v1.0.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+	github.com/BurntSushi/toml v1.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.3.0 // indirect
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/charmbracelet/colorprofile v0.4.1 // indirect
+	github.com/charmbracelet/x/ansi v0.11.4 // indirect
+	github.com/charmbracelet/x/cellbuf v0.0.14 // indirect
+	github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 // indirect
+	github.com/charmbracelet/x/term v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.7.0 // indirect
+	github.com/clipperhouse/stringish v0.1.1 // indirect
+	github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.6.2 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/go-connections v0.6.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.7.0 // indirect
+	github.com/go-logfmt/logfmt v0.6.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/kevinburke/ssh_config v1.4.0 // indirect
+	github.com/klauspost/compress v1.18.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.19 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/go-archive v0.2.0 // indirect
+	github.com/moby/moby/api v1.53.0-rc.1 // indirect
+	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/morikuni/aec v1.1.0 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/termenv v0.16.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/runc v1.1.13 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.5.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.67.5 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
+	github.com/skeema/knownhosts v1.3.2 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
+	go.opentelemetry.io/otel v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
+	go.opentelemetry.io/otel/metric v1.39.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.39.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
+	go.opentelemetry.io/otel/trace v1.39.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.47.0 // indirect
+	golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
+	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260114163908-3f89685c29c3 // indirect
+	google.golang.org/grpc v1.78.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)
+
+require (
+	github.com/containers/image v3.0.2+incompatible
+	github.com/containers/storage v1.38.2 // indirect
+	github.com/decentral1se/passgen v1.0.1
+	github.com/docker/docker-credential-helpers v0.9.5 // indirect
+	github.com/fvbommel/sortorder v1.1.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/sergi/go-diff v1.4.0 // indirect
+	github.com/spf13/cobra v1.10.2
+	github.com/stretchr/testify v1.11.1
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	golang.org/x/sys v0.40.0
+)

pkg/app/app.go

@@ -0,0 +1,701 @@
+package app
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"regexp"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+
+	"coopcloud.tech/abra/pkg/log"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/schollz/progressbar/v3"
+)
+
+// Get retrieves an app
+func Get(appName string) (App, error) {
+	files, err := LoadAppFiles("")
+	if err != nil {
+		return App{}, err
+	}
+
+	app, err := GetApp(files, appName)
+	if err != nil {
+		return App{}, err
+	}
+
+	log.Debug(i18n.G("loaded app %s: %s", appName, app))
+
+	return app, nil
+}
+
+// GetApp loads an apps settings, reading it from file, in preparation to use
+// it. It should only be used when ready to use the env file to keep IO
+// operations down.
+func GetApp(apps AppFiles, name AppName) (App, error) {
+	appFile, exists := apps[name]
+	if !exists {
+		return App{}, errors.New(i18n.G("cannot find app with name %s", name))
+	}
+
+	app, err := ReadAppEnvFile(appFile, name)
+	if err != nil {
+		return App{}, err
+	}
+
+	return app, nil
+}
+
+// GetApps returns a slice of Apps with their env files read from a given
+// slice of AppFiles.
+func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) {
+	var apps []App
+
+	for name := range appFiles {
+		app, err := GetApp(appFiles, name)
+		if err != nil {
+			return nil, err
+		}
+
+		if recipeFilter != "" {
+			if app.Recipe.Name == recipeFilter {
+				apps = append(apps, app)
+			}
+		} else {
+			apps = append(apps, app)
+		}
+	}
+
+	return apps, nil
+}
+
+// App reprents an app with its env file read into memory
+type App struct {
+	Name   AppName
+	Recipe recipe.Recipe
+	Domain string
+	Env    envfile.AppEnv
+	Server string
+	Path   string
+}
+
+// String outputs a human-friendly string representation.
+func (a App) String() string {
+	out := fmt.Sprintf("{name: %s, ", a.Name)
+	out += fmt.Sprintf("recipe: %s, ", a.Recipe)
+	out += fmt.Sprintf("domain: %s, ", a.Domain)
+	out += fmt.Sprintf("env %s, ", a.Env)
+	out += fmt.Sprintf("server %s, ", a.Server)
+	out += fmt.Sprintf("path %s}", a.Path)
+	return out
+}
+
+// Type aliases to make code hints easier to understand
+
+// AppName is AppName
+type AppName = string
+
+// AppFile represents app env files on disk without reading the contents
+type AppFile struct {
+	Path   string
+	Server string
+}
+
+// AppFiles is a slice of appfiles
+type AppFiles map[AppName]AppFile
+
+// See documentation of config.StackName
+func (a App) StackName() string {
+	if _, exists := a.Env["STACK_NAME"]; exists {
+		return a.Env["STACK_NAME"]
+	}
+
+	stackName := StackName(a.Name)
+
+	a.Env["STACK_NAME"] = stackName
+
+	return stackName
+}
+
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
+func StackName(appName string) string {
+	stackName := SanitiseAppName(appName)
+
+	if len(stackName) > config.MAX_SANITISED_APP_NAME_LENGTH {
+		log.Debug(i18n.G("trimming %s to %s to avoid runtime limits", stackName, stackName[:config.MAX_SANITISED_APP_NAME_LENGTH]))
+		stackName = stackName[:config.MAX_SANITISED_APP_NAME_LENGTH]
+	}
+
+	return stackName
+}
+
+// Filters retrieves app filters for querying the container runtime. By default
+// it filters on all services in the app. It is also possible to pass an
+// otional list of service names, which get filtered instead.
+//
+// Due to upstream issues, filtering works different depending on what you're
+// querying. So, for example, secrets don't work with regex! The caller needs
+// to implement their own validation that the right secrets are matched. In
+// order to handle these cases, we provide the `appendServiceNames` /
+// `exactMatch` modifiers.
+func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) {
+	filters := filters.NewArgs()
+	if len(services) > 0 {
+		for _, serviceName := range services {
+			filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch))
+		}
+		return filters, nil
+	}
+
+	// When not appending the service name, just add one filter for the whole
+	// stack.
+	if !appendServiceNames {
+		f := fmt.Sprintf("%s", a.StackName())
+		if exactMatch {
+			f = fmt.Sprintf("^%s", f)
+		}
+		filters.Add("name", f)
+		return filters, nil
+	}
+
+	composeFiles, err := a.Recipe.GetComposeFiles(a.Env)
+	if err != nil {
+		return filters, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	compose, err := GetAppComposeConfig(a.Recipe.Name, opts, a.Env)
+	if err != nil {
+		return filters, err
+	}
+
+	for _, service := range compose.Services {
+		f := ServiceFilter(a.StackName(), service.Name, exactMatch)
+		filters.Add("name", f)
+	}
+
+	return filters, nil
+}
+
+// ServiceFilter creates a filter string for filtering a service in the docker
+// container runtime. When exact match is true, it uses regex to match the
+// string exactly.
+func ServiceFilter(stack, service string, exact bool) string {
+	if exact {
+		return fmt.Sprintf("^%s_%s", stack, service)
+	}
+	return fmt.Sprintf("%s_%s", stack, service)
+}
+
+// ByServer sort a slice of Apps
+type ByServer []App
+
+func (a ByServer) Len() int      { return len(a) }
+func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServer) Less(i, j int) bool {
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByServerAndRecipe sort a slice of Apps
+type ByServerAndRecipe []App
+
+func (a ByServerAndRecipe) Len() int      { return len(a) }
+func (a ByServerAndRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServerAndRecipe) Less(i, j int) bool {
+	if a[i].Server == a[j].Server {
+		return strings.ToLower(a[i].Recipe.Name) < strings.ToLower(a[j].Recipe.Name)
+	}
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByRecipe sort a slice of Apps
+type ByRecipe []App
+
+func (a ByRecipe) Len() int      { return len(a) }
+func (a ByRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByRecipe) Less(i, j int) bool {
+	return strings.ToLower(a[i].Recipe.Name) < strings.ToLower(a[j].Recipe.Name)
+}
+
+// ByName sort a slice of Apps
+type ByName []App
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
+}
+
+func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
+	env, err := envfile.ReadEnv(appFile.Path)
+	if err != nil {
+		return App{}, errors.New(i18n.G("env file for %s couldn't be read: %s", name, err.Error()))
+	}
+
+	app, err := NewApp(env, name, appFile)
+	if err != nil {
+		return App{}, errors.New(i18n.G("env file for %s has issues: %s", name, err.Error()))
+	}
+
+	return app, nil
+}
+
+// NewApp creates new App object
+func NewApp(env envfile.AppEnv, name string, appFile AppFile) (App, error) {
+	domain := env["DOMAIN"]
+
+	recipeName, exists := env["RECIPE"]
+	if !exists {
+		recipeName, exists = env["TYPE"]
+		if !exists {
+			return App{}, errors.New(i18n.G("%s is missing the TYPE env var?", name))
+		}
+	}
+
+	return App{
+		Name:   name,
+		Domain: domain,
+		Recipe: recipe.Get(recipeName),
+		Env:    env,
+		Server: appFile.Server,
+		Path:   appFile.Path,
+	}, nil
+}
+
+// LoadAppFiles gets all app files for a given set of servers or all servers.
+func LoadAppFiles(servers ...string) (AppFiles, error) {
+	appFiles := make(AppFiles)
+	if len(servers) == 1 {
+		if servers[0] == "" {
+			// Empty servers flag, one string will always be passed
+			var err error
+			servers, err = config.GetAllFoldersInDirectory(config.SERVERS_DIR)
+			if err != nil {
+				return appFiles, err
+			}
+		}
+	}
+
+	log.Debug(i18n.G("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", ")))
+
+	for _, server := range servers {
+		serverDir := path.Join(config.SERVERS_DIR, server)
+		files, err := config.GetAllFilesInDirectory(serverDir)
+		if err != nil {
+			return appFiles, errors.New(i18n.G("server %s doesn't exist? Run \"abra server ls\" to check", server))
+		}
+
+		for _, file := range files {
+			appName := strings.TrimSuffix(file.Name(), ".env")
+			appFilePath := path.Join(config.SERVERS_DIR, server, file.Name())
+			appFiles[appName] = AppFile{
+				Path:   appFilePath,
+				Server: server,
+			}
+		}
+	}
+
+	return appFiles, nil
+}
+
+// GetAppServiceNames retrieves a list of app service names.
+func GetAppServiceNames(appName string) ([]string, error) {
+	var serviceNames []string
+
+	appFiles, err := LoadAppFiles("")
+	if err != nil {
+		return serviceNames, err
+	}
+
+	app, err := GetApp(appFiles, appName)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	compose, err := GetAppComposeConfig(app.Recipe.Name, opts, app.Env)
+	if err != nil {
+		return serviceNames, err
+	}
+
+	for _, service := range compose.Services {
+		serviceNames = append(serviceNames, service.Name)
+	}
+
+	return serviceNames, nil
+}
+
+// GetAppNames retrieves a list of app names.
+func GetAppNames() ([]string, error) {
+	var appNames []string
+
+	appFiles, err := LoadAppFiles("")
+	if err != nil {
+		return appNames, err
+	}
+
+	apps, err := GetApps(appFiles, "")
+	if err != nil {
+		return appNames, err
+	}
+
+	for _, app := range apps {
+		appNames = append(appNames, app.Name)
+	}
+
+	return appNames, nil
+}
+
+// TemplateAppEnvSample copies the example env file for the app into the users
+// env files.
+func TemplateAppEnvSample(r recipe.Recipe, appName, server, domain string) error {
+	envSample, err := os.ReadFile(r.SampleEnvPath)
+	if err != nil {
+		return err
+	}
+
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
+	if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) {
+		return errors.New(i18n.G("%s already exists?", appEnvPath))
+	}
+
+	err = os.WriteFile(appEnvPath, envSample, 0o664)
+	if err != nil {
+		return err
+	}
+
+	read, err := os.ReadFile(appEnvPath)
+	if err != nil {
+		return err
+	}
+
+	newContents := strings.Replace(
+		string(read),
+		fmt.Sprintf("%s.example.com", r.Name),
+		domain,
+		-1,
+	)
+
+	err = os.WriteFile(appEnvPath, []byte(newContents), 0)
+	if err != nil {
+		return err
+	}
+
+	log.Debug(i18n.G("copied & templated %s to %s", r.SampleEnvPath, appEnvPath))
+
+	return nil
+}
+
+// SanitiseAppName makes a app name usable with Docker by replacing illegal
+// characters.
+func SanitiseAppName(name string) string {
+	return strings.ReplaceAll(name, ".", "_")
+}
+
+// GetAppStatuses queries servers to check the deployment status of given apps.
+func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) {
+	statuses := make(map[string]map[string]string)
+
+	servers := make(map[string]struct{})
+	for _, app := range apps {
+		if _, ok := servers[app.Server]; !ok {
+			servers[app.Server] = struct{}{}
+		}
+	}
+
+	var bar *progressbar.ProgressBar
+	if !MachineReadable {
+		bar = formatter.CreateProgressbar(len(servers), i18n.G("querying remote servers..."))
+	}
+
+	ch := make(chan stack.StackStatus, len(servers))
+	for server := range servers {
+		cl, err := client.New(server)
+		if err != nil {
+			log.Warn(err)
+			ch <- stack.StackStatus{}
+			continue
+		}
+
+		go func(s string) {
+			ch <- stack.GetAllDeployedServices(cl, s)
+			if !MachineReadable {
+				bar.Add(1)
+			}
+		}(server)
+	}
+
+	for range servers {
+		status := <-ch
+		if status.Err != nil {
+			return statuses, status.Err
+		}
+
+		for _, service := range status.Services {
+			result := make(map[string]string)
+			name := service.Spec.Labels[convert.LabelNamespace]
+
+			if _, ok := statuses[name]; !ok {
+				result["status"] = "deployed"
+			}
+
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name)
+			chaos, ok := service.Spec.Labels[labelKey]
+			if ok {
+				result["chaos"] = chaos
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name)
+			if chaosVersion, ok := service.Spec.Labels[labelKey]; ok {
+				result["chaosVersion"] = chaosVersion
+			}
+
+			labelKey = fmt.Sprintf("coop-cloud.%s.version", name)
+			if version, ok := service.Spec.Labels[labelKey]; ok {
+				result["version"] = version
+			} else {
+				continue
+			}
+
+			statuses[name] = result
+		}
+	}
+
+	log.Debug(i18n.G("retrieved app statuses: %s", statuses))
+
+	return statuses, nil
+}
+
+// GetAppComposeConfig retrieves a compose specification for a recipe. This
+// specification is the result of a merge of all the compose.**.yml files in
+// the recipe repository.
+func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv envfile.AppEnv) (*composetypes.Config, error) {
+	compose, err := loader.LoadComposefile(opts, appEnv)
+	if err != nil {
+		return &composetypes.Config{}, err
+	}
+
+	log.Debug(i18n.G("retrieved %s for %s", compose.Filename, recipe))
+
+	return compose, nil
+}
+
+// ExposeAllEnv exposes all env variables to the app container
+func ExposeAllEnv(
+	stackName string,
+	compose *composetypes.Config,
+	appEnv envfile.AppEnv,
+	toDeployVersion string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			log.Debug(i18n.G("adding env vars to %s service config", stackName))
+			for k, v := range appEnv {
+				_, exists := service.Environment[k]
+				if !exists {
+					value := v
+					if k == "TYPE" || k == "RECIPE" {
+						// NOTE(d1): don't use the wrong version from the app env
+						//           since we are deploying a new version
+						value = toDeployVersion
+					}
+					service.Environment[k] = &value
+					log.Debug(i18n.G("%s: %s: %s", stackName, k, value))
+				}
+			}
+		}
+	}
+}
+
+func CheckEnv(app App) ([]envfile.EnvVar, error) {
+	var envVars []envfile.EnvVar
+
+	envSample, err := app.Recipe.SampleEnv()
+	if err != nil {
+		return envVars, err
+	}
+
+	var keys []string
+	for key := range envSample {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		if _, ok := app.Env[key]; ok {
+			envVars = append(envVars, envfile.EnvVar{Name: key, Present: true})
+		} else {
+			envVars = append(envVars, envfile.EnvVar{Name: key, Present: false})
+		}
+	}
+
+	return envVars, nil
+}
+
+// ReadAbraShCmdNames reads the names of commands.
+func ReadAbraShCmdNames(abraSh string) ([]string, error) {
+	var cmdNames []string
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cmdNames, nil
+		}
+		return cmdNames, err
+	}
+	defer file.Close()
+
+	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
+	if err != nil {
+		return cmdNames, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := cmdNameRegex.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			cmdNames = append(cmdNames, matches[1])
+		}
+	}
+
+	if len(cmdNames) > 0 {
+		log.Debug(i18n.G("read %s from %s", strings.Join(cmdNames, " "), abraSh))
+	} else {
+		log.Debug(i18n.G("read 0 command names from %s", abraSh))
+	}
+
+	return cmdNames, nil
+}
+
+// Wipe removes the version from the app .env file.
+func (a App) WipeRecipeVersion() error {
+	file, err := os.Open(a.Path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	var (
+		lines   []string
+		scanner = bufio.NewScanner(file)
+	)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "RECIPE=") && !strings.HasPrefix(line, "TYPE=") {
+			lines = append(lines, line)
+			continue
+		}
+
+		if strings.HasPrefix(line, "#") {
+			lines = append(lines, line)
+			continue
+		}
+
+		splitted := strings.Split(line, ":")
+		lines = append(lines, splitted[0])
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := os.WriteFile(a.Path, []byte(strings.Join(lines, "\n")), os.ModePerm); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Debug(i18n.G("version wiped from %s.env", a.Domain))
+
+	return nil
+}
+
+// WriteRecipeVersion writes the recipe version to the app .env file.
+func (a App) WriteRecipeVersion(version string, dryRun bool) error {
+	if version == config.UNKNOWN_DEFAULT {
+		log.Debug(i18n.G("version is unknown, skipping env write"))
+		return nil
+	}
+
+	file, err := os.Open(a.Path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	var (
+		dirtyVersion string
+		skipped      bool
+		lines        []string
+		scanner      = bufio.NewScanner(file)
+	)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "RECIPE=") && !strings.HasPrefix(line, "TYPE=") {
+			lines = append(lines, line)
+			continue
+		}
+
+		if strings.HasPrefix(line, "#") {
+			lines = append(lines, line)
+			continue
+		}
+
+		if strings.Contains(line, version) && !a.Recipe.Dirty && !strings.HasSuffix(line, config.DIRTY_DEFAULT) {
+			skipped = true
+			lines = append(lines, line)
+			continue
+		}
+
+		splitted := strings.Split(line, ":")
+
+		line = fmt.Sprintf("%s:%s", splitted[0], version)
+		lines = append(lines, line)
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	if a.Recipe.Dirty && dirtyVersion != "" {
+		version = dirtyVersion
+	}
+
+	if !dryRun {
+		if err := os.WriteFile(a.Path, []byte(strings.Join(lines, "\n")), os.ModePerm); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		log.Debug(i18n.G("skipping writing version %s because dry run", version))
+	}
+
+	if !skipped {
+		log.Debug(i18n.G("version %s saved to %s.env", version, a.Domain))
+	} else {
+		log.Debug(i18n.G("skipping version %s write as already exists in %s.env", version, a.Domain))
+	}
+
+	return nil
+}

pkg/app/app_test.go

@@ -0,0 +1,239 @@
+package app_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
+	"coopcloud.tech/abra/pkg/recipe"
+	testPkg "coopcloud.tech/abra/pkg/test"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewApp(t *testing.T) {
+	app, err := appPkg.NewApp(testPkg.ExpectedAppEnv, testPkg.AppName, testPkg.ExpectedAppFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, testPkg.ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, testPkg.ExpectedApp)
+	}
+}
+
+func TestReadAppEnvFile(t *testing.T) {
+	app, err := appPkg.ReadAppEnvFile(testPkg.ExpectedAppFile, testPkg.AppName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, testPkg.ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, testPkg.ExpectedApp)
+	}
+}
+
+func TestGetApp(t *testing.T) {
+	app, err := appPkg.GetApp(testPkg.ExpectedAppFiles, testPkg.AppName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, testPkg.ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, testPkg.ExpectedApp)
+	}
+}
+
+func TestGetComposeFiles(t *testing.T) {
+	r := recipe.Get("abra-test-recipe")
+	err := r.EnsureExists()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		appEnv       map[string]string
+		composeFiles []string
+	}{
+		{
+			map[string]string{},
+			[]string{
+				fmt.Sprintf("%s/compose.yml", r.Dir),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml"},
+			[]string{
+				fmt.Sprintf("%s/compose.yml", r.Dir),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/compose.extra_secret.yml", r.Dir),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml:compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/compose.yml", r.Dir),
+				fmt.Sprintf("%s/compose.extra_secret.yml", r.Dir),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		composeFiles, err := r.GetComposeFiles(test.appEnv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, composeFiles, test.composeFiles)
+	}
+}
+
+func TestGetComposeFilesError(t *testing.T) {
+	r := recipe.Get("abra-test-recipe")
+	err := r.EnsureExists()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct{ appEnv map[string]string }{
+		{map[string]string{"COMPOSE_FILE": "compose.yml::compose.foo.yml"}},
+		{map[string]string{"COMPOSE_FILE": "doesnt.exist.yml"}},
+	}
+
+	for _, test := range tests {
+		_, err := r.GetComposeFiles(test.appEnv)
+		if err == nil {
+			t.Fatalf("should have failed: %v", test.appEnv)
+		}
+	}
+}
+
+func TestFilters(t *testing.T) {
+	oldDir := config.RECIPES_DIR
+	config.RECIPES_DIR = "./testdata"
+	defer func() {
+		config.RECIPES_DIR = oldDir
+	}()
+
+	app, err := appPkg.NewApp(envfile.AppEnv{
+		"DOMAIN": "test.example.com",
+		"RECIPE": "test-recipe",
+	}, "test_example_com", appPkg.AppFile{
+		Path:   "./testdata/filtertest.end",
+		Server: "local",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := app.Filters(false, false)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f, map[string]map[string]bool{
+		"name": {
+			"test_example_com": true,
+		},
+	})
+
+	f2, err := app.Filters(false, true)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f2, map[string]map[string]bool{
+		"name": {
+			"^test_example_com": true,
+		},
+	})
+
+	f3, err := app.Filters(true, false)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f3, map[string]map[string]bool{
+		"name": {
+			"test_example_com_bar": true,
+			"test_example_com_foo": true,
+		},
+	})
+
+	f4, err := app.Filters(true, true)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f4, map[string]map[string]bool{
+		"name": {
+			"^test_example_com_bar": true,
+			"^test_example_com_foo": true,
+		},
+	})
+
+	f5, err := app.Filters(false, false, "foo")
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f5, map[string]map[string]bool{
+		"name": {
+			"test_example_com_foo": true,
+		},
+	})
+}
+
+func compareFilter(t *testing.T, f1 filters.Args, f2 map[string]map[string]bool) {
+	t.Helper()
+	j1, err := f1.MarshalJSON()
+	if err != nil {
+		t.Error(err)
+	}
+	j2, err := json.Marshal(f2)
+	if err != nil {
+		t.Error(err)
+	}
+	if diff := cmp.Diff(string(j2), string(j1)); diff != "" {
+		t.Errorf("filters mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func TestWriteRecipeVersionOverwrite(t *testing.T) {
+	app, err := appPkg.GetApp(testPkg.ExpectedAppFiles, testPkg.AppName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer t.Cleanup(func() {
+		if err := app.WipeRecipeVersion(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	assert.Equal(t, "", app.Recipe.EnvVersion)
+
+	if err := app.WriteRecipeVersion("foo", false); err != nil {
+		t.Fatal(err)
+	}
+
+	app, err = appPkg.GetApp(testPkg.ExpectedAppFiles, testPkg.AppName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, "foo", app.Recipe.EnvVersion)
+}
+
+func TestWriteRecipeVersionUnknown(t *testing.T) {
+	app, err := appPkg.GetApp(testPkg.ExpectedAppFiles, testPkg.AppName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := app.WriteRecipeVersion(config.UNKNOWN_DEFAULT, false); err != nil {
+		t.Fatal(err)
+	}
+
+	assert.NotEqual(t, config.UNKNOWN_DEFAULT, app.Recipe.EnvVersion)
+}

pkg/app/compose.go

@@ -0,0 +1,90 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	composetypes "github.com/docker/cli/cli/compose/types"
+)
+
+// SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container
+// to signal which recipe is connected to the deployed app
+func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			log.Debug(i18n.G("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName))
+			labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName)
+			service.Deploy.Labels[labelKey] = recipe
+		}
+	}
+}
+
+// SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container
+// to signal if the app is deployed in chaos mode
+func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			log.Debug(i18n.G("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName))
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName)
+			service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos)
+		}
+	}
+}
+
+// SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container
+func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			log.Debug(i18n.G("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName))
+			labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName)
+			service.Deploy.Labels[labelKey] = chaosVersion
+		}
+	}
+}
+
+func SetVersionLabel(compose *composetypes.Config, stackName string, version string) {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			log.Debug(i18n.G("set label 'coop-cloud.%s.version' to %v for %s", stackName, version, stackName))
+			labelKey := fmt.Sprintf("coop-cloud.%s.version", stackName)
+			service.Deploy.Labels[labelKey] = version
+		}
+	}
+}
+
+// GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files
+func GetLabel(compose *composetypes.Config, stackName string, label string) string {
+	for _, service := range compose.Services {
+		if service.Name == "app" {
+			labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+			log.Debug(i18n.G("get label '%s'", labelKey))
+			if labelValue, ok := service.Deploy.Labels[labelKey]; ok {
+				return labelValue
+			}
+		}
+	}
+	log.Debug(i18n.G("no %s label found for %s", label, stackName))
+	return ""
+}
+
+// GetTimeoutFromLabel reads the timeout value from docker label
+// `coop-cloud.${STACK_NAME}.timeout=...` if present. A value is present if the
+// operator uses a `TIMEOUT=...` in their app env.
+func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
+	var timeout int
+
+	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
+		log.Debug(i18n.G("timeout label: %s", timeoutLabel))
+
+		var err error
+		timeout, err = strconv.Atoi(timeoutLabel)
+		if err != nil {
+			return timeout, errors.New(i18n.G("unable to convert timeout label %s to int: %s", timeoutLabel, err))
+		}
+	}
+
+	return timeout, nil
+}

pkg/app/compose_test.go

@@ -0,0 +1,62 @@
+package app_test
+
+import (
+	"testing"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	testPkg "coopcloud.tech/abra/pkg/test"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetTimeoutFromLabel(t *testing.T) {
+	testPkg.MkServerAppRecipe()
+	defer testPkg.RmServerAppRecipe()
+
+	tests := []struct {
+		configuredTimeout string
+		expectedTimeout   int
+	}{
+		{"0", 0},
+		{"DOESNTEXIST", 0}, // NOTE(d1): test when missing from .env
+		{"80", 80},
+		{"120", 120},
+	}
+
+	for _, test := range tests {
+		app, err := appPkg.GetApp(testPkg.ExpectedAppFiles, testPkg.AppName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if test.configuredTimeout != "DOESNTEXIST" {
+			app.Env["TIMEOUT"] = test.configuredTimeout
+		}
+
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    app.StackName(),
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
+		}
+
+		compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		timeout, err := appPkg.GetTimeoutFromLabel(compose, app.StackName())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, timeout, test.expectedTimeout)
+	}
+}

pkg/app/testdata/filtertest.env

@@ -0,0 +1,2 @@
+RECIPE=test-recipe
+DOMAIN=test.example.com

pkg/app/testdata/test-recipe/compose.yml

@@ -0,0 +1,6 @@
+version: "3.8"
+services:
+  foo:
+    image: debian
+  bar:
+    image: debian

pkg/autocomplete/autocomplete.go

@@ -0,0 +1,135 @@
+package autocomplete
+
+import (
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/app"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/spf13/cobra"
+)
+
+// AppNameComplete copletes app names.
+func AppNameComplete() ([]string, cobra.ShellCompDirective) {
+	appFiles, err := app.LoadAppFiles("")
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	var appNames []string
+	for appName := range appFiles {
+		appNames = append(appNames, appName)
+	}
+
+	return appNames, cobra.ShellCompDirectiveDefault
+}
+
+func ServiceNameComplete(appName string) ([]string, cobra.ShellCompDirective) {
+	serviceNames, err := app.GetAppServiceNames(appName)
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	return serviceNames, cobra.ShellCompDirectiveDefault
+}
+
+// RecipeNameComplete completes recipe names.
+func RecipeNameComplete() ([]string, cobra.ShellCompDirective) {
+	catl, err := recipe.ReadRecipeCatalogue(true)
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	localRecipes, err := recipe.GetRecipesLocal()
+	if err != nil && !strings.Contains(err.Error(), "empty") {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	var recipeNames []string
+	for name := range catl {
+		recipeNames = append(recipeNames, name)
+	}
+
+	for _, recipeLocal := range localRecipes {
+		recipeNames = append(recipeNames, recipeLocal)
+	}
+
+	return recipeNames, cobra.ShellCompDirectiveDefault
+}
+
+// RecipeVersionComplete completes versions for the recipe.
+func RecipeVersionComplete(recipeName string) ([]string, cobra.ShellCompDirective) {
+	catl, err := recipe.ReadRecipeCatalogue(true)
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	var recipeVersions []string
+	for _, v := range catl[recipeName].Versions {
+		for v2 := range v {
+			recipeVersions = append(recipeVersions, v2)
+		}
+	}
+
+	return recipeVersions, cobra.ShellCompDirectiveDefault
+}
+
+// ServerNameComplete completes server names.
+func ServerNameComplete() ([]string, cobra.ShellCompDirective) {
+	files, err := app.LoadAppFiles("")
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	var serverNames []string
+	for _, appFile := range files {
+		serverNames = append(serverNames, appFile.Server)
+	}
+
+	return serverNames, cobra.ShellCompDirectiveDefault
+}
+
+// CommandNameComplete completes recipe commands.
+func CommandNameComplete(appName string) ([]string, cobra.ShellCompDirective) {
+	app, err := app.Get(appName)
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	cmdNames, err := appPkg.ReadAbraShCmdNames(app.Recipe.AbraShPath)
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	sort.Strings(cmdNames)
+
+	return cmdNames, cobra.ShellCompDirectiveDefault
+}
+
+// SecretsComplete completes recipe secrets.
+func SecretComplete(recipeName string) ([]string, cobra.ShellCompDirective) {
+	r := recipe.Get(recipeName)
+
+	config, err := r.GetComposeConfig(nil)
+	if err != nil {
+		err := i18n.G("autocomplete failed: %s", err)
+		return []string{err}, cobra.ShellCompDirectiveError
+	}
+
+	var secretNames []string
+	for name := range config.Secrets {
+		secretNames = append(secretNames, name)
+	}
+
+	return secretNames, cobra.ShellCompDirectiveDefault
+}

pkg/catalogue/catalogue.go

@@ -0,0 +1,87 @@
+package catalogue
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/go-git/go-git/v5"
+)
+
+// EnsureCatalogue ensures that the catalogue is cloned locally & present.
+func EnsureCatalogue() error {
+	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
+	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		log.Debug(i18n.G("catalogue is missing, retrieving now"))
+
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
+		if err := gitPkg.Clone(catalogueDir, url); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// EnsureIsClean makes sure that the catalogue has no unstaged changes.
+func EnsureIsClean() error {
+	isClean, err := gitPkg.IsClean(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		return errors.New(i18n.G("%s has locally unstaged changes? please commit/remove your changes before proceeding", config.CATALOGUE_DIR))
+	}
+
+	return nil
+}
+
+// EnsureUpToDate ensures that the local catalogue is up to date.
+func EnsureUpToDate() error {
+	repo, err := git.PlainOpen(config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	remotes, err := repo.Remotes()
+	if err != nil {
+		return err
+	}
+
+	if len(remotes) == 0 {
+		log.Debug(i18n.G("cannot ensure %s is up-to-date, no git remotes configured", config.CATALOGUE_DIR))
+		return nil
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	branch, err := gitPkg.CheckoutDefaultBranch(repo, config.CATALOGUE_DIR)
+	if err != nil {
+		return err
+	}
+
+	opts := &git.PullOptions{
+		Force:         true,
+		ReferenceName: branch,
+	}
+
+	if err := worktree.Pull(opts); err != nil {
+		if !strings.Contains(err.Error(), "already up-to-date") {
+			return err
+		}
+	}
+
+	log.Debug(i18n.G("fetched latest git changes for %s", config.CATALOGUE_DIR))
+
+	return nil
+}

pkg/client/client.go

@@ -0,0 +1,127 @@
+// Package client provides Docker client initiatialisation functions.
+package client
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/pkg/config"
+	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	sshPkg "coopcloud.tech/abra/pkg/ssh"
+	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
+	"github.com/docker/docker/client"
+)
+
+// Conf is a Docker client configuration.
+type Conf struct {
+	Timeout int
+}
+
+// Opt is a Docker client option.
+type Opt func(c *Conf)
+
+// WithTimeout specifies a timeout for a Docker client.
+func WithTimeout(timeout int) Opt {
+	return func(c *Conf) {
+		c.Timeout = timeout
+	}
+}
+
+// New initiates a new Docker client. New client connections are validated so
+// that we ensure connections via SSH to the daemon can succeed. It takes into
+// account that you may only want the local client and not communicate via SSH.
+// For this use-case, please pass "default" as the serverName.
+func New(serverName string, opts ...Opt) (*client.Client, error) {
+	var clientOpts []client.Opt
+
+	ctx, err := GetContext(serverName)
+	if err != nil {
+		serverDir := path.Join(config.SERVERS_DIR, serverName)
+		if _, err := os.Stat(serverDir); err != nil {
+			return nil, errors.New(i18n.G("server missing, run \"abra server add %s\"?", serverName))
+		}
+
+		// NOTE(p4u1): when the docker context does not exist but the server folder
+		//             is there, let's create a new docker context.
+		if err = CreateContext(serverName); err != nil {
+			return nil, errors.New(i18n.G("server missing context, context creation failed: %s", err))
+		}
+
+		ctx, err = GetContext(serverName)
+		if err != nil {
+			return nil, errors.New(i18n.G("server missing context, run \"abra server add %s\"?", serverName))
+		}
+	}
+
+	ctxEndpoint, err := contextPkg.GetContextEndpoint(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var isUnix bool
+	if strings.Contains(ctxEndpoint, "unix://") {
+		isUnix = true
+	}
+
+	if serverName != "default" && !isUnix {
+		conf := &Conf{}
+
+		for _, opt := range opts {
+			opt(conf)
+		}
+
+		helper, err := commandconnPkg.NewConnectionHelper(ctxEndpoint, conf.Timeout)
+		if err != nil {
+			return nil, err
+		}
+
+		httpClient := &http.Client{
+			Transport: &http.Transport{
+				DialContext:     helper.Dialer,
+				IdleConnTimeout: 30 * time.Second,
+			},
+		}
+
+		clientOpts = append(clientOpts,
+			client.WithHTTPClient(httpClient),
+			client.WithHost(helper.Host),
+			client.WithDialContext(helper.Dialer),
+		)
+	}
+
+	version := os.Getenv("DOCKER_API_VERSION")
+	if version != "" {
+		clientOpts = append(clientOpts, client.WithVersion(version))
+	} else {
+		clientOpts = append(clientOpts, client.WithAPIVersionNegotiation())
+	}
+
+	cl, err := client.NewClientWithOpts(clientOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug(i18n.G("created client for %s", serverName))
+
+	info, err := cl.Info(context.Background())
+	if err != nil {
+		return cl, sshPkg.Fatal(serverName, err)
+	}
+
+	if info.Swarm.LocalNodeState == "inactive" {
+		if serverName != "default" && !isUnix {
+			return cl, errors.New(i18n.G("swarm mode not enabled on %s?", serverName))
+		}
+
+		return cl, errors.New(i18n.G("swarm mode not enabled on local server?"))
+	}
+
+	return cl, nil
+}

pkg/client/configs.go

@@ -0,0 +1,39 @@
+package client
+
+import (
+	"context"
+	"errors"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+)
+
+func GetConfigs(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]swarm.Config, error) {
+	configList, err := cl.ConfigList(ctx, swarm.ConfigListOptions{Filters: fs})
+	if err != nil {
+		return configList, err
+	}
+
+	return configList, nil
+}
+
+func GetConfigNames(configs []swarm.Config) []string {
+	var confNames []string
+
+	for _, conf := range configs {
+		confNames = append(confNames, conf.Spec.Name)
+	}
+
+	return confNames
+}
+
+func RemoveConfigs(cl *client.Client, ctx context.Context, configNames []string, force bool) error {
+	for _, confName := range configNames {
+		if err := cl.ConfigRemove(context.Background(), confName); err != nil {
+			return errors.New(i18n.G("conf %s: %s", confName, err))
+		}
+	}
+	return nil
+}

pkg/client/context.go

@@ -0,0 +1,89 @@
+package client
+
+import (
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
+	dConfig "github.com/docker/cli/cli/config"
+	"github.com/docker/cli/cli/context/docker"
+	contextStore "github.com/docker/cli/cli/context/store"
+)
+
+type Context = contextStore.Metadata
+
+// CreateContext creates a new Docker context.
+func CreateContext(contextName string) error {
+	host := fmt.Sprintf("ssh://%s", contextName)
+
+	if err := createContext(contextName, host); err != nil {
+		return err
+	}
+
+	log.Debug(i18n.G("created the %s context", contextName))
+
+	return nil
+}
+
+// createContext interacts with Docker Context to create a Docker context config
+func createContext(name string, host string) error {
+	s := context.NewDefaultDockerContextStore()
+	contextMetadata := contextStore.Metadata{
+		Endpoints: make(map[string]interface{}),
+		Name:      name,
+	}
+
+	contextTLSData := contextStore.ContextTLSData{
+		Endpoints: make(map[string]contextStore.EndpointTLSData),
+	}
+
+	dockerEP, dockerTLS, err := commandconnPkg.GetDockerEndpointMetadataAndTLS(host)
+	if err != nil {
+		return err
+	}
+
+	contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP
+	if dockerTLS != nil {
+		contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerTLS
+	}
+
+	if err := s.CreateOrUpdate(contextMetadata); err != nil {
+		return err
+	}
+
+	if err := s.ResetTLSMaterial(name, &contextTLSData); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DeleteContext(name string) error {
+	if name == "default" {
+		return errors.New(i18n.G("context 'default' cannot be removed"))
+	}
+
+	if _, err := GetContext(name); err != nil {
+		return err
+	}
+
+	cfg := dConfig.LoadDefaultConfigFile(nil)
+	cfg.CurrentContext = ""
+	if err := cfg.Save(); err != nil {
+		return err
+	}
+
+	return context.NewDefaultDockerContextStore().Remove(name)
+}
+
+func GetContext(contextName string) (contextStore.Metadata, error) {
+	ctx, err := context.NewDefaultDockerContextStore().GetMetadata(contextName)
+	if err != nil {
+		return contextStore.Metadata{}, err
+	}
+
+	return ctx, nil
+}

pkg/client/registry.go

@@ -0,0 +1,30 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/containers/image/docker"
+	"github.com/containers/image/types"
+	"github.com/distribution/reference"
+)
+
+// GetRegistryTags retrieves all tags of an image from a container registry.
+func GetRegistryTags(img reference.Named) ([]string, error) {
+	var tags []string
+
+	ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
+	if err != nil {
+		return tags, errors.New(i18n.G("failed to parse image %s, saw: %s", img, err.Error()))
+	}
+
+	ctx := context.Background()
+	tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
+	if err != nil {
+		return tags, err
+	}
+
+	return tags, nil
+}

pkg/client/secret.go

@@ -0,0 +1,27 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+)
+
+func StoreSecret(cl *client.Client, secretName, secretValue string) error {
+	ann := swarm.Annotations{Name: secretName}
+	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
+
+	if _, err := cl.SecretCreate(context.Background(), spec); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetSecretNames(secrets []swarm.Secret) []string {
+	var secretNames []string
+	for _, secret := range secrets {
+		secretNames = append(secretNames, secret.Spec.Name)
+	}
+	return secretNames
+}

pkg/client/secret_test.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetSecretNames(t *testing.T) {
+	tests := []struct {
+		name        string
+		secrets     []swarm.Secret
+		expected    []string
+		description string
+	}{
+		{
+			name:        "empty secrets list",
+			secrets:     []swarm.Secret{},
+			expected:    nil,
+			description: "should return nil for empty input",
+		},
+		{
+			name: "single secret",
+			secrets: []swarm.Secret{
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "database_password"}}},
+			},
+			expected:    []string{"database_password"},
+			description: "should return single secret name",
+		},
+		{
+			name: "multiple secrets",
+			secrets: []swarm.Secret{
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "db_password"}}},
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "api_key"}}},
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "ssl_cert"}}},
+			},
+			expected:    []string{"db_password", "api_key", "ssl_cert"},
+			description: "should return all secret names in order",
+		},
+		{
+			name: "secrets with empty names",
+			secrets: []swarm.Secret{
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: ""}}},
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "valid_name"}}},
+			},
+			expected:    []string{"", "valid_name"},
+			description: "should include empty names if present",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := GetSecretNames(tt.secrets)
+			assert.Equal(t, tt.expected, result, tt.description)
+		})
+	}
+}

pkg/client/volumes.go

@@ -0,0 +1,63 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
+)
+
+func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
+	volumeListOKBody, err := cl.VolumeList(ctx, volume.ListOptions{Filters: fs})
+	volumeList := volumeListOKBody.Volumes
+	if err != nil {
+		return volumeList, err
+	}
+
+	return volumeList, nil
+}
+
+func GetVolumeNames(volumes []*volume.Volume) []string {
+	var volumeNames []string
+
+	for _, vol := range volumes {
+		volumeNames = append(volumeNames, vol.Name)
+	}
+
+	return volumeNames
+}
+
+func RemoveVolumes(cl *client.Client, ctx context.Context, volumeNames []string, force bool, retries int) error {
+	for _, volName := range volumeNames {
+		err := retryFunc(5, func() error {
+			return cl.VolumeRemove(context.Background(), volName, force)
+		})
+		if err != nil {
+			return errors.New(i18n.G("volume %s: %s", volName, err))
+		}
+	}
+	return nil
+}
+
+// retryFunc retries the given function for the given retries. After the nth
+// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
+// It returns an error if the function still failed after the last retry.
+func retryFunc(retries int, fn func() error) error {
+	for i := 0; i < retries; i++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		if i+1 < retries {
+			sleep := time.Duration(i+1) * time.Duration(i+1)
+			log.Info(i18n.G("%s: waiting %d seconds before next retry", err, sleep))
+			time.Sleep(sleep * time.Second)
+		}
+	}
+	return errors.New(i18n.G("%d retries failed", retries))
+}

pkg/client/volumes_test.go

@@ -0,0 +1,26 @@
+package client
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestRetryFunc(t *testing.T) {
+	err := retryFunc(1, func() error { return nil })
+	if err != nil {
+		t.Errorf("should not return an error: %s", err)
+	}
+
+	i := 0
+	fn := func() error {
+		i++
+		return fmt.Errorf("oh no, something went wrong!")
+	}
+	err = retryFunc(2, fn)
+	if err == nil {
+		t.Error("should return an error")
+	}
+	if i != 2 {
+		t.Errorf("The function should have been called 1 times, got %d", i)
+	}
+}

pkg/config/abra.go

@@ -0,0 +1,122 @@
+package config
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"gopkg.in/yaml.v3"
+)
+
+// LoadAbraConfig returns the abra configuration. It tries to find a abra
+// configuration file (see findAbraConfig for lookup logic). When no
+// configuration was found it returns the default config.
+func LoadAbraConfig() Abra {
+	wd, _ := os.Getwd()
+	configFile := findAbraConfig(wd)
+	if configFile == "" {
+		log.Debug(i18n.G("no config file found"))
+		return Abra{}
+	}
+	data, err := os.ReadFile(configFile)
+	if err != nil {
+		// Do nothing, when an error occurs
+		log.Debug(i18n.G("error reading config file: %s", err))
+		return Abra{}
+	}
+
+	config := Abra{}
+	err = yaml.Unmarshal(data, &config)
+	if err != nil {
+		// Do nothing, when an error occurs
+		log.Debug(i18n.G("error loading config file: %s", err))
+		return Abra{}
+	}
+	log.Debug(i18n.G("config file loaded from: %s", configFile))
+	config.configPath = filepath.Dir(configFile)
+	return config
+}
+
+// findAbraConfig recursively looks for a abra.y(a)ml file in the given directory.
+// When the file was not found it calls the function again with the parent
+// directory until the home directory is hit. When no abra config was found it
+// returns an empty string.
+func findAbraConfig(dir string) string {
+	dir, err := filepath.Abs(dir)
+	if err != nil {
+		return ""
+	}
+	if dir == os.ExpandEnv("$HOME") || dir == "/" {
+		return ""
+	}
+	p := path.Join(dir, "abra.yaml")
+	if _, err := os.Stat(p); err == nil {
+		return p
+	}
+	p = path.Join(dir, "abra.yml")
+	if _, err := os.Stat(p); err == nil {
+		return p
+	}
+	return findAbraConfig(filepath.Dir(dir))
+}
+
+// Abra defines the configuration file for abra.
+type Abra struct {
+	configPath string
+	AbraDir    string `yaml:"abraDir"`
+}
+
+// GetAbraDir returns the abra dir. It has the following logic:
+// 1. check if $ABRA_DIR is set
+// 2. check if abraDir was set in a config file
+// 3. use $HOME/.abra when above two options failed
+func (a Abra) GetAbraDir() string {
+	if dir, exists := os.LookupEnv("ABRA_DIR"); exists && dir != "" {
+		log.Debug(i18n.G("read abra dir from $ABRA_DIR"))
+		return dir
+	}
+	if a.AbraDir != "" {
+		log.Debug(i18n.G("read abra dir from config file"))
+		if path.IsAbs(a.AbraDir) {
+			return a.AbraDir
+		}
+		// Make the path absolute
+		return path.Join(a.configPath, a.AbraDir)
+	}
+	log.Debug(i18n.G("using default abra dir"))
+	return os.ExpandEnv("$HOME/.abra")
+}
+
+func (a Abra) GetServersDir() string   { return path.Join(a.GetAbraDir(), "servers") }
+func (a Abra) GetRecipesDir() string   { return path.Join(a.GetAbraDir(), "recipes") }
+func (a Abra) GetLogsDir() string      { return path.Join(a.GetAbraDir(), "logs") }
+func (a Abra) GetCatalogueDir() string { return path.Join(a.GetAbraDir(), "catalogue") }
+
+var config = LoadAbraConfig()
+
+var (
+	ABRA_DIR                  = config.GetAbraDir()
+	SERVERS_DIR               = config.GetServersDir()
+	RECIPES_DIR               = config.GetRecipesDir()
+	LOGS_DIR                  = config.GetLogsDir()
+	CATALOGUE_DIR             = config.GetCatalogueDir()
+	RECIPES_JSON              = path.Join(config.GetCatalogueDir(), "recipes.json")
+	REPOS_BASE_URL            = "https://git.coopcloud.tech/coop-cloud"
+	CATALOGUE_JSON_REPO_NAME  = "recipes-catalogue-json"
+	TOOLSHED_SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/toolshed/%s.git"
+	RECIPES_SSH_URL_TEMPLATE  = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
+
+	// NOTE(d1): please note, this was done purely out of laziness on our part
+	// AFAICR. it's easy to punt the value into the label because that is what is
+	// expects. it's not particularly useful in terms of UI/UX but hey, nobody
+	// complained yet!
+	CHAOS_DEFAULT = "false"
+
+	DIRTY_DEFAULT = "+U"
+
+	MISSING_DEFAULT = "-"
+
+	UNKNOWN_DEFAULT = "unknown"
+)

pkg/config/abra_test.go

@@ -0,0 +1,133 @@
+package config
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestFindAbraConfig(t *testing.T) {
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	tests := []struct {
+		Dir    string
+		Config string
+	}{
+		{
+			Dir:    "testdata/abraconfig1",
+			Config: filepath.Join(wd, "testdata/abraconfig1/abra.yaml"),
+		},
+		{
+			Dir:    "testdata/abraconfig1/subdir",
+			Config: filepath.Join(wd, "testdata/abraconfig1/abra.yaml"),
+		},
+		{
+			Dir:    "testdata/abraconfig2",
+			Config: filepath.Join(wd, "testdata/abraconfig2/abra.yml"),
+		},
+		{
+			Dir:    "testdata/abraconfig2/subdir",
+			Config: filepath.Join(wd, "testdata/abraconfig2/abra.yml"),
+		},
+		{
+			Dir:    "testdata",
+			Config: "",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.Dir, func(t *testing.T) {
+			config := findAbraConfig(tc.Dir)
+			if config != tc.Config {
+				t.Errorf("\nwant: %s\ngot:  %s", tc.Config, config)
+			}
+		})
+	}
+}
+
+func TestLoadAbraConfigGetAbraDir(t *testing.T) {
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	t.Setenv("ABRA_DIR", "")
+
+	t.Run("default", func(t *testing.T) {
+		cfg := LoadAbraConfig()
+		wantAbraDir := os.ExpandEnv("$HOME/.abra")
+		if cfg.GetAbraDir() != wantAbraDir {
+			t.Errorf("\nwant: %s\ngot:  %s", wantAbraDir, cfg.GetAbraDir())
+		}
+	})
+
+	t.Run("from config file", func(t *testing.T) {
+		t.Cleanup(func() { os.Chdir(wd) })
+		err = os.Chdir(filepath.Join(wd, "testdata/abraconfig1"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		cfg := LoadAbraConfig()
+		wantAbraDir := filepath.Join(wd, "testdata/abraconfig1/foobar")
+		if cfg.GetAbraDir() != wantAbraDir {
+			t.Errorf("\nwant: %s\ngot:  %s", wantAbraDir, cfg.GetAbraDir())
+		}
+	})
+
+	t.Run("default when config file is empty", func(t *testing.T) {
+		t.Cleanup(func() { os.Chdir(wd) })
+		err := os.Chdir(filepath.Join(wd, "testdata/abraconfig2"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		cfg := LoadAbraConfig()
+		wantAbraDir := os.ExpandEnv("$HOME/.abra")
+		if cfg.GetAbraDir() != wantAbraDir {
+			t.Errorf("\nwant: %s\ngot:  %s", wantAbraDir, cfg.GetAbraDir())
+		}
+	})
+
+	t.Run("from env variable", func(t *testing.T) {
+		t.Setenv("ABRA_DIR", "foo")
+		cfg := LoadAbraConfig()
+		wantAbraDir := "foo"
+		if cfg.GetAbraDir() != wantAbraDir {
+			t.Errorf("\nwant: %s\ngot:  %s", wantAbraDir, cfg.GetAbraDir())
+		}
+	})
+}
+
+func TestLoadAbraConfigServersDir(t *testing.T) {
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	t.Setenv("ABRA_DIR", "")
+
+	t.Run("default", func(t *testing.T) {
+		cfg := LoadAbraConfig()
+		wantServersDir := os.ExpandEnv("$HOME/.abra/servers")
+		if cfg.GetServersDir() != wantServersDir {
+			t.Errorf("\nwant: %s\ngot:  %s", wantServersDir, cfg.GetServersDir())
+		}
+	})
+
+	t.Run("from config file", func(t *testing.T) {
+		t.Cleanup(func() { os.Chdir(wd) })
+		err = os.Chdir(filepath.Join(wd, "testdata/abraconfig1"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		cfg := LoadAbraConfig()
+		log.Println(cfg)
+		wantServersDir := filepath.Join(wd, "testdata/abraconfig1/foobar/servers")
+		if cfg.GetServersDir() != wantServersDir {
+			t.Errorf("\nwant: %s\ngot:  %s", wantServersDir, cfg.GetServersDir())
+		}
+	})
+}

pkg/config/env.go

@@ -0,0 +1,116 @@
+package config
+
+import (
+	"errors"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+)
+
+const MAX_SANITISED_APP_NAME_LENGTH = 45
+const MAX_DOCKER_SECRET_LENGTH = 64
+
+var BackupbotLabel = "coop-cloud.backupbot.enabled"
+
+// GetServers retrieves all servers.
+func GetServers() ([]string, error) {
+	var servers []string
+
+	servers, err := GetAllFoldersInDirectory(SERVERS_DIR)
+	if err != nil {
+		return servers, err
+	}
+
+	var filtered []string
+	for _, s := range servers {
+		if !strings.HasPrefix(s, ".") {
+			filtered = append(filtered, s)
+		}
+	}
+
+	log.Debug(i18n.G("retrieved %v servers: %s", len(filtered), filtered))
+
+	return filtered, nil
+}
+
+// ReadServerNames retrieves all server names.
+func ReadServerNames() ([]string, error) {
+	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
+
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug(i18n.G("read %s from %s", strings.Join(serverNames, ","), SERVERS_DIR))
+
+	return serverNames, nil
+}
+
+// GetAllFilesInDirectory returns filenames of all files in directory
+func GetAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
+	var realFiles []fs.FileInfo
+
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		// Follow any symlinks
+		filePath := path.Join(directory, file.Name())
+
+		if filepath.Ext(strings.TrimSpace(filePath)) != ".env" {
+			continue
+		}
+
+		realPath, err := filepath.EvalSymlinks(filePath)
+		if err != nil {
+			log.Warn(i18n.G("broken symlink in your abra config folders: %s", filePath))
+		} else {
+			realFile, err := os.Stat(realPath)
+			if err != nil {
+				return nil, err
+			}
+			if !realFile.IsDir() {
+				realFiles = append(realFiles, file)
+			}
+		}
+	}
+
+	return realFiles, nil
+}
+
+// GetAllFoldersInDirectory returns both folder and symlink paths
+func GetAllFoldersInDirectory(directory string) ([]string, error) {
+	var folders []string
+
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+	if len(files) == 0 {
+		return nil, errors.New(i18n.G("directory is empty: %s", directory))
+	}
+
+	for _, file := range files {
+		// Check if file is directory or symlink
+		if file.IsDir() || file.Mode()&fs.ModeSymlink != 0 {
+			filePath := path.Join(directory, file.Name())
+			realDir, err := filepath.EvalSymlinks(filePath)
+			if err != nil {
+				log.Warn(i18n.G("broken symlink in your abra config folders: %s", filePath))
+			} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
+				// path is a directory
+				folders = append(folders, file.Name())
+			}
+		}
+	}
+
+	return folders, nil
+}

pkg/config/testdata/abraconfig1/abra.yaml

@@ -0,0 +1 @@
+abraDir: foobar

pkg/container/container.go

@@ -0,0 +1,86 @@
+package container
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+)
+
+// GetContainer retrieves a container. If noInput is false and the retrievd
+// count of containers does not match 1, then a prompt is presented to let the
+// user choose. A count of 0 is handled gracefully.
+func GetContainer(c context.Context, cl *client.Client, filters filters.Args, noInput bool) (types.Container, error) {
+	containerOpts := containerTypes.ListOptions{Filters: filters}
+	containers, err := cl.ContainerList(c, containerOpts)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	if len(containers) == 0 {
+		filter := filters.Get("name")[0]
+		return types.Container{}, errors.New(i18n.G("no containers matching the %v filter found?", filter))
+	}
+
+	if len(containers) > 1 {
+		var containersRaw []string
+		for _, container := range containers {
+			containerName := strings.Join(container.Names, " ")
+			trimmed := strings.TrimPrefix(containerName, "/")
+			created := formatter.HumanDuration(container.Created)
+			containersRaw = append(containersRaw, i18n.G("%s (created %v)", trimmed, created))
+		}
+
+		if noInput {
+			err := errors.New(i18n.G("expected 1 container but found %v: %s", len(containers), strings.Join(containersRaw, " ")))
+			return types.Container{}, err
+		}
+
+		log.Warnf(i18n.G("ambiguous container list received, prompting for input"))
+
+		var response string
+		prompt := &survey.Select{
+			Message: i18n.G("which container are you looking for?"),
+			Options: containersRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return types.Container{}, err
+		}
+
+		chosenContainer := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, container := range containers {
+			containerName := strings.TrimSpace(strings.Join(container.Names, " "))
+			trimmed := strings.TrimPrefix(containerName, "/")
+			if trimmed == chosenContainer {
+				return container, nil
+			}
+		}
+
+		log.Fatal(i18n.G("failed to match chosen container"))
+	}
+
+	return containers[0], nil
+}
+
+// GetContainerFromStackAndService retrieves the container for the given stack and service.
+func GetContainerFromStackAndService(cl *client.Client, stack, service string) (types.Container, error) {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", stack, service))
+
+	container, err := GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	return container, nil
+}

pkg/context/context.go

@@ -0,0 +1,141 @@
+package context
+
+import (
+	"errors"
+	"os"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"github.com/docker/cli/cli/command"
+	dConfig "github.com/docker/cli/cli/config"
+	"github.com/docker/cli/cli/context"
+	"github.com/docker/cli/cli/context/docker"
+	"github.com/docker/cli/cli/context/store"
+	contextStore "github.com/docker/cli/cli/context/store"
+	cliflags "github.com/docker/cli/cli/flags"
+	dopts "github.com/docker/cli/opts"
+	"github.com/moby/moby/client"
+)
+
+func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
+	contextDir := dConfig.ContextStoreDir()
+	storeConfig := command.DefaultContextStoreConfig()
+	store := newContextStore(contextDir, storeConfig)
+
+	opts := &cliflags.ClientOptions{Context: "default"}
+
+	dockerContextStore := &command.ContextStoreWithDefault{
+		Store: store,
+		Resolver: func() (*command.DefaultContext, error) {
+			return resolveDefaultContext(opts, storeConfig)
+		},
+	}
+
+	return dockerContextStore
+}
+
+// resolveDefaultContext creates a Metadata for the current CLI invocation parameters
+func resolveDefaultContext(opts *cliflags.ClientOptions, config store.Config) (*command.DefaultContext, error) {
+	contextTLSData := store.ContextTLSData{
+		Endpoints: make(map[string]store.EndpointTLSData),
+	}
+
+	contextMetadata := store.Metadata{
+		Endpoints: make(map[string]any),
+		Metadata: command.DockerContext{
+			Description: "",
+		},
+		Name: "default",
+	}
+
+	dockerEP, err := resolveDefaultDockerEndpoint(opts)
+	if err != nil {
+		return nil, err
+	}
+
+	contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP.EndpointMeta
+	if dockerEP.TLSData != nil {
+		contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerEP.TLSData.ToStoreTLSData()
+	}
+
+	if err := config.ForeachEndpointType(func(n string, get store.TypeGetter) error {
+		if n == docker.DockerEndpoint { // handled above
+			return nil
+		}
+		ep := get()
+		if i, ok := ep.(command.EndpointDefaultResolver); ok {
+			meta, tls, err := i.ResolveDefault()
+			if err != nil {
+				return err
+			}
+			if meta == nil {
+				return nil
+			}
+			contextMetadata.Endpoints[n] = meta
+			if tls != nil {
+				contextTLSData.Endpoints[n] = *tls
+			}
+		}
+		// Nothing to be done
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return &command.DefaultContext{Meta: contextMetadata, TLS: contextTLSData}, nil
+}
+
+// Resolve the Docker endpoint for the default context (based on config, env vars and CLI flags)
+func resolveDefaultDockerEndpoint(opts *cliflags.ClientOptions) (docker.Endpoint, error) {
+	// defaultToTLS determines whether we should use a TLS host as default
+	// if nothing was configured by the user.
+	defaultToTLS := opts.TLSOptions != nil
+	host, err := getServerHost(opts.Hosts, defaultToTLS)
+	if err != nil {
+		return docker.Endpoint{}, err
+	}
+
+	var (
+		skipTLSVerify bool
+		tlsData       *context.TLSData
+	)
+
+	if opts.TLSOptions != nil {
+		skipTLSVerify = opts.TLSOptions.InsecureSkipVerify
+		tlsData, err = context.TLSDataFromFiles(opts.TLSOptions.CAFile, opts.TLSOptions.CertFile, opts.TLSOptions.KeyFile)
+		if err != nil {
+			return docker.Endpoint{}, err
+		}
+	}
+
+	return docker.Endpoint{
+		EndpointMeta: docker.EndpointMeta{
+			Host:          host,
+			SkipTLSVerify: skipTLSVerify,
+		},
+		TLSData: tlsData,
+	}, nil
+}
+
+func getServerHost(hosts []string, defaultToTLS bool) (string, error) {
+	switch len(hosts) {
+	case 0:
+		return dopts.ParseHost(defaultToTLS, os.Getenv(client.EnvOverrideHost))
+	case 1:
+		return dopts.ParseHost(defaultToTLS, hosts[0])
+	default:
+		return "", errors.New("specify only one -H")
+	}
+}
+
+func GetContextEndpoint(ctx contextStore.Metadata) (string, error) {
+	endpointmeta, ok := ctx.Endpoints["docker"].(context.EndpointMetaBase)
+	if !ok {
+		err := errors.New(i18n.G("context lacks Docker endpoint"))
+		return "", err
+	}
+	return endpointmeta.Host, nil
+}
+
+func newContextStore(dir string, config contextStore.Config) contextStore.Store {
+	return contextStore.New(dir, config)
+}

pkg/deploy/utils.go

@@ -0,0 +1,316 @@
+package deploy
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"strings"
+
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/envfile"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+
+	"github.com/distribution/reference"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types/swarm"
+	dockerClient "github.com/docker/docker/client"
+)
+
+// MergeAbraShEnv merges abra.sh env vars into the app env vars.
+func MergeAbraShEnv(recipe recipe.Recipe, env envfile.AppEnv) error {
+	abraShEnv, err := envfile.ReadAbraShEnvVars(recipe.AbraShPath)
+	if err != nil {
+		return err
+	}
+
+	for k, v := range abraShEnv {
+		log.Debugf(i18n.G("read v:%s k: %s", v, k))
+		env[k] = v
+	}
+
+	return nil
+}
+
+// GetEntityNameAndVersion parses a full name like `app_example_com_someconf_v1` to extract name and version, ("someconf", "v1")
+func GetEntityNameAndVersion(fullName string, stackName string) (string, string, error) {
+	name := strings.TrimPrefix(fullName, stackName+"_")
+	if lastUnderscore := strings.LastIndex(name, "_"); lastUnderscore != -1 {
+		return name[0:lastUnderscore], name[lastUnderscore+1:], nil
+	}
+	return "", "", errors.New(i18n.G("can't parse version from '%s'", fullName))
+}
+
+func GetSecretsForStack(cl *dockerClient.Client, app appPkg.App) (map[string]string, error) {
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// List all services in the stack
+	// NOTE: we could do cl.SecretList, but we want to know which secrets are actually attached
+	services, err := cl.ServiceList(context.Background(), swarm.ServiceListOptions{
+		Filters: filters,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	secrets := make(map[string]string)
+
+	for _, service := range services {
+		if service.Spec.TaskTemplate.ContainerSpec.Secrets != nil {
+			for _, secretRef := range service.Spec.TaskTemplate.ContainerSpec.Secrets {
+				secretName := secretRef.SecretName
+				if secretName == "" {
+					continue
+				}
+				secretBaseName, secretVersion, err := GetEntityNameAndVersion(secretName, app.StackName())
+				if err != nil {
+					log.Warn(err)
+					continue
+				}
+
+				existingSecretVersion, exists := secrets[secretBaseName]
+				if !exists {
+					// First time seeing this, add to map
+					secrets[secretBaseName] = secretVersion
+				} else {
+					// Just make sure the versions are the same..
+					if existingSecretVersion != secretVersion {
+						log.Warnf(i18n.G("different versions for secret '%s', '%s' and %s'", secretBaseName, existingSecretVersion, secretVersion))
+					}
+				}
+			}
+		}
+	}
+	return secrets, nil
+}
+
+// GetConfigsForStack retrieves all Docker configs attached to services in a given stack.
+func GetConfigsForStack(cl *dockerClient.Client, app appPkg.App) (map[string]string, error) {
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// List all services in the stack
+	services, err := cl.ServiceList(context.Background(), swarm.ServiceListOptions{
+		Filters: filters,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Collect unique config names with versions
+	configs := make(map[string]string)
+	for _, service := range services {
+		if service.Spec.TaskTemplate.ContainerSpec != nil {
+			for _, configRef := range service.Spec.TaskTemplate.ContainerSpec.Configs {
+				configName := configRef.ConfigName
+				if configName == "" {
+					continue
+				}
+				configBaseName, configVersion, err := GetEntityNameAndVersion(configName, app.StackName())
+				if err != nil {
+					log.Warn(err)
+					continue
+				}
+
+				existingConfigVersion, ok := configs[configBaseName]
+				if !ok {
+					// First time seeing this, add to map
+					configs[configBaseName] = configVersion
+				} else {
+					// Just make sure the versions are the same..
+					if existingConfigVersion != configVersion {
+						log.Warnf(i18n.G("different versions for config '%s', '%s' and %s'", configBaseName, existingConfigVersion, configVersion))
+					}
+				}
+			}
+		}
+	}
+
+	return configs, nil
+}
+
+// GetImagesForStack retrieves all Docker images for services in a given stack.
+func GetImagesForStack(cl *dockerClient.Client, app appPkg.App) (map[string]string, error) {
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// List all services in the stack
+	services, err := cl.ServiceList(context.Background(), swarm.ServiceListOptions{
+		Filters: filters,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Collect unique image names with versions
+	images := make(map[string]string)
+	for _, service := range services {
+		if service.Spec.TaskTemplate.ContainerSpec != nil {
+			imageName := service.Spec.TaskTemplate.ContainerSpec.Image
+
+			imageParsed, err := reference.ParseNormalizedNamed(imageName)
+			if err != nil {
+				log.Warn(err)
+				continue
+			}
+
+			imageBaseName := reference.Path(imageParsed)
+			imageTag := imageParsed.(reference.NamedTagged).Tag()
+
+			existingImageVersion, ok := images[imageBaseName]
+			if !ok {
+				// First time seeing this, add to map
+				images[imageBaseName] = imageTag
+			} else {
+				// Just make sure the versions are the same..
+				if existingImageVersion != imageTag {
+					log.Warnf(i18n.G("different versions for image '%s', '%s' and %s'", imageBaseName, existingImageVersion, imageTag))
+				}
+			}
+		}
+	}
+
+	return images, nil
+}
+
+func GatherSecretsForDeploy(cl *dockerClient.Client, app appPkg.App, showUnchanged bool) ([]string, error) {
+	// Get current secrets from existing deployment
+	currentSecrets, err := GetSecretsForStack(cl, app)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf(i18n.G("current secrets: %v", currentSecrets))
+
+	newSecrets, err := secret.PollSecretsStatus(cl, app)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf(i18n.G("new secrets: %v", newSecrets))
+
+	// Sort secrets to ensure reproducible output
+	sort.Slice(newSecrets, func(i, j int) bool {
+		return newSecrets[i].LocalName < newSecrets[j].LocalName
+	})
+
+	var secretInfo []string
+
+	for _, newSecret := range newSecrets {
+		if currentVersion, exists := currentSecrets[newSecret.LocalName]; exists {
+			if currentVersion == newSecret.Version {
+				if showUnchanged {
+					secretInfo = append(secretInfo, i18n.G("%s: %s (unchanged)", newSecret.LocalName, newSecret.Version))
+				}
+			} else {
+				secretInfo = append(secretInfo, i18n.G("%s: %s → %s", newSecret.LocalName, currentVersion, newSecret.Version))
+			}
+		} else {
+			secretInfo = append(secretInfo, i18n.G("%s: %s (new)", newSecret.LocalName, newSecret.Version))
+		}
+	}
+
+	return secretInfo, nil
+}
+
+func GatherConfigsForDeploy(cl *dockerClient.Client, app appPkg.App, compose *composetypes.Config, abraShEnv map[string]string, showUnchanged bool) ([]string, error) {
+	// Get current configs from existing deployment
+	currentConfigs, err := GetConfigsForStack(cl, app)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf(i18n.G("deployed config names: %v", currentConfigs))
+
+	// Get new configs from the compose specification
+	newConfigs := compose.Configs
+
+	var configInfo []string
+	for configName := range newConfigs {
+		log.Debugf(i18n.G("searching abra.sh for version for %s", configName))
+		versionKey := strings.ToUpper(configName) + "_VERSION"
+		newVersion, exists := abraShEnv[versionKey]
+		if !exists {
+			log.Warnf(i18n.G("no version found for config %s", configName))
+			configInfo = append(configInfo, i18n.G("%s: ? (missing version)", configName))
+			continue
+		}
+
+		if currentVersion, exists := currentConfigs[configName]; exists {
+			if currentVersion == newVersion {
+				if showUnchanged {
+					configInfo = append(configInfo, i18n.G("%s: %s (unchanged)", configName, newVersion))
+				}
+			} else {
+				configInfo = append(configInfo, i18n.G("%s: %s → %s", configName, currentVersion, newVersion))
+			}
+		} else {
+			configInfo = append(configInfo, i18n.G("%s: %s (new)", configName, newVersion))
+		}
+	}
+
+	return configInfo, nil
+}
+
+func GatherImagesForDeploy(cl *dockerClient.Client, app appPkg.App, compose *composetypes.Config, showUnchanged bool) ([]string, error) {
+	// Get current images from existing deployment
+	currentImages, err := GetImagesForStack(cl, app)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf(i18n.G("deployed images: %v", currentImages))
+
+	// Proposed new images from the compose files
+	newImages := make(map[string]string)
+
+	for _, service := range compose.Services {
+		imageParsed, err := reference.ParseNormalizedNamed(service.Image)
+		if err != nil {
+			log.Warn(err)
+			continue
+		}
+
+		imageBaseName := reference.Path(imageParsed)
+		imageTag := imageParsed.(reference.NamedTagged).Tag()
+
+		existingImageVersion, ok := newImages[imageBaseName]
+		if !ok {
+			// First time seeing this, add to map
+			newImages[imageBaseName] = imageTag
+		} else {
+			// Just make sure the versions are the same..
+			if existingImageVersion != imageTag {
+				log.Warnf(i18n.G("different versions for image '%s', '%s' and %s'", imageBaseName, existingImageVersion, imageTag))
+			}
+		}
+	}
+	log.Debugf(i18n.G("proposed images: %v", newImages))
+
+	var imageInfo []string
+	for newImageName, newImageVersion := range newImages {
+		if currentVersion, exists := currentImages[newImageName]; exists {
+			if currentVersion == newImageVersion {
+				if showUnchanged {
+					imageInfo = append(imageInfo, i18n.G("%s: %s (unchanged)", formatter.StripTagMeta(newImageName), newImageVersion))
+				}
+			} else {
+				imageInfo = append(imageInfo, i18n.G("%s: %s → %s", formatter.StripTagMeta(newImageName), currentVersion, newImageVersion))
+			}
+		} else {
+			imageInfo = append(imageInfo, i18n.G("%s: %s (new)", formatter.StripTagMeta(newImageName), newImageVersion))
+		}
+	}
+
+	return imageInfo, nil
+}

pkg/deploy/utils_test.go

@@ -0,0 +1,89 @@
+package deploy
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetEntityNameAndVersion(t *testing.T) {
+	tests := []struct {
+		name        string
+		fullName    string
+		stackName   string
+		expected    string
+		expectedVer string
+		expectError bool
+	}{
+		{
+			name:        "valid config with version",
+			fullName:    "myapp_database_v2",
+			stackName:   "myapp",
+			expected:    "database",
+			expectedVer: "v2",
+			expectError: false,
+		},
+		{
+			name:        "valid config with numeric version",
+			fullName:    "myapp_redis_1",
+			stackName:   "myapp",
+			expected:    "redis",
+			expectedVer: "1",
+			expectError: false,
+		},
+		{
+			name:        "config without underscore in name",
+			fullName:    "myapp_db_v1",
+			stackName:   "myapp",
+			expected:    "db",
+			expectedVer: "v1",
+			expectError: false,
+		},
+		{
+			name:        "config with multiple underscores",
+			fullName:    "myapp_my_database_v3",
+			stackName:   "myapp",
+			expected:    "my_database",
+			expectedVer: "v3",
+			expectError: false,
+		},
+		{
+			name:        "invalid config - no version",
+			fullName:    "myapp_database",
+			stackName:   "myapp",
+			expectError: true,
+		},
+		{
+			name:        "empty config name",
+			fullName:    "myapp__v1",
+			stackName:   "myapp",
+			expected:    "",
+			expectedVer: "v1",
+			expectError: false,
+		},
+		{
+			name:        "wrong stack prefix",
+			fullName:    "otherapp_database_v1",
+			stackName:   "myapp",
+			expected:    "otherapp_database",
+			expectedVer: "v1",
+			expectError: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			name, version, err := GetEntityNameAndVersion(tt.fullName, tt.stackName)
+
+			if tt.expectError {
+				assert.Error(t, err)
+				assert.Empty(t, name)
+				assert.Empty(t, version)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expected, name)
+				assert.Equal(t, tt.expectedVer, version)
+			}
+		})
+	}
+}

pkg/dns/dns.go

@@ -0,0 +1,60 @@
+package dns
+
+import (
+	"errors"
+	"net"
+
+	"coopcloud.tech/abra/pkg/i18n"
+)
+
+// EnsureIPv4 ensures that an ipv4 address is set for a domain name
+func EnsureIPv4(domainName string) (string, error) {
+	ipv4, err := net.ResolveIPAddr("ip4", domainName)
+	if err != nil {
+		return "", errors.New(i18n.G("%s: unable to resolve IPv4 address: %s", domainName, err))
+	}
+
+	if ipv4 == nil {
+		return "", errors.New(i18n.G("%s: no IPv4 available", domainName))
+	}
+
+	return ipv4.String(), nil
+}
+
+// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address
+func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
+	if server == "default" || server == "local" {
+		return "", nil
+	}
+
+	var ipv4 string
+
+	domainIPv4, err := EnsureIPv4(domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if domainIPv4 == "" {
+		return ipv4, errors.New(i18n.G("cannot resolve ipv4 for %s?", domainName))
+	}
+
+	serverIPv4, err := EnsureIPv4(server)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if serverIPv4 == "" {
+		return ipv4, errors.New(i18n.G("cannot resolve ipv4 for %s?", server))
+	}
+
+	if domainIPv4 != serverIPv4 {
+		return ipv4, errors.New(
+			i18n.G(
+				"app domain %s (%s) does not appear to resolve to app server %s (%s)?",
+				domainName, domainIPv4, server, serverIPv4,
+			),
+		)
+	}
+
+	return ipv4, nil
+}

pkg/dns/dns_test.go

@@ -0,0 +1,64 @@
+package dns
+
+import (
+	"fmt"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
+	tests := []struct {
+		domainName     string
+		serverName     string
+		shouldValidate bool
+	}{
+		// NOTE(d1): DNS records get checked, so use something that is maintained
+		// within the federation. if you're here because of a failing test, try
+		// `dig +short <domain>` to ensure stuff matches first! If flakyness
+		// becomes an issue we can look into mocking
+		{"docs.coopcloud.tech", "swarm-0.coopcloud.tech", true},
+		{"docs.coopcloud.tech", "coopcloud.tech", true},
+
+		// NOTE(d1): special case handling for "--local"
+		{"", "default", true},
+		{"", "local", true},
+
+		{"", "", false},
+		{"123", "", false},
+	}
+
+	for _, test := range tests {
+		_, err := EnsureDomainsResolveSameIPv4(test.domainName, test.serverName)
+		if err != nil && test.shouldValidate {
+			t.Fatal(err)
+		}
+
+		if err == nil && !test.shouldValidate {
+			t.Fatal(fmt.Errorf("should have failed but did not: %v", test))
+		}
+	}
+}
+
+func TestEnsureIpv4(t *testing.T) {
+	// NOTE(d1): DNS records get checked, so use something that is maintained
+	// within the federation. if you're here because of a failing test, try `dig
+	// +short <domain>` to ensure stuff matches first! If flakyness becomes an
+	// issue we can look into mocking
+	domainName := "collabora.ostrom.collective.tools"
+	serverName := "ostrom.collective.tools"
+
+	for i := 0; i < 15; i++ {
+		domainIpv4, err := EnsureIPv4(domainName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		serverIpv4, err := EnsureIPv4(serverName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, domainIpv4, serverIpv4)
+	}
+}

pkg/envfile/envfile.go

@@ -0,0 +1,91 @@
+package envfile
+
+import (
+	"bufio"
+	"errors"
+	"os"
+	"regexp"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/i18n"
+	"coopcloud.tech/abra/pkg/log"
+	"git.coopcloud.tech/toolshed/godotenv"
+)
+
+// AppEnv is a map of the values in an apps env config
+type AppEnv = map[string]string
+
+// AppModifiers is a map of modifiers in an apps env config
+type AppModifiers = map[string]map[string]string
+
+// ReadEnv loads an app envivornment into a map.
+func ReadEnv(filePath string) (AppEnv, error) {
+	var envVars AppEnv
+
+	envVars, _, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return envVars, nil
+}
+
+// ReadEnv loads an app envivornment and their modifiers in two different maps.
+func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) {
+	var envVars AppEnv
+
+	envVars, mods, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, mods, err
+	}
+
+	log.Debug(i18n.G("read %s from %s", envVars, filePath))
+
+	return envVars, mods, nil
+}
+
+// ReadAbraShEnvVars reads env vars from an abra.sh recipe file.
+func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
+	envVars := make(map[string]string)
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return envVars, nil
+		}
+		return envVars, err
+	}
+	defer file.Close()
+
+	exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`)
+	if err != nil {
+		return envVars, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		txt := scanner.Text()
+		if exportRegex.MatchString(txt) {
+			splitVals := strings.Split(txt, "export ")
+			envVarDef := splitVals[len(splitVals)-1]
+			keyVal := strings.Split(envVarDef, "=")
+			if len(keyVal) != 2 {
+				return envVars, errors.New(i18n.G("couldn't parse %s", txt))
+			}
+			envVars[keyVal[0]] = keyVal[1]
+		}
+	}
+
+	if len(envVars) > 0 {
+		log.Debug(i18n.G("read %s from %s", envVars, abraSh))
+	} else {
+		log.Debug(i18n.G("read 0 env var exports from %s", abraSh))
+	}
+
+	return envVars, nil
+}
+
+type EnvVar struct {
+	Name    string
+	Present bool
+}