Update the server used to deploy the installer script

See #547

.drone.yml

@@ -3,17 +3,17 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.21
+    image: golang:1.24
     commands:
       - make check
 
   - name: make test
-    image: golang:1.21
+    image: golang:1.24
     environment:
-      ABRA_DIR: "/root/.abra"
+      CATL_URL: https://git.coopcloud.tech/toolshed/recipes-catalogue-json.git
     commands:
-      - make build-abra
-      - ./abra help # show version, initialise $ABRA_DIR
+      - mkdir -p $HOME/.abra
+      - git clone $CATL_URL $HOME/.abra/catalogue
       - make test
     depends_on:
       - make check
@@ -29,7 +29,7 @@ steps:
       event: tag
 
   - name: release
-    image: goreleaser/goreleaser:v1.24.0
+    image: goreleaser/goreleaser:v2.5.1
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token
@@ -47,19 +47,72 @@ steps:
     image: plugins/docker
     settings:
       auto_tag: true
-      username: 3wordchant
+      username: abra-bot
       password:
-        from_secret: git_coopcloud_tech_token_3wc
-      repo: git.coopcloud.tech/coop-cloud/abra
+        from_secret: git_coopcloud_tech_token_abra_bot
+      repo: git.coopcloud.tech/toolshed/abra
       tags: dev
       registry: git.coopcloud.tech
     when:
-      event:
-        exclude:
-          - pull_request
+      branch:
+        - main
     depends_on:
       - make check
+      - make test
+
+  - name: on-demand integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      request_pty: true
+      script:
+        - |
+          wget https://git.coopcloud.tech/toolshed/abra/raw/branch/main/scripts/tests/run-ci-int -O run-ci-int
+          chmod +x run-ci-int
+          sh run-ci-int
+    when:
+      ref:
+        - refs/heads/int-*
+    depends_on:
+      - make check
+      - make test
+
+  - name: nightly integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      request_pty: true
+      script:
+        - |
+          wget https://git.coopcloud.tech/toolshed/abra/raw/branch/main/scripts/tests/run-ci-int -O run-ci-int
+          chmod +x run-ci-int
+          sh run-ci-int
+    when:
+      event:
+        - cron:
+      cron:
+        # @daily https://docs.drone.io/cron/
+        - integration 
 
 volumes:
   - name: deps
     temp: {}
+
+trigger:
+  action:
+    exclude:
+      - synchronized

.envrc.sample

@@ -1,7 +1,7 @@
-go env -w GOPRIVATE=coopcloud.tech
-
-# export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
-
+# integration test suite
 # export ABRA_DIR="$HOME/.abra_test"
 # export ABRA_TEST_DOMAIN=test.example.com
-# export ABRA_SKIP_TEARDOWN=1 # for faster feedback when developing tests
+# export ABRA_CI=1
+
+# release automation
+# export GITEA_TOKEN=

.gitea/ISSUE_TEMPLATE.md

@@ -1,8 +0,0 @@
----
-name: "Do not use this issue tracker"
-about: "Do not use this issue tracker"
-title: "Do not use this issue tracker"
-labels: []
----
-
-Please report your issue on [`coop-cloud/organising`](https://git.coopcloud.tech/coop-cloud/organising)

.gitignore

@@ -2,8 +2,7 @@
 .e2e.env
 .envrc
 .vscode/
+/abra
 /kadabra
-abra
 dist/
 tests/integration/.bats
-vendor/

.goreleaser.yml

@@ -29,6 +29,8 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
+      - "-s"
+      - "-w"
 
   - id: kadabra
     binary: kadabra
@@ -47,9 +49,13 @@ builds:
       - 5
       - 6
       - 7
+    gcflags:
+      - "all=-l -B"
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
+      - "-s"
+      - "-w"
 
 checksum:
   name_template: "checksums.txt"

AUTHORS.md

@@ -4,9 +4,11 @@
 > please do add yourself! This is a community project, let's show some 💞
 
 - 3wordchant
+- ammaratef45
 - cassowary
 - codegod100
 - decentral1se
+- fauno
 - frando
 - kawaiipunk
 - knoflook
@@ -16,3 +18,5 @@
 - roxxers
 - vera
 - yksflip
+- basebuilder
+- mayel

Dockerfile

@@ -1,23 +1,29 @@
-FROM golang:1.21-alpine AS build
+# Build image
+FROM golang:1.24-alpine AS build
 
-ENV GOPRIVATE coopcloud.tech
+ENV GOPRIVATE=coopcloud.tech
 
 RUN apk add --no-cache \
-  ca-certificates \
   gcc \
   git \
   make \
   musl-dev
 
-RUN update-ca-certificates
-
 COPY . /app
 
 WORKDIR /app
 
 RUN CGO_ENABLED=0 make build
 
-FROM scratch
+# Release image ("slim")
+FROM alpine:3.19.1
+
+RUN apk add --no-cache \
+  ca-certificates \
+  git \
+  openssh
+
+RUN update-ca-certificates
 
 COPY --from=build /app/abra /abra
 

Makefile

@@ -2,9 +2,10 @@ ABRA         := ./cmd/abra
 KADABRA      := ./cmd/kadabra
 COMMIT       := $(shell git rev-list -1 HEAD)
 GOPATH       := $(shell go env GOPATH)
-GOVERSION    := 1.21
+GOVERSION    := 1.24
 LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+GCFLAGS      := "all=-l -B"
 
 export GOPRIVATE=coopcloud.tech
 
@@ -12,22 +13,24 @@ export GOPRIVATE=coopcloud.tech
 all: format check build-abra test
 
 run-abra:
-	@go run -ldflags=$(LDFLAGS) $(ABRA)
+	@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
 
 run-kadabra:
-	@go run -ldflags=$(LDFLAGS) $(KADABRA)
+	@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(KADABRA)
 
 install-abra:
-	@go install -ldflags=$(LDFLAGS) $(ABRA)
+	@go install -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
 
 install-kadabra:
-	@go install -ldflags=$(LDFLAGS) $(KADABRA)
+	@go install -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(KADABRA)
+
+install: install-abra install-kadabra
 
 build-abra:
-	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -v -gcflags=$(GCFLAGS) -ldflags=$(DIST_LDFLAGS) $(ABRA)
 
 build-kadabra:
-	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
+	@go build -v -gcflags=$(GCFLAGS) -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
 build: build-abra build-kadabra
 
@@ -42,10 +45,10 @@ clean:
 	@rm '$(GOPATH)/bin/kadabra'
 
 format:
-	@gofmt -s -w .
+	@gofmt -s -w $$(find . -type f -name '*.go' | grep -v "/vendor/")
 
 check:
-	@test -z $$(gofmt -l .) || \
+	@test -z $$(gofmt -l $$(find . -type f -name '*.go' | grep -v "/vendor/")) || \
 	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
 
 test:
@@ -53,3 +56,6 @@ test:
 
 loc:
 	@find . -name "*.go" | xargs wc -l
+
+deps:
+	@go get -t -u ./...

README.md

@@ -1,7 +1,7 @@
 # `abra`
 
-[![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
-[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
+[![Build Status](https://build.coopcloud.tech/api/badges/toolshed/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/toolshed/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/toolshed/abra)](https://goreportcard.com/report/git.coopcloud.tech/toolshed/abra)
 [![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
 
 The Co-op Cloud utility belt 🎩🐇

cli/app/app.go

@@ -1,37 +1,11 @@
 package app
 
 import (
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var AppCommand = cli.Command{
-	Name:        "app",
-	Aliases:     []string{"a"},
-	Usage:       "Manage apps",
-	ArgsUsage:   "<domain>",
-	Description: "Functionality for managing the life cycle of your apps",
-	Subcommands: []cli.Command{
-		appBackupCommand,
-		appCheckCommand,
-		appCmdCommand,
-		appConfigCommand,
-		appCpCommand,
-		appDeployCommand,
-		appErrorsCommand,
-		appListCommand,
-		appLogsCommand,
-		appNewCommand,
-		appPsCommand,
-		appRemoveCommand,
-		appRestartCommand,
-		appRestoreCommand,
-		appRollbackCommand,
-		appRunCommand,
-		appSecretCommand,
-		appServicesCommand,
-		appUndeployCommand,
-		appUpgradeCommand,
-		appVersionCommand,
-		appVolumeCommand,
-	},
+var AppCommand = &cobra.Command{
+	Use:     "app [cmd] [args] [flags]",
+	Aliases: []string{"a"},
+	Short:   "Manage apps",
 }

cli/app/backup.go

@@ -1,414 +1,307 @@
 package app
 
 import (
-	"archive/tar"
-	"context"
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/klauspost/pgzip"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
 )
 
-type backupConfig struct {
-	preHookCmd  string
-	postHookCmd string
-	backupPaths []string
+var AppBackupListCommand = &cobra.Command{
+	Use:     "list <domain> [flags]",
+	Aliases: []string{"ls"},
+	Short:   "List the contents of a snapshot",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if snapshot != "" {
+			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if showAllPaths {
+			log.Debugf("including SHOW_ALL=%v in backupbot exec invocation", showAllPaths)
+			execEnv = append(execEnv, fmt.Sprintf("SHOW_ALL=%v", showAllPaths))
+		}
+
+		if timestamps {
+			log.Debugf("including TIMESTAMPS=%v in backupbot exec invocation", timestamps)
+			execEnv = append(execEnv, fmt.Sprintf("TIMESTAMPS=%v", timestamps))
+		}
+
+		if _, err = internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-var appBackupCommand = cli.Command{
-	Name:      "backup",
-	Aliases:   []string{"bk"},
-	Usage:     "Run app backup",
-	ArgsUsage: "<domain> [<service>]",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		internal.ChaosFlag,
+var AppBackupDownloadCommand = &cobra.Command{
+	Use:     "download <domain> [flags]",
+	Aliases: []string{"d"},
+	Short:   "Download a snapshot",
+	Long: `Downloads a backup.tar.gz to the current working directory.
+
+"--volumes/-v" includes data contained in volumes alongide paths specified in
+"backupbot.backup.path" labels.`,
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app backup.
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		backupConfigs := make(map[string]backupConfig)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName != "" {
-			backupConfig, ok := backupConfigs[serviceName]
-			if !ok {
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
-
-			logrus.Infof("running backup for the %s service", serviceName)
-
-			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if len(backupConfigs) == 0 {
-				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
-			}
-
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
-
-				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
 		}
 
-		return nil
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if snapshot != "" {
+			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if includePath != "" {
+			log.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
+
+		if includeSecrets {
+			log.Debugf("including SECRETS=%v in backupbot exec invocation", includeSecrets)
+			execEnv = append(execEnv, fmt.Sprintf("SECRETS=%v", includeSecrets))
+		}
+
+		if includeVolumes {
+			log.Debugf("including VOLUMES=%v in backupbot exec invocation", includeVolumes)
+			execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%v", includeVolumes))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+
+		remoteBackupDir := "/tmp/backup.tar.gz"
+		currentWorkingDir := "."
+		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
+			log.Fatal(err)
+		}
 	},
 }
 
-// TimeStamp generates a file name friendly timestamp.
-func TimeStamp() string {
-	ts := time.Now().UTC().Format(time.RFC3339)
-	return strings.Replace(ts, ":", "-", -1)
-}
+var AppBackupCreateCommand = &cobra.Command{
+	Use:     "create <domain> [flags]",
+	Aliases: []string{"c"},
+	Short:   "Create a new snapshot",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-// runBackup does the actual backup logic.
-func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-		}
-		defer content.Close()
-
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			log.Fatal(err)
 		}
 
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
-	}
-
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
-
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
 		}
 
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
+		if retries != "" {
+			log.Debugf("including RETRIES=%s in backupbot exec invocation", retries)
+			execEnv = append(execEnv, fmt.Sprintf("RETRIES=%s", retries))
+		}
 
-	return nil
+		if _, err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
+var AppBackupSnapshotsCommand = &cobra.Command{
+	Use:     "snapshots <domain> [flags]",
+	Aliases: []string{"s"},
+	Short:   "List all snapshots",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
 		}
 
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	return nil
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if _, err = internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
-
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
-		}
-	}
-
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
-		}
-	}
-
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
-
-	return nil
+var AppBackupCommand = &cobra.Command{
+	Use:     "backup [cmd] [args] [flags]",
+	Aliases: []string{"b"},
+	Short:   "Manage app backups",
 }
 
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
+var (
+	snapshot       string
+	retries        string
+	includePath    string
+	showAllPaths   bool
+	timestamps     bool
+	includeSecrets bool
+	includeVolumes bool
+)
 
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
+func init() {
+	AppBackupListCommand.Flags().StringVarP(
+		&snapshot,
+		"snapshot",
+		"s",
+		"",
+		"list specific snapshot",
+	)
 
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
-			}
-			break
-		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
-		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
-}
-
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
-
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
-
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
-
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
-
-	rc = fin
-	tr = tar.NewReader(rc)
-
-	return tr, rc, nil
+	AppBackupListCommand.Flags().BoolVarP(
+		&showAllPaths,
+		"all",
+		"a",
+		false,
+		"show all paths",
+	)
+
+	AppBackupListCommand.Flags().BoolVarP(
+		&timestamps,
+		"timestamps",
+		"t",
+		false,
+		"include timestamps",
+	)
+
+	AppBackupDownloadCommand.Flags().StringVarP(
+		&snapshot,
+		"snapshot",
+		"s",
+		"",
+		"list specific snapshot",
+	)
+
+	AppBackupDownloadCommand.Flags().StringVarP(
+		&includePath,
+		"path",
+		"p",
+		"",
+		"volumes path",
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&includeSecrets,
+		"secrets",
+		"S",
+		false,
+		"include secrets",
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&includeVolumes,
+		"volumes",
+		"v",
+		false,
+		"include volumes",
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+
+	AppBackupCreateCommand.Flags().StringVarP(
+		&retries,
+		"retries",
+		"r",
+		"1",
+		"number of retry attempts",
+	)
+
+	AppBackupCreateCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
 }

cli/app/check.go

@@ -1,23 +1,22 @@
 package app
 
 import (
+	"fmt"
+
 	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/spf13/cobra"
 )
 
-var appCheckCommand = cli.Command{
-	Name:    "check",
+var AppCheckCommand = &cobra.Command{
+	Use:     "check <domain> [flags]",
 	Aliases: []string{"chk"},
-	Usage:   "Ensure an app is well configured",
-	Description: `
-This command compares env vars in both the app ".env" and recipe ".env.sample"
-file.
+	Short:   "Ensure an app is well configured",
+	Long: `Compare env vars in both the app ".env" and recipe ".env.sample" file.
 
 The goal is to ensure that recipe ".env.sample" env vars are defined in your
 app ".env" file. Only env var definitions in the ".env.sample" which are
@@ -27,55 +26,66 @@ these env vars, then "check" will complain.
 Recipe maintainers may or may not provide defaults for env vars within their
 recipes regardless of commenting or not (e.g. through the use of
 ${FOO:<default>} syntax). "check" does not confirm or deny this for you.`,
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.ChaosFlag,
-		internal.OfflineFlag,
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		tableCol := []string{"recipe env sample", "app env"}
-		table := formatter.CreateTable(tableCol)
-
-		envVars, err := config.CheckEnv(app)
+		table, err := formatter.CreateTable()
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
+		}
+
+		table.
+			Headers(
+				fmt.Sprintf("%s .env.sample", app.Recipe.Name),
+				fmt.Sprintf("%s.env", app.Name),
+			).
+			StyleFunc(func(row, col int) lipgloss.Style {
+				switch {
+				case col == 1:
+					return lipgloss.NewStyle().Padding(0, 1, 0, 1).Align(lipgloss.Center)
+				default:
+					return lipgloss.NewStyle().Padding(0, 1, 0, 1)
+				}
+			})
+
+		envVars, err := appPkg.CheckEnv(app)
+		if err != nil {
+			log.Fatal(err)
 		}
 
 		for _, envVar := range envVars {
 			if envVar.Present {
-				table.Append([]string{envVar.Name, "✅"})
+				val := []string{envVar.Name, "✅"}
+				table.Row(val...)
 			} else {
-				table.Append([]string{envVar.Name, "❌"})
+				val := []string{envVar.Name, "❌"}
+				table.Row(val...)
 			}
 		}
 
-		table.Render()
-
-		return nil
+		if err := formatter.PrintTable(table); err != nil {
+			log.Fatal(err)
+		}
 	},
 }
+
+func init() {
+	AppCheckCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+}

cli/app/cmd.go

@@ -5,106 +5,117 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path"
+	"slices"
 	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/app"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
 )
 
-var appCmdCommand = cli.Command{
-	Name:    "command",
+var AppCmdCommand = &cobra.Command{
+	Use:     "command <domain> [service | --local] <cmd> [[args] [flags] | [flags] -- [args]]",
 	Aliases: []string{"cmd"},
-	Usage:   "Run app commands",
-	Description: `Run an app specific command.
+	Short:   "Run app commands",
+	Long: `Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
-work station by passing "--local". Arguments can be passed into these functions
-using the "-- <args>" syntax.
+work station by passing "--local/-l".
 
-Example:
+N.B. If using the "--" style to pass arguments, flags (e.g. "--local/-l") must
+be passed *before* the "--". It is possible to pass arguments without the "--"
+as long as no dashes are present (i.e. "foo" works without "--", "-foo"
+does not).`,
+	Example: `  # pass <cmd> args/flags without "--"
+  abra app cmd 1312.net app my_cmd_arg foo --user bar
 
-  abra app cmd example.com app create_user -- me@example.com
-`,
-	ArgsUsage: "<domain> [<service>] <command> [-- <args>]",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.LocalCmdFlag,
-		internal.RemoteUserFlag,
-		internal.TtyFlag,
-		internal.OfflineFlag,
-		internal.ChaosFlag,
-	},
-	Before:      internal.SubCommandBefore,
-	Subcommands: []cli.Command{appCmdListCommand},
-	BashComplete: func(ctx *cli.Context) {
-		args := ctx.Args()
-		switch len(args) {
-		case 0:
-			autocomplete.AppNameComplete(ctx)
-		case 1:
-			autocomplete.ServiceNameComplete(args.Get(0))
-		case 2:
-			cmdNameComplete(args.Get(0))
-		}
-	},
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+  # pass <cmd> args/flags with "--"
+  abra app cmd 1312.net app my_cmd_args --user bar -- foo -vvv
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
+  # drop the [service] arg if using "--local/-l"
+  abra app cmd 1312.net my_cmd --local`,
+	Args: func(cmd *cobra.Command, args []string) error {
+		if local {
+			if !(len(args) >= 2) {
+				return errors.New("requires at least 2 arguments with --local/-l")
 			}
 
-			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
+			if slices.Contains(os.Args, "--") {
+				if cmd.ArgsLenAtDash() > 2 {
+					return errors.New("accepts at most 2 args with --local/-l")
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
+			// NOTE(d1): it is unclear how to correctly validate this case
+			//
+			// abra app cmd 1312.net app test_cmd_args foo --local
+			// FATAL <recipe> doesn't have a app function
+			//
+			// "app" should not be there, but there is no reliable way to detect arg
+			// count when the user can pass an arbitrary amount of recipe command
+			// arguments
+			return nil
+		}
+
+		if !(len(args) >= 3) {
+			return errors.New("requires at least 3 arguments")
+		}
+
+		return nil
+	},
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			if !local {
+				return autocomplete.ServiceNameComplete(args[0])
 			}
+			return autocomplete.CommandNameComplete(args[0])
+		case 2:
+			if !local {
+				return autocomplete.CommandNameComplete(args[0])
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		if internal.LocalCmd && internal.RemoteUser != "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
+		if local && remoteUser != "" {
+			log.Fatal("cannot use --local & --user together")
 		}
 
-		hasCmdArgs, parsedCmdArgs := parseCmdArgs(c.Args(), internal.LocalCmd)
+		hasCmdArgs, parsedCmdArgs := parseCmdArgs(args, local)
 
-		abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
-		if _, err := os.Stat(abraSh); err != nil {
+		if _, err := os.Stat(app.Recipe.AbraShPath); err != nil {
 			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist for %s?", abraSh, app.Name)
+				log.Fatalf("%s does not exist for %s?", app.Recipe.AbraShPath, app.Name)
 			}
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		if internal.LocalCmd {
-			if !(len(c.Args()) >= 2) {
-				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+		if local {
+			cmdName := args[1]
+			if err := internal.EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+				log.Fatal(err)
 			}
 
-			cmdName := c.Args().Get(1)
-			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
-				logrus.Fatal(err)
-			}
-
-			logrus.Debugf("--local detected, running %s on local work station", cmdName)
+			log.Debugf("--local detected, running %s on local work station", cmdName)
 
 			var exportEnv string
 			for k, v := range app.Env {
@@ -113,70 +124,95 @@ Example:
 
 			var sourceAndExec string
 			if hasCmdArgs {
-				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName, parsedCmdArgs)
+				log.Debugf("parsed following command arguments: %s", parsedCmdArgs)
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, app.Recipe.AbraShPath, cmdName, parsedCmdArgs)
 			} else {
-				logrus.Debug("did not detect any command arguments")
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, abraSh, cmdName)
+				log.Debug("did not detect any command arguments")
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, app.Recipe.AbraShPath, cmdName)
 			}
 
 			shell := "/bin/bash"
 			if _, err := os.Stat(shell); errors.Is(err, os.ErrNotExist) {
-				logrus.Debugf("%s does not exist locally, use /bin/sh as fallback", shell)
+				log.Debugf("%s does not exist locally, use /bin/sh as fallback", shell)
 				shell = "/bin/sh"
 			}
 			cmd := exec.Command(shell, "-c", sourceAndExec)
 
 			if err := internal.RunCmd(cmd); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if !(len(c.Args()) >= 3) {
-				internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments"))
+				log.Fatal(err)
 			}
 
-			targetServiceName := c.Args().Get(1)
+			return
+		}
 
-			cmdName := c.Args().Get(2)
-			if err := internal.EnsureCommand(abraSh, app.Recipe, cmdName); err != nil {
-				logrus.Fatal(err)
-			}
+		cmdName := args[2]
+		if err := internal.EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+			log.Fatal(err)
+		}
 
-			serviceNames, err := config.GetAppServiceNames(app.Name)
-			if err != nil {
-				logrus.Fatal(err)
-			}
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-			matchingServiceName := false
-			for _, serviceName := range serviceNames {
-				if serviceName == targetServiceName {
-					matchingServiceName = true
-				}
-			}
-
-			if !matchingServiceName {
-				logrus.Fatalf("no service %s for %s?", targetServiceName, app.Name)
-			}
-
-			logrus.Debugf("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName)
-
-			if hasCmdArgs {
-				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-			} else {
-				logrus.Debug("did not detect any command arguments")
-			}
-
-			cl, err := client.New(app.Server)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if err := internal.RunCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
-				logrus.Fatal(err)
+		matchingServiceName := false
+		targetServiceName := args[1]
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
 			}
 		}
 
-		return nil
+		if !matchingServiceName {
+			log.Fatalf("no service %s for %s?", targetServiceName, app.Name)
+		}
+
+		log.Debugf("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName)
+
+		if hasCmdArgs {
+			log.Debugf("parsed following command arguments: %s", parsedCmdArgs)
+		} else {
+			log.Debug("did not detect any command arguments")
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := internal.RunCmdRemote(
+			cl,
+			app,
+			disableTTY,
+			app.Recipe.AbraShPath,
+			targetServiceName, cmdName, parsedCmdArgs, remoteUser); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+var AppCmdListCommand = &cobra.Command{
+	Use:     "list <domain> [flags]",
+	Aliases: []string{"ls"},
+	Short:   "List all available commands",
+	Args:    cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cmdNames, err := appPkg.ReadAbraShCmdNames(app.Recipe.AbraShPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		sort.Strings(cmdNames)
+
+		for _, cmdName := range cmdNames {
+			fmt.Println(cmdName)
+		}
 	},
 }
 
@@ -199,75 +235,42 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 	return hasCmdArgs, parsedCmdArgs
 }
 
-func cmdNameComplete(appName string) {
-	app, err := app.Get(appName)
-	if err != nil {
-		return
-	}
-	cmdNames, _ := getShCmdNames(app)
-	if err != nil {
-		return
-	}
-	for _, n := range cmdNames {
-		fmt.Println(n)
-	}
-}
-
-var appCmdListCommand = cli.Command{
-	Name:      "list",
-	Aliases:   []string{"ls"},
-	Usage:     "List all available commands",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		internal.ChaosFlag,
-	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cmdNames, err := getShCmdNames(app)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		for _, cmdName := range cmdNames {
-			fmt.Println(cmdName)
-		}
-
-		return nil
-	},
-}
-
-func getShCmdNames(app config.App) ([]string, error) {
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
-	if err != nil {
-		return nil, err
-	}
-
-	sort.Strings(cmdNames)
-	return cmdNames, nil
+var (
+	local      bool
+	remoteUser string
+	disableTTY bool
+)
+
+func init() {
+	AppCmdCommand.Flags().BoolVarP(
+		&local,
+		"local",
+		"l",
+		false,
+		"run command locally",
+	)
+
+	AppCmdCommand.Flags().StringVarP(
+		&remoteUser,
+		"user",
+		"u",
+		"",
+		"request remote user",
+	)
+
+	AppCmdCommand.Flags().BoolVarP(
+		&disableTTY,
+		"tty",
+		"T",
+		false,
+		"disable remote TTY",
+	)
+
+	AppCmdCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
 }

cli/app/cmd_test.go

@@ -13,7 +13,7 @@ func TestParseCmdArgs(t *testing.T) {
 	}{
 		// `--` is not parsed when passed in from the command-line e.g. -- foo bar baz
 		// so we need to eumlate that as missing when testing if bash args are passed in
-		// see https://git.coopcloud.tech/coop-cloud/organising/issues/336 for more
+		// see https://git.coopcloud.tech/toolshed/organising/issues/336 for more
 		{[]string{"foo.com", "app", "test"}, false, ""},
 		{[]string{"foo.com", "app", "test", "foo"}, true, "foo "},
 		{[]string{"foo.com", "app", "test", "foo", "bar", "baz"}, true, "foo bar baz "},

cli/app/config.go

@@ -1,64 +1,57 @@
 package app
 
 import (
-	"errors"
 	"os"
 	"os/exec"
 
-	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/log"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appConfigCommand = cli.Command{
-	Name:      "config",
-	Aliases:   []string{"cfg"},
-	Usage:     "Edit app config",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
+var AppConfigCommand = &cobra.Command{
+	Use:     "config <domain> [flags]",
+	Aliases: []string{"cfg"},
+	Short:   "Edit app config",
+	Example: "  abra config 1312.net",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		appName := c.Args().First()
-
-		if appName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no app provided"))
-		}
-
-		files, err := config.LoadAppFiles("")
+	Run: func(cmd *cobra.Command, args []string) {
+		files, err := appPkg.LoadAppFiles("")
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
+		appName := args[0]
 		appFile, exists := files[appName]
 		if !exists {
-			logrus.Fatalf("cannot find app with name %s", appName)
+			log.Fatalf("cannot find app with name %s", appName)
 		}
 
 		ed, ok := os.LookupEnv("EDITOR")
 		if !ok {
 			edPrompt := &survey.Select{
-				Message: "Which editor do you wish to use?",
+				Message: "which editor do you wish to use?",
 				Options: []string{"vi", "vim", "nvim", "nano", "pico", "emacs"},
 			}
 			if err := survey.AskOne(edPrompt, &ed); err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 		}
 
-		cmd := exec.Command(ed, appFile.Path)
-		cmd.Stdin = os.Stdin
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if err := cmd.Run(); err != nil {
-			logrus.Fatal(err)
+		c := exec.Command(ed, appFile.Path)
+		c.Stdin = os.Stdin
+		c.Stdout = os.Stdout
+		c.Stderr = os.Stderr
+		if err := c.Run(); err != nil {
+			log.Fatal(err)
 		}
-
-		return nil
 	},
 }

cli/app/cp.go

@@ -15,76 +15,70 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/log"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
+	containertypes "github.com/docker/docker/api/types/container"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appCpCommand = cli.Command{
-	Name:      "cp",
-	Aliases:   []string{"c"},
-	ArgsUsage: "<domain> <src> <dst>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
+var AppCpCommand = &cobra.Command{
+	Use:     "cp <domain> <src> <dst> [flags]",
+	Aliases: []string{"c"},
+	Short:   "Copy files to/from a deployed app service",
+	Example: `  # copy myfile.txt to the root of the app service
+  abra app cp 1312.net myfile.txt app:/
+
+  # copy that file back to your current working directory locally
+  abra app cp 1312.net app:/myfile.txt ./`,
+	Args: cobra.ExactArgs(3),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
 	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Copy files to/from a deployed app service",
-	Description: `
-Copy files to and from any app service file system.
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-If you want to copy a myfile.txt to the root of the app service:
-
-    abra app cp <domain> myfile.txt app:/
-
-And if you want to copy that file back to your current working directory locally:
-
-    abra app cp <domain> app:/myfile.txt .
-`,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		src := c.Args().Get(1)
-		dst := c.Args().Get(2)
-		if src == "" {
-			logrus.Fatal("missing <src> argument")
-		}
-		if dst == "" {
-			logrus.Fatal("missing <dest> argument")
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
+		src := args[1]
+		dst := args[2]
 		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
 		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
-		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+		log.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 		if toContainer {
-			err = copyToContainer(cl, container.ID, srcPath, dstPath)
+			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
 		} else {
-			err = copyFromContainer(cl, container.ID, srcPath, dstPath)
+			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
 		}
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
-
-		return nil
 	},
 }
 
@@ -106,9 +100,9 @@ func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service st
 	return "", "", "", false, errServiceMissing
 }
 
-// copyToContainer copies a file or directory from the local file system to the container.
+// CopyToContainer copies a file or directory from the local file system to the container.
 // See the possible copy modes and their documentation.
-func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := os.Stat(srcPath)
 	if err != nil {
 		return fmt.Errorf("local %s ", err)
@@ -140,7 +134,7 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if _, err := container.RunExec(dcli, cl, containerID, &containertypes.ExecOptions{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -167,8 +161,8 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		return err
 	}
 
-	logrus.Debugf("copy %s from local to %s on container", srcPath, dstPath)
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	log.Debugf("copy %s from local to %s on container", srcPath, dstPath)
+	copyOpts := containertypes.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
 	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
 		return err
 	}
@@ -179,7 +173,7 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if _, err := container.RunExec(dcli, cl, containerID, &containertypes.ExecOptions{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -194,9 +188,9 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 	return nil
 }
 
-// copyFromContainer copies a file or directory from the given container to the local file system.
+// CopyFromContainer copies a file or directory from the given container to the local file system.
 // See the possible copy modes and their documentation.
-func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
 	if err != nil {
 		if errdefs.IsNotFound(err) {
@@ -377,3 +371,13 @@ func moveFile(sourcePath, destPath string) error {
 	}
 	return nil
 }
+
+func init() {
+	AppCpCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+}

cli/app/deploy.go

@@ -3,252 +3,348 @@ package app
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/envfile"
 	"coopcloud.tech/abra/pkg/secret"
 
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/lint"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/log"
 	"coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
 )
 
-var appDeployCommand = cli.Command{
-	Name:      "deploy",
-	Aliases:   []string{"d"},
-	Usage:     "Deploy an app",
-	ArgsUsage: "<domain> [<version>]",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.ForceFlag,
-		internal.ChaosFlag,
-		internal.NoDomainChecksFlag,
-		internal.DontWaitConvergeFlag,
-		internal.OfflineFlag,
+var AppDeployCommand = &cobra.Command{
+	Use:     "deploy <domain> [version] [flags]",
+	Aliases: []string{"d"},
+	Short:   "Deploy an app",
+	Long: `Deploy an app.
+
+This command supports chaos operations. Use "--chaos/-C" to deploy your recipe
+checkout as-is. Recipe commit hashes are also supported as values for
+"[version]". Please note, "upgrade"/"rollback" do not support chaos operations.`,
+	Example: `  # standard deployment
+  abra app deploy 1312.net
+
+  # chaos deployment
+  abra app deploy 1312.net --chaos
+  
+  # deploy specific version
+  abra app deploy 1312.net 2.0.0+1.2.3
+
+  # deploy a specific git hash
+  abra app deploy 1312.net 886db76d`,
+	Args: cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string,
+	) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				errMsg := fmt.Sprintf("autocomplete failed: %s", err)
+				return []string{errMsg}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.RecipeVersionComplete(app.Recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
 	},
-	Before: internal.SubCommandBefore,
-	Description: `
-Deploy an app. It does not support incrementing the version of a deployed app,
-for this you need to look at the "abra app upgrade <domain>" command.
+	Run: func(cmd *cobra.Command, args []string) {
+		var (
+			deployWarnMessages []string
+			toDeployVersion    string
+		)
 
-You may pass "--force" to re-deploy the same version again. This can be useful
-if the container runtime has gotten into a weird state.
+		app := internal.ValidateApp(args)
 
-Chaos mode ("--chaos") will deploy your local checkout of a recipe as-is,
-including unstaged changes and can be useful for live hacking and testing new
-recipes.
-`,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-		stackName := app.StackName()
-
-		specificVersion := c.Args().Get(1)
-		if specificVersion != "" && internal.Chaos {
-			logrus.Fatal("cannot use <version> and --chaos together")
+		if err := validateArgsAndFlags(args); err != nil {
+			log.Fatal(err)
 		}
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		r, err := recipe.Get(app.Recipe, internal.Offline)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := lint.LintForErrors(r); err != nil {
-			logrus.Fatal(err)
-		}
-
-		logrus.Debugf("checking whether %s is already deployed", stackName)
-
 		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+		log.Debugf("checking whether %s is already deployed", app.StackName())
+
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		secStats, err := secret.PollSecretsStatus(cl, app)
+		if deployMeta.IsDeployed && !(internal.Force || internal.Chaos) {
+			log.Fatalf("%s is already deployed", app.Name)
+		}
+
+		toDeployVersion, err = getDeployVersion(args, deployMeta, app)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(fmt.Errorf("get deploy version: %s", err))
 		}
 
-		for _, secStat := range secStats {
-			if !secStat.CreatedOnRemote {
-				logrus.Fatalf("unable to deploy, secrets not generated (%s)?", secStat.LocalName)
-			}
-		}
-
-		if isDeployed {
-			if internal.Force || internal.Chaos {
-				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
-			} else {
-				logrus.Fatalf("%s is already deployed", app.Name)
-			}
-		}
-
-		version := deployedVersion
-		if specificVersion != "" {
-			version = specificVersion
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		if !internal.Chaos && specificVersion == "" {
-			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if !internal.Chaos {
+			_, err = app.Recipe.EnsureVersion(toDeployVersion)
 			if err != nil {
-				logrus.Fatal(err)
-			}
-			versions, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			if len(versions) == 0 && !internal.Chaos {
-				logrus.Warn("no published versions in catalogue, trying local recipe repository")
-				recipeVersions, err := recipe.GetRecipeVersions(app.Recipe, internal.Offline)
-				if err != nil {
-					logrus.Warn(err)
-				}
-				for _, recipeVersion := range recipeVersions {
-					for version := range recipeVersion {
-						versions = append(versions, version)
-					}
-				}
-			}
-
-			if len(versions) > 0 && !internal.Chaos {
-				version = versions[len(versions)-1]
-				logrus.Debugf("choosing %s as version to deploy", version)
-				if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-					logrus.Fatal(err)
-				}
-			} else {
-				head, err := git.GetRecipeHead(app.Recipe)
-				if err != nil {
-					logrus.Fatal(err)
-				}
-				version = formatter.SmallSHA(head.String())
-				logrus.Warn("no versions detected, using latest commit")
+				log.Fatalf("ensure recipe: %s", err)
 			}
 		}
 
-		if internal.Chaos {
-			logrus.Warnf("chaos mode engaged")
-			var err error
-			version, err = recipe.ChaosVersion(app.Recipe)
-			if err != nil {
-				logrus.Fatal(err)
-			}
+		if err := lint.LintForErrors(app.Recipe); err != nil {
+			log.Fatal(err)
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err := validateSecrets(cl, app); err != nil {
+			log.Fatal(err)
+		}
+
+		abraShEnv, err := envfile.ReadAbraShEnvVars(app.Recipe.AbraShPath)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 		for k, v := range abraShEnv {
 			app.Env[k] = v
 		}
 
-		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+		composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
+		stackName := app.StackName()
 		deployOpts := stack.Deploy{
 			Composefiles: composeFiles,
 			Namespace:    stackName,
 			Prune:        false,
 			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
 		}
-		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		config.ExposeAllEnv(stackName, compose, app.Env)
-		config.SetRecipeLabel(compose, stackName, app.Recipe)
-		config.SetChaosLabel(compose, stackName, internal.Chaos)
-		config.SetChaosVersionLabel(compose, stackName, version)
-		config.SetUpdateLabel(compose, stackName, app.Env)
+		appPkg.ExposeAllEnv(stackName, compose, app.Env)
+		appPkg.SetRecipeLabel(compose, stackName, app.Recipe.Name)
+		appPkg.SetChaosLabel(compose, stackName, internal.Chaos)
+		if internal.Chaos {
+			appPkg.SetChaosVersionLabel(compose, stackName, toDeployVersion)
+		}
+		appPkg.SetUpdateLabel(compose, stackName, app.Env)
+		appPkg.SetVersionLabel(compose, stackName, toDeployVersion)
 
-		envVars, err := config.CheckEnv(app)
+		envVars, err := appPkg.CheckEnv(app)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
 		for _, envVar := range envVars {
 			if !envVar.Present {
-				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+				deployWarnMessages = append(deployWarnMessages,
+					fmt.Sprintf("%s missing from %s.env", envVar.Name, app.Domain),
+				)
 			}
 		}
 
-		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
-			logrus.Fatal(err)
-		}
-
 		if !internal.NoDomainChecks {
-			domainName, ok := app.Env["DOMAIN"]
-			if ok {
+			if domainName, ok := app.Env["DOMAIN"]; ok {
 				if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
-					logrus.Fatal(err)
+					log.Fatal(err)
 				}
 			} else {
-				logrus.Warn("skipping domain checks as no DOMAIN=... configured for app")
+				log.Debug("skipping domain checks, no DOMAIN=... configured")
 			}
 		} else {
-			logrus.Warn("skipping domain checks as requested")
+			log.Debug("skipping domain checks")
 		}
 
-		stack.WaitTimeout, err = config.GetTimeoutFromLabel(compose, stackName)
+		deployedVersion := config.NO_VERSION_DEFAULT
+		if deployMeta.IsDeployed {
+			deployedVersion = deployMeta.Version
+		}
+
+		if err := internal.DeployOverview(
+			app,
+			deployedVersion,
+			toDeployVersion,
+			"",
+			deployWarnMessages,
+		); err != nil {
+			log.Fatal(err)
+		}
+
+		stack.WaitTimeout, err = appPkg.GetTimeoutFromLabel(compose, stackName)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
-		logrus.Debugf("set waiting timeout to %d s", stack.WaitTimeout)
 
-		if err := stack.RunDeploy(cl, deployOpts, compose, app.Name, internal.DontWaitConverge); err != nil {
-			logrus.Fatal(err)
+		log.Debugf("set waiting timeout to %d second(s)", stack.WaitTimeout)
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		f, err := app.Filters(true, false, serviceNames...)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(
+			cl,
+			deployOpts,
+			compose,
+			app.Name,
+			app.Server,
+			internal.DontWaitConverge,
+			f,
+		); err != nil {
+			log.Fatal(err)
 		}
 
 		postDeployCmds, ok := app.Env["POST_DEPLOY_CMDS"]
 		if ok && !internal.DontWaitConverge {
-			logrus.Debugf("run the following post-deploy commands: %s", postDeployCmds)
+			log.Debugf("run the following post-deploy commands: %s", postDeployCmds)
 			if err := internal.PostCmds(cl, app, postDeployCmds); err != nil {
-				logrus.Fatalf("attempting to run post deploy commands, saw: %s", err)
+				log.Fatalf("attempting to run post deploy commands, saw: %s", err)
 			}
 		}
-		return nil
+
+		if err := app.WriteRecipeVersion(toDeployVersion, false); err != nil {
+			log.Fatalf("writing recipe version failed: %s", err)
+		}
 	},
 }
+
+func getLatestVersionOrCommit(app app.App) (string, error) {
+	versions, err := app.Recipe.Tags()
+	if err != nil {
+		return "", err
+	}
+
+	if len(versions) > 0 && !internal.Chaos {
+		return versions[len(versions)-1], nil
+	}
+
+	head, err := app.Recipe.Head()
+	if err != nil {
+		return "", err
+	}
+
+	return formatter.SmallSHA(head.String()), nil
+}
+
+// validateArgsAndFlags ensures compatible args/flags.
+func validateArgsAndFlags(args []string) error {
+	if len(args) == 2 && args[1] != "" && internal.Chaos {
+		return fmt.Errorf("cannot use [version] and --chaos together")
+	}
+
+	return nil
+}
+
+func validateSecrets(cl *dockerClient.Client, app app.App) error {
+	secStats, err := secret.PollSecretsStatus(cl, app)
+	if err != nil {
+		return err
+	}
+
+	for _, secStat := range secStats {
+		if !secStat.CreatedOnRemote {
+			return fmt.Errorf("secret not generated: %s", secStat.LocalName)
+		}
+	}
+
+	return nil
+}
+
+func getDeployVersion(cliArgs []string, deployMeta stack.DeployMeta, app app.App) (string, error) {
+	// Chaos mode overrides everything
+	if internal.Chaos {
+		v, err := app.Recipe.ChaosVersion()
+		if err != nil {
+			return "", err
+		}
+		log.Debugf("version: taking chaos version: %s", v)
+		return v, nil
+	}
+
+	// Check if the deploy version is set with a cli argument
+	if len(cliArgs) == 2 && cliArgs[1] != "" {
+		log.Debugf("version: taking version from cli arg: %s", cliArgs[1])
+		return cliArgs[1], nil
+	}
+
+	// Check if the recipe has a version in the .env file
+	if app.Recipe.EnvVersion != "" && !internal.IgnoreEnvVersion {
+		if strings.HasSuffix(app.Recipe.EnvVersionRaw, "+U") {
+			return "", fmt.Errorf("version: can not redeploy chaos version %s", app.Recipe.EnvVersionRaw)
+		}
+		log.Debugf("version: taking version from .env file: %s", app.Recipe.EnvVersion)
+		return app.Recipe.EnvVersion, nil
+	}
+
+	// Take deployed version
+	if deployMeta.IsDeployed {
+		log.Debugf("version: taking deployed version: %s", deployMeta.Version)
+		return deployMeta.Version, nil
+	}
+
+	v, err := getLatestVersionOrCommit(app)
+	log.Debugf("version: taking new recipe version: %s", v)
+	if err != nil {
+		return "", err
+	}
+	return v, nil
+}
+
+func init() {
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.Force,
+		"force",
+		"f",
+		false,
+		"perform action without further prompt",
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.NoDomainChecks,
+		"no-domain-checks",
+		"D",
+		false,
+		"disable public DNS checks",
+	)
+
+	AppDeployCommand.Flags().BoolVarP(
+		&internal.DontWaitConverge,
+		"no-converge-checks",
+		"c",
+		false,
+		"disable converge logic checks",
+	)
+}

cli/app/env.go

@@ -0,0 +1,43 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
+	"github.com/spf13/cobra"
+)
+
+var AppEnvCommand = &cobra.Command{
+	Use:     "env <domain> [flags]",
+	Aliases: []string{"e"},
+	Short:   "Show app .env values",
+	Example: "  abra app env 1312.net",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		var envKeys []string
+		for k := range app.Env {
+			envKeys = append(envKeys, k)
+		}
+
+		sort.Strings(envKeys)
+
+		var rows [][]string
+		for _, k := range envKeys {
+			rows = append(rows, []string{k, app.Env[k]})
+		}
+
+		overview := formatter.CreateOverview("ENV OVERVIEW", rows)
+		fmt.Println(overview)
+	},
+}

cli/app/errors.go

@@ -1,142 +0,0 @@
-package app
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var appErrorsCommand = cli.Command{
-	Name:      "errors",
-	Usage:     "List errors for a deployed app",
-	ArgsUsage: "<domain>",
-	Description: `
-List errors for a deployed app.
-
-This is a best-effort implementation and an attempt to gather a number of tips
-& tricks for finding errors together into one convenient command. When an app
-is failing to deploy or having issues, it could be a lot of things.
-
-This command currently takes into account:
-
-    Is the service deployed?
-    Is the service killed by an OOM error?
-    Is the service reporting an error (like in "ps --no-trunc" output)
-    Is the service healthcheck failing? what are the healthcheck logs?
-
-Got any more ideas? Please let us know:
-
-    https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose
-
-This command is best accompanied by "abra app logs <domain>" which may reveal
-further information which can help you debug the cause of an app failure via
-the logs.
-`,
-	Aliases: []string{"e"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.WatchFlag,
-		internal.OfflineFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
-		}
-
-		if !internal.Watch {
-			if err := checkErrors(c, cl, app); err != nil {
-				logrus.Fatal(err)
-			}
-			return nil
-		}
-
-		for {
-			if err := checkErrors(c, cl, app); err != nil {
-				logrus.Fatal(err)
-			}
-			time.Sleep(2 * time.Second)
-		}
-	},
-}
-
-func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
-	recipe, err := recipe.Get(app.Recipe, internal.Offline)
-	if err != nil {
-		return err
-	}
-
-	for _, service := range recipe.Config.Services {
-		filters := filters.NewArgs()
-		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
-
-		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
-		if err != nil {
-			return err
-		}
-
-		if len(containers) == 0 {
-			logrus.Warnf("%s is not up, something seems wrong", service.Name)
-			continue
-		}
-
-		container := containers[0]
-		containerState, err := cl.ContainerInspect(context.Background(), container.ID)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if containerState.State.OOMKilled {
-			logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
-		}
-
-		if containerState.State.Error != "" {
-			logrus.Warnf("%s reports this error: %s", service.Name, containerState.State.Error)
-		}
-
-		if containerState.State.Health != nil {
-			if containerState.State.Health.Status != "healthy" {
-				logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
-				logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
-				for _, log := range containerState.State.Health.Log {
-					logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func getServiceName(names []string) string {
-	containerName := strings.Join(names, " ")
-	trimmed := strings.TrimPrefix(containerName, "/")
-	return strings.Split(trimmed, ".")[0]
-}

cli/app/labels.go

@@ -0,0 +1,139 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/spf13/cobra"
+)
+
+var AppLabelsCommand = &cobra.Command{
+	Use:     "labels <domain> [flags]",
+	Aliases: []string{"lb"},
+	Short:   "Show deployment labels",
+	Long:    "Both local recipe and live deployment labels are shown.",
+	Example: "  abra app labels 1312.net",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		remoteLabels, err := getLabels(cl, app.StackName())
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		rows := [][]string{
+			{"DEPLOYED LABELS", "---"},
+		}
+
+		remoteLabelKeys := make([]string, 0, len(remoteLabels))
+		for k := range remoteLabels {
+			remoteLabelKeys = append(remoteLabelKeys, k)
+		}
+
+		sort.Strings(remoteLabelKeys)
+
+		for _, k := range remoteLabelKeys {
+			rows = append(rows, []string{
+				k,
+				remoteLabels[k],
+			})
+		}
+
+		if len(remoteLabelKeys) == 0 {
+			rows = append(rows, []string{"unknown"})
+		}
+
+		rows = append(rows, []string{"RECIPE LABELS", "---"})
+
+		config, err := app.Recipe.GetComposeConfig(app.Env)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var localLabelKeys []string
+		var appServiceConfig composetypes.ServiceConfig
+		for _, service := range config.Services {
+			if service.Name == "app" {
+				appServiceConfig = service
+
+				for k := range service.Deploy.Labels {
+					localLabelKeys = append(localLabelKeys, k)
+				}
+			}
+		}
+
+		sort.Strings(localLabelKeys)
+
+		for _, k := range localLabelKeys {
+			rows = append(rows, []string{
+				k,
+				appServiceConfig.Deploy.Labels[k],
+			})
+		}
+
+		overview := formatter.CreateOverview("LABELS OVERVIEW", rows)
+		fmt.Println(overview)
+	},
+}
+
+// getLabels reads docker labels from running services in the format of "coop-cloud.${STACK_NAME}.${LABEL}".
+func getLabels(cl *dockerClient.Client, stackName string) (map[string]string, error) {
+	labels := make(map[string]string)
+
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return labels, err
+	}
+
+	for _, service := range services {
+		if service.Spec.Name != fmt.Sprintf("%s_app", stackName) {
+			continue
+		}
+
+		for k, v := range service.Spec.Labels {
+			labels[k] = v
+		}
+	}
+
+	return labels, nil
+}
+
+func init() {
+	AppLabelsCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+}

cli/app/list.go

@@ -8,37 +8,14 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/config"
+	appPkg "coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/log"
 	"coopcloud.tech/tagcmp"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var status bool
-var statusFlag = &cli.BoolFlag{
-	Name:        "status, S",
-	Usage:       "Show app deployment status",
-	Destination: &status,
-}
-
-var recipeFilter string
-var recipeFlag = &cli.StringFlag{
-	Name:        "recipe, r",
-	Value:       "",
-	Usage:       "Show apps of a specific recipe",
-	Destination: &recipeFilter,
-}
-
-var listAppServer string
-var listAppServerFlag = &cli.StringFlag{
-	Name:        "server, s",
-	Value:       "",
-	Usage:       "Show apps of a specific server",
-	Destination: &listAppServer,
-}
-
 type appStatus struct {
 	Server       string `json:"server"`
 	Recipe       string `json:"recipe"`
@@ -61,42 +38,36 @@ type serverStatus struct {
 	UpgradeCount     int         `json:"upgradeCount"`
 }
 
-var appListCommand = cli.Command{
-	Name:    "list",
+var AppListCommand = &cobra.Command{
+	Use:     "list [flags]",
 	Aliases: []string{"ls"},
-	Usage:   "List all managed apps",
-	Description: `
-Read the local file system listing of apps and servers (e.g. ~/.abra/) to
-generate a report of all your apps.
+	Short:   "List all managed apps",
+	Long: `Generate a report of all managed apps.
 
-By passing the "--status/-S" flag, you can query all your servers for the
-actual live deployment status. Depending on how many servers you manage, this
-can take some time.
-`,
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.MachineReadableFlag,
-		statusFlag,
-		listAppServerFlag,
-		recipeFlag,
-		internal.OfflineFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		appFiles, err := config.LoadAppFiles(listAppServer)
+Use "--status/-S" flag to query all servers for the live deployment status.`,
+	Example: `  # list apps of all servers without live status
+  abra app ls
+
+  # list apps of a specific server with live status
+  abra app ls -s 1312.net -S
+
+  # list apps of all servers which match a specific recipe
+  abra app ls -r gitea`,
+	Args: cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		appFiles, err := appPkg.LoadAppFiles(listAppServer)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		apps, err := config.GetApps(appFiles, recipeFilter)
+		apps, err := appPkg.GetApps(appFiles, recipeFilter)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		sort.Sort(config.ByServerAndRecipe(apps))
+		sort.Sort(appPkg.ByServerAndRecipe(apps))
 
 		statuses := make(map[string]map[string]string)
-		var catl recipe.RecipeCatalogue
 		if status {
 			alreadySeen := make(map[string]bool)
 			for _, app := range apps {
@@ -105,14 +76,9 @@ can take some time.
 				}
 			}
 
-			statuses, err = config.GetAppStatuses(apps, internal.MachineReadable)
+			statuses, err = appPkg.GetAppStatuses(apps, internal.MachineReadable)
 			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			catl, err = recipe.ReadRecipeCatalogue(internal.Offline)
-			if err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 		}
 
@@ -130,7 +96,7 @@ can take some time.
 				}
 			}
 
-			if app.Recipe == recipeFilter || recipeFilter == "" {
+			if app.Recipe.Name == recipeFilter || recipeFilter == "" {
 				if recipeFilter != "" {
 					// only count server if matches filter
 					totalServersCount++
@@ -176,21 +142,25 @@ can take some time.
 					appStats.AutoUpdate = autoUpdate
 
 					var newUpdates []string
-					if version != "unknown" {
-						updates, err := recipe.GetRecipeCatalogueVersions(app.Recipe, catl)
+					if version != "unknown" && chaosVersion == "unknown" {
+						if err := app.Recipe.EnsureExists(); err != nil {
+							log.Fatalf("unable to clone %s: %s", app.Name, err)
+						}
+
+						updates, err := app.Recipe.Tags()
 						if err != nil {
-							logrus.Fatal(err)
+							log.Fatalf("unable to retrieve tags for %s: %s", app.Name, err)
 						}
 
 						parsedVersion, err := tagcmp.Parse(version)
 						if err != nil {
-							logrus.Fatal(err)
+							log.Fatal(err)
 						}
 
 						for _, update := range updates {
 							parsedUpdate, err := tagcmp.Parse(update)
 							if err != nil {
-								logrus.Fatal(err)
+								log.Fatal(err)
 							}
 
 							if update != version && parsedUpdate.IsGreaterThan(parsedVersion) {
@@ -207,14 +177,14 @@ can take some time.
 							stats.LatestCount++
 						}
 					} else {
-						newUpdates = internal.ReverseStringList(newUpdates)
+						newUpdates = internal.SortVersionsDesc(newUpdates)
 						appStats.Upgrade = strings.Join(newUpdates, "\n")
 						stats.UpgradeCount++
 					}
 				}
 
 				appStats.Server = app.Server
-				appStats.Recipe = app.Recipe
+				appStats.Recipe = app.Recipe.Name
 				appStats.AppName = app.Name
 				appStats.Domain = app.Domain
 
@@ -226,11 +196,12 @@ can take some time.
 		if internal.MachineReadable {
 			jsonstring, err := json.Marshal(allStats)
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			} else {
 				fmt.Println(string(jsonstring))
 			}
-			return nil
+
+			return
 		}
 
 		alreadySeen := make(map[string]bool)
@@ -241,60 +212,118 @@ can take some time.
 
 			serverStat := allStats[app.Server]
 
-			tableCol := []string{"recipe", "domain"}
+			headers := []string{"RECIPE", "DOMAIN", "SERVER"}
 			if status {
-				tableCol = append(tableCol, []string{"status", "chaos", "version", "upgrade", "autoupdate"}...)
+				headers = append(headers, []string{
+					"STATUS",
+					"CHAOS",
+					"VERSION",
+					"UPGRADE",
+					"AUTOUPDATE"}...,
+				)
 			}
 
-			table := formatter.CreateTable(tableCol)
+			table, err := formatter.CreateTable()
+			if err != nil {
+				log.Fatal(err)
+			}
 
+			table.Headers(headers...)
+
+			var rows [][]string
 			for _, appStat := range serverStat.Apps {
-				tableRow := []string{appStat.Recipe, appStat.Domain}
+				row := []string{appStat.Recipe, appStat.Domain, appStat.Server}
 				if status {
 					chaosStatus := appStat.Chaos
 					if chaosStatus != "unknown" {
 						chaosEnabled, err := strconv.ParseBool(chaosStatus)
 						if err != nil {
-							logrus.Fatal(err)
+							log.Fatal(err)
 						}
 						if chaosEnabled && appStat.ChaosVersion != "unknown" {
 							chaosStatus = appStat.ChaosVersion
 						}
 					}
-					tableRow = append(tableRow, []string{appStat.Status, chaosStatus, appStat.Version, appStat.Upgrade, appStat.AutoUpdate}...)
+
+					row = append(row, []string{
+						appStat.Status,
+						chaosStatus,
+						appStat.Version,
+						appStat.Upgrade,
+						appStat.AutoUpdate}...,
+					)
 				}
-				table.Append(tableRow)
+
+				rows = append(rows, row)
 			}
 
-			if table.NumLines() > 0 {
-				table.Render()
+			table.Rows(rows...)
 
-				if status {
-					fmt.Println(fmt.Sprintf(
-						"server: %s | total apps: %v | versioned: %v | unversioned: %v | latest: %v | upgrade: %v",
-						app.Server,
-						serverStat.AppCount,
-						serverStat.VersionCount,
-						serverStat.UnversionedCount,
-						serverStat.LatestCount,
-						serverStat.UpgradeCount,
-					))
-				} else {
-					fmt.Println(fmt.Sprintf("server: %s | total apps: %v", app.Server, serverStat.AppCount))
+			if len(rows) > 0 {
+				if err := formatter.PrintTable(table); err != nil {
+					log.Fatal(err)
 				}
-			}
 
-			if len(allStats) > 1 && table.NumLines() > 0 {
-				fmt.Println() // newline separator for multiple servers
+				if len(allStats) > 1 && len(rows) > 0 {
+					fmt.Println() // newline separator for multiple servers
+				}
 			}
 
 			alreadySeen[app.Server] = true
 		}
-
-		if len(allStats) > 1 {
-			fmt.Println(fmt.Sprintf("total servers: %v | total apps: %v ", totalServersCount, totalAppsCount))
-		}
-
-		return nil
 	},
 }
+
+var (
+	status        bool
+	recipeFilter  string
+	listAppServer string
+)
+
+func init() {
+	AppListCommand.Flags().BoolVarP(
+		&status,
+		"status",
+		"S",
+		false,
+		"show app deployment status",
+	)
+
+	AppListCommand.Flags().StringVarP(
+		&recipeFilter,
+		"recipe",
+		"r",
+		"",
+		"show apps of a specific recipe",
+	)
+
+	AppListCommand.RegisterFlagCompletionFunc(
+		"recipe",
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.RecipeNameComplete()
+		},
+	)
+
+	AppListCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		"machine",
+		"m",
+		false,
+		"print machine-readable output",
+	)
+
+	AppListCommand.Flags().StringVarP(
+		&listAppServer,
+		"server",
+		"s",
+		"",
+		"show apps of a specific server",
+	)
+
+	AppListCommand.RegisterFlagCompletionFunc(
+		"server",
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.ServerNameComplete()
+		},
+	)
+}

cli/app/logs.go

@@ -2,138 +2,106 @@ package app
 
 import (
 	"context"
-	"io"
-	"os"
-	"slices"
-	"sync"
-	"time"
+	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/logs"
 	"coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appLogsCommand = cli.Command{
-	Name:      "logs",
-	Aliases:   []string{"l"},
-	ArgsUsage: "<domain> [<service>]",
-	Usage:     "Tail app logs",
-	Flags: []cli.Flag{
-		internal.StdErrOnlyFlag,
-		internal.SinceLogsFlag,
-		internal.DebugFlag,
+var AppLogsCommand = &cobra.Command{
+	Use:     "logs <domain> [service] [flags]",
+	Aliases: []string{"l"},
+	Short:   "Tail app logs",
+	Args:    cobra.RangeArgs(1, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			app, err := appPkg.Get(args[0])
+			if err != nil {
+				errMsg := fmt.Sprintf("autocomplete failed: %s", err)
+				return []string{errMsg}, cobra.ShellCompDirectiveError
+			}
+			return autocomplete.ServiceNameComplete(app.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 		stackName := app.StackName()
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
+		if err := app.Recipe.EnsureExists(); err != nil {
+			log.Fatal(err)
 		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, stackName)
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, stackName)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+		if !deployMeta.IsDeployed {
+			log.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		serviceName := c.Args().Get(1)
-		serviceNames := []string{}
-		if serviceName != "" {
-			serviceNames = []string{serviceName}
+		var serviceNames []string
+		if len(args) == 2 {
+			serviceNames = []string{args[1]}
 		}
-		err = tailLogs(cl, app, serviceNames)
+
+		f, err := app.Filters(true, false, serviceNames...)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		return nil
+		opts := logs.TailOpts{
+			AppName:  app.Name,
+			Services: serviceNames,
+			StdErr:   stdErr,
+			Since:    sinceLogs,
+			Filters:  f,
+		}
+
+		if err := logs.TailLogs(cl, opts); err != nil {
+			log.Fatal(err)
+		}
 	},
 }
 
-// tailLogs prints logs for the given app with optional service names to be
-// filtered on. It also checks if the latest task is not runnning and then
-// prints the past tasks.
-func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) error {
-	f, err := app.Filters(true, false, serviceNames...)
-	if err != nil {
-		return err
-	}
+var (
+	stdErr    bool
+	sinceLogs string
+)
 
-	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: f})
-	if err != nil {
-		return err
-	}
+func init() {
+	AppLogsCommand.Flags().BoolVarP(
+		&stdErr,
+		"stderr",
+		"s",
+		false,
+		"only tail stderr",
+	)
 
-	var wg sync.WaitGroup
-	for _, service := range services {
-		filters := filters.NewArgs()
-		filters.Add("name", service.Spec.Name)
-		tasks, err := cl.TaskList(context.Background(), types.TaskListOptions{Filters: f})
-		if err != nil {
-			return err
-		}
-		if len(tasks) > 0 {
-			// Need to sort the tasks by the CreatedAt field in the inverse order.
-			// Otherwise they are in the reversed order and not sorted properly.
-			slices.SortFunc[[]swarm.Task](tasks, func(t1, t2 swarm.Task) int {
-				return int(t2.Meta.CreatedAt.Unix() - t1.Meta.CreatedAt.Unix())
-			})
-			lastTask := tasks[0].Status
-			if lastTask.State != swarm.TaskStateRunning {
-				for _, task := range tasks {
-					logrus.Errorf("[%s] %s State %s: %s", service.Spec.Name, task.Meta.CreatedAt.Format(time.RFC3339), task.Status.State, task.Status.Err)
-				}
-			}
-		}
-
-		// Collect the logs in a go routine, so the logs from all services are
-		// collected in parallel.
-		wg.Add(1)
-		go func(serviceID string) {
-			logs, err := cl.ServiceLogs(context.Background(), serviceID, types.ContainerLogsOptions{
-				ShowStderr: true,
-				ShowStdout: !internal.StdErrOnly,
-				Since:      internal.SinceLogs,
-				Until:      "",
-				Timestamps: true,
-				Follow:     true,
-				Tail:       "20",
-				Details:    false,
-			})
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			defer logs.Close()
-
-			_, err = io.Copy(os.Stdout, logs)
-			if err != nil && err != io.EOF {
-				logrus.Fatal(err)
-			}
-		}(service.ID)
-	}
-
-	// Wait for all log streams to be closed.
-	wg.Wait()
-
-	return nil
+	AppLogsCommand.Flags().StringVarP(
+		&sinceLogs,
+		"since",
+		"S",
+		"",
+		"tail logs since YYYY-MM-DDTHH:MM:SSZ",
+	)
 }

cli/app/new.go

@@ -2,33 +2,36 @@ package app
 
 import (
 	"fmt"
-	"path"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/jsontable"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/log"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/AlecAivazis/survey/v2"
+	"github.com/charmbracelet/lipgloss/table"
 	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appNewDescription = `
-Take a recipe and uses it to create a new app. This new app configuration is
-stored in your ~/.abra directory under the appropriate server.
+var appNewDescription = `Creates a new app from a default recipe.
+
+This new app configuration is stored in your $ABRA_DIR directory under the
+appropriate server.
 
 This command does not deploy your app for you. You will need to run "abra app
 deploy <domain>" to do so.
 
-You can see what recipes are available (i.e. values for the <recipe> argument)
+You can see what recipes are available (i.e. values for the [recipe] argument)
 by running "abra recipe ls".
 
+Recipe commit hashes are supported values for "[version]".
+
 Passing the "--secrets/-S" flag will automatically generate secrets for your
 app and store them encrypted at rest on the chosen target server. These
 generated secrets are only visible at generation time, so please take care to
@@ -36,145 +39,188 @@ store them somewhere safe.
 
 You can use the "--pass/-P" to store these generated passwords locally in a
 pass store (see passwordstore.org for more). The pass command must be available
-on your $PATH.
-`
+on your $PATH.`
 
-var appNewCommand = cli.Command{
-	Name:        "new",
-	Aliases:     []string{"n"},
-	Usage:       "Create a new app",
-	Description: appNewDescription,
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-		internal.NewAppServerFlag,
-		internal.DomainFlag,
-		internal.PassFlag,
-		internal.SecretsFlag,
-		internal.OfflineFlag,
-		internal.ChaosFlag,
-	},
-	Before:    internal.SubCommandBefore,
-	ArgsUsage: "[<recipe>] [<version>]",
-	BashComplete: func(ctx *cli.Context) {
-		args := ctx.Args()
-		switch len(args) {
+var AppNewCommand = &cobra.Command{
+	Use:     "new [recipe] [version] [flags]",
+	Aliases: []string{"n"},
+	Short:   "Create a new app",
+	Long:    appNewDescription,
+	Args:    cobra.RangeArgs(0, 2),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
 		case 0:
-			autocomplete.RecipeNameComplete(ctx)
+			return autocomplete.RecipeNameComplete()
 		case 1:
-			autocomplete.RecipeVersionComplete(ctx.Args().Get(0))
+			recipe := internal.ValidateRecipe(args, cmd.Name())
+			return autocomplete.RecipeVersionComplete(recipe.Name)
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
 		}
 	},
-	Action: func(c *cli.Context) error {
-		recipe := internal.ValidateRecipe(c)
+	Run: func(cmd *cobra.Command, args []string) {
+		recipe := internal.ValidateRecipe(args, cmd.Name())
 
-		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(recipe.Name); err != nil {
-				logrus.Fatal(err)
+		if len(args) == 2 && internal.Chaos {
+			log.Fatal("cannot use [version] and --chaos together")
+		}
+
+		var recipeVersion string
+		if len(args) == 2 {
+			recipeVersion = args[1]
+		}
+
+		chaosVersion := config.CHAOS_DEFAULT
+		if internal.Chaos {
+			var err error
+			chaosVersion, err = recipe.ChaosVersion()
+			if err != nil {
+				log.Fatal(err)
 			}
-			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(recipe.Name); err != nil {
-					logrus.Fatal(err)
+
+			recipeVersion = chaosVersion
+		} else {
+			if err := recipe.EnsureIsClean(); err != nil {
+				log.Fatal(err)
+			}
+
+			var recipeVersions recipePkg.RecipeVersions
+			if recipeVersion == "" {
+				var err error
+				recipeVersions, _, err = recipe.GetRecipeVersions()
+				if err != nil {
+					log.Fatal(err)
 				}
 			}
-			if c.Args().Get(1) == "" {
-				if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
-					logrus.Fatal(err)
+
+			if len(recipeVersions) > 0 {
+				latest := recipeVersions[len(recipeVersions)-1]
+				for tag := range latest {
+					recipeVersion = tag
+				}
+
+				if _, err := recipe.EnsureVersion(recipeVersion); err != nil {
+					log.Fatal(err)
 				}
 			} else {
-				if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil {
-					logrus.Fatal(err)
+				if err := recipe.EnsureLatest(); err != nil {
+					log.Fatal(err)
+				}
+
+				if recipeVersion == "" {
+					head, err := recipe.Head()
+					if err != nil {
+						log.Fatalf("failed to retrieve latest commit for %s: %s", recipe.Name, err)
+					}
+
+					recipeVersion = formatter.SmallSHA(head.String())
 				}
 			}
 		}
 
 		if err := ensureServerFlag(); err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		if err := ensureDomainFlag(recipe, internal.NewAppServer); err != nil {
-			logrus.Fatal(err)
+		if err := ensureDomainFlag(recipe, newAppServer); err != nil {
+			log.Fatal(err)
 		}
 
-		sanitisedAppName := config.SanitiseAppName(internal.Domain)
-		logrus.Debugf("%s sanitised as %s for new app", internal.Domain, sanitisedAppName)
+		sanitisedAppName := appPkg.SanitiseAppName(appDomain)
+		log.Debugf("%s sanitised as %s for new app", appDomain, sanitisedAppName)
 
-		if err := config.TemplateAppEnvSample(
-			recipe.Name,
-			internal.Domain,
-			internal.NewAppServer,
-			internal.Domain,
+		if err := appPkg.TemplateAppEnvSample(
+			recipe,
+			appDomain,
+			newAppServer,
+			appDomain,
 		); err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		var secrets AppSecrets
-		var secretTable *jsontable.JSONTable
-		if internal.Secrets {
+		var appSecrets AppSecrets
+		var secretsTable *table.Table
+		if generateSecrets {
 			sampleEnv, err := recipe.SampleEnv()
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 
-			composeFiles, err := config.GetComposeFiles(recipe.Name, sampleEnv)
+			composeFiles, err := recipe.GetComposeFiles(sampleEnv)
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 
-			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, config.StackName(internal.Domain))
+			secretsConfig, err := secret.ReadSecretsConfig(
+				recipe.SampleEnvPath,
+				composeFiles,
+				appPkg.StackName(appDomain),
+			)
 			if err != nil {
-				return err
+				log.Fatal(err)
 			}
 
 			if err := promptForSecrets(recipe.Name, secretsConfig); err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 
-			cl, err := client.New(internal.NewAppServer)
+			cl, err := client.New(newAppServer)
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 
-			secrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
+			appSecrets, err = createSecrets(cl, secretsConfig, sanitisedAppName)
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 
-			secretCols := []string{"Name", "Value"}
-			secretTable = formatter.CreateTable(secretCols)
-			for name, val := range secrets {
-				secretTable.Append([]string{name, val})
+			secretsTable, err = formatter.CreateTable()
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			headers := []string{"NAME", "VALUE"}
+			secretsTable.Headers(headers...)
+
+			for name, val := range appSecrets {
+				secretsTable.Row(name, val)
 			}
 		}
 
-		if internal.NewAppServer == "default" {
-			internal.NewAppServer = "local"
+		if newAppServer == "default" {
+			newAppServer = "local"
 		}
 
-		tableCol := []string{"server", "recipe", "domain"}
-		table := formatter.CreateTable(tableCol)
-		table.Append([]string{internal.NewAppServer, recipe.Name, internal.Domain})
+		log.Infof("%s created (version: %s)", appDomain, recipeVersion)
 
-		fmt.Println(fmt.Sprintf("A new %s app has been created! Here is an overview:", recipe.Name))
-		fmt.Println("")
-		table.Render()
-		fmt.Println("")
-		fmt.Println("You can configure this app by running the following:")
-		fmt.Println(fmt.Sprintf("\n    abra app config %s", internal.Domain))
-		fmt.Println("")
-		fmt.Println("You can deploy this app by running the following:")
-		fmt.Println(fmt.Sprintf("\n    abra app deploy %s", internal.Domain))
+		if len(appSecrets) > 0 {
+			rows := [][]string{}
+			for k, v := range appSecrets {
+				rows = append(rows, []string{k, v})
+			}
 
-		if len(secrets) > 0 {
-			fmt.Println("")
-			fmt.Println("Here are your generated secrets:")
-			fmt.Println("")
-			secretTable.Render()
-			logrus.Warn("generated secrets are not shown again, please take note of them NOW")
+			overview := formatter.CreateOverview("SECRETS OVERVIEW", rows)
+
+			fmt.Println(overview)
+
+			log.Warnf(
+				"secrets are %s shown again, please save them %s",
+				formatter.BoldUnderlineStyle.Render("NOT"),
+				formatter.BoldUnderlineStyle.Render("NOW"),
+			)
 		}
 
-		return nil
+		app, err := app.Get(appDomain)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := app.WriteRecipeVersion(recipeVersion, false); err != nil {
+			log.Fatalf("writing recipe version failed: %s", err)
+		}
 	},
 }
 
@@ -183,19 +229,25 @@ type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
 func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) {
-	secrets, err := secret.GenerateSecrets(cl, secretsConfig, internal.NewAppServer)
+	// NOTE(d1): trim to match app.StackName() implementation
+	if len(sanitisedAppName) > config.MAX_SANITISED_APP_NAME_LENGTH {
+		log.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH])
+		sanitisedAppName = sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH]
+	}
+
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, newAppServer)
 	if err != nil {
 		return nil, err
 	}
 
-	if internal.Pass {
+	if saveInPass {
 		for secretName := range secrets {
 			secretValue := secrets[secretName]
 			if err := secret.PassInsertSecret(
 				secretValue,
 				secretName,
-				internal.Domain,
-				internal.NewAppServer,
+				appDomain,
+				newAppServer,
 			); err != nil {
 				return nil, err
 			}
@@ -206,18 +258,18 @@ func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secr
 }
 
 // ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
-func ensureDomainFlag(recipe recipe.Recipe, server string) error {
-	if internal.Domain == "" && !internal.NoInput {
+func ensureDomainFlag(recipe recipePkg.Recipe, server string) error {
+	if appDomain == "" && !internal.NoInput {
 		prompt := &survey.Input{
 			Message: "Specify app domain",
 			Default: fmt.Sprintf("%s.%s", recipe.Name, server),
 		}
-		if err := survey.AskOne(prompt, &internal.Domain); err != nil {
+		if err := survey.AskOne(prompt, &appDomain); err != nil {
 			return err
 		}
 	}
 
-	if internal.Domain == "" {
+	if appDomain == "" {
 		return fmt.Errorf("no domain provided")
 	}
 
@@ -227,15 +279,15 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 // promptForSecrets asks if we should generate secrets for a new app.
 func promptForSecrets(recipeName string, secretsConfig map[string]secret.Secret) error {
 	if len(secretsConfig) == 0 {
-		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
+		log.Debugf("%s has no secrets to generate, skipping...", recipeName)
 		return nil
 	}
 
-	if !internal.Secrets && !internal.NoInput {
+	if !generateSecrets && !internal.NoInput {
 		prompt := &survey.Confirm{
 			Message: "Generate app secrets?",
 		}
-		if err := survey.AskOne(prompt, &internal.Secrets); err != nil {
+		if err := survey.AskOne(prompt, &generateSecrets); err != nil {
 			return err
 		}
 	}
@@ -250,19 +302,82 @@ func ensureServerFlag() error {
 		return err
 	}
 
-	if internal.NewAppServer == "" && !internal.NoInput {
+	if len(servers) == 1 {
+		newAppServer = servers[0]
+		log.Infof("single server detected, choosing %s automatically", newAppServer)
+		return nil
+	}
+
+	if newAppServer == "" && !internal.NoInput {
 		prompt := &survey.Select{
 			Message: "Select app server:",
 			Options: servers,
 		}
-		if err := survey.AskOne(prompt, &internal.NewAppServer); err != nil {
+		if err := survey.AskOne(prompt, &newAppServer); err != nil {
 			return err
 		}
 	}
 
-	if internal.NewAppServer == "" {
+	if newAppServer == "" {
 		return fmt.Errorf("no server provided")
 	}
 
 	return nil
 }
+
+var (
+	newAppServer    string
+	appDomain       string
+	saveInPass      bool
+	generateSecrets bool
+)
+
+func init() {
+	AppNewCommand.Flags().StringVarP(
+		&newAppServer,
+		"server",
+		"s",
+		"",
+		"specify server for new app",
+	)
+
+	AppNewCommand.RegisterFlagCompletionFunc(
+		"server",
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return autocomplete.ServerNameComplete()
+		},
+	)
+
+	AppNewCommand.Flags().StringVarP(
+		&appDomain,
+		"domain",
+		"D",
+		"",
+		"domain name for app",
+	)
+
+	AppNewCommand.Flags().BoolVarP(
+		&saveInPass,
+		"pass",
+		"p",
+		false,
+		"store secrets in a local pass store",
+	)
+
+	AppNewCommand.Flags().BoolVarP(
+		&generateSecrets,
+		"secrets",
+		"S",
+		false,
+		"automatically generate secrets",
+	)
+
+	AppNewCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+
+}

cli/app/ps.go

@@ -2,100 +2,209 @@ package app
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
 	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/log"
+	abraService "coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/buger/goterm"
 	dockerFormatter "github.com/docker/cli/cli/command/formatter"
-	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appPsCommand = cli.Command{
-	Name:        "ps",
-	Aliases:     []string{"p"},
-	Usage:       "Check app status",
-	ArgsUsage:   "<domain>",
-	Description: "Show a more detailed status output of a specific deployed app",
-	Flags: []cli.Flag{
-		internal.WatchFlag,
-		internal.DebugFlag,
+var AppPsCommand = &cobra.Command{
+	Use:     "ps <domain> [flags]",
+	Aliases: []string{"p"},
+	Short:   "Check app deployment status",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
 
 		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		deployMeta, err := stack.IsDeployed(context.Background(), cl, app.StackName())
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		if !isDeployed {
-			logrus.Fatalf("%s is not deployed?", app.Name)
+		if !deployMeta.IsDeployed {
+			log.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		if !internal.Watch {
-			showPSOutput(c, app, cl)
-			return nil
+		chaosVersion := config.CHAOS_DEFAULT
+		statuses, err := appPkg.GetAppStatuses([]appPkg.App{app}, true)
+		if statusMeta, ok := statuses[app.StackName()]; ok {
+			if isChaos, exists := statusMeta["chaos"]; exists && isChaos == "true" {
+				if cVersion, exists := statusMeta["chaosVersion"]; exists {
+					chaosVersion = cVersion
+					if strings.HasSuffix(chaosVersion, config.DIRTY_DEFAULT) {
+						chaosVersion = formatter.BoldDirtyDefault(chaosVersion)
+					}
+				}
+			}
 		}
 
-		goterm.Clear()
-		for {
-			goterm.MoveCursor(1, 1)
-			showPSOutput(c, app, cl)
-			goterm.Flush()
-			time.Sleep(2 * time.Second)
-		}
+		showPSOutput(app, cl, deployMeta.Version, chaosVersion)
 	},
 }
 
 // showPSOutput renders ps output.
-func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
-	filters, err := app.Filters(true, true)
+func showPSOutput(app appPkg.App, cl *dockerClient.Client, deployedVersion, chaosVersion string) {
+	composeFiles, err := app.Recipe.GetComposeFiles(app.Env)
 	if err != nil {
-		logrus.Fatal(err)
+		log.Fatal(err)
+		return
 	}
 
-	containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+	deployOpts := stack.Deploy{
+		Composefiles: composeFiles,
+		Namespace:    app.StackName(),
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+	compose, err := appPkg.GetAppComposeConfig(app.Name, deployOpts, app.Env)
 	if err != nil {
-		logrus.Fatal(err)
+		log.Fatal(err)
+		return
 	}
 
-	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
-	table := formatter.CreateTable(tableCol)
+	services := compose.Services
+	sort.Slice(services, func(i, j int) bool {
+		return services[i].Name < services[j].Name
+	})
 
-	for _, container := range containers {
-		var containerNames []string
-		for _, containerName := range container.Names {
-			trimmed := strings.TrimPrefix(containerName, "/")
-			containerNames = append(containerNames, trimmed)
+	var rows [][]string
+	allContainerStats := make(map[string]map[string]string)
+	for _, service := range services {
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
+
+		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
+		if err != nil {
+			log.Fatal(err)
+			return
 		}
 
-		tableRow := []string{
-			service.ContainerToServiceName(container.Names, app.StackName()),
-			formatter.RemoveSha(container.Image),
-			formatter.HumanDuration(container.Created),
-			container.Status,
-			container.State,
-			dockerFormatter.DisplayablePorts(container.Ports),
+		var containerStats map[string]string
+		if len(containers) == 0 {
+			containerStats = map[string]string{
+				"version": deployedVersion,
+				"chaos":   chaosVersion,
+				"service": service.Name,
+				"image":   "unknown",
+				"created": "unknown",
+				"status":  "unknown",
+				"state":   "unknown",
+				"ports":   "unknown",
+			}
+		} else {
+			container := containers[0]
+			containerStats = map[string]string{
+				"version": deployedVersion,
+				"chaos":   chaosVersion,
+				"service": abraService.ContainerToServiceName(container.Names, app.StackName()),
+				"image":   formatter.RemoveSha(container.Image),
+				"created": formatter.HumanDuration(container.Created),
+				"status":  container.Status,
+				"state":   container.State,
+				"ports":   dockerFormatter.DisplayablePorts(container.Ports),
+			}
 		}
-		table.Append(tableRow)
+
+		allContainerStats[containerStats["service"]] = containerStats
+
+		// NOTE(d1): don't clobber these variables for --machine output
+		dVersion := deployedVersion
+		cVersion := chaosVersion
+
+		if containerStats["service"] != "app" {
+			// NOTE(d1): don't repeat info which only relevant for the "app" service
+			dVersion = ""
+			cVersion = ""
+		}
+
+		row := []string{
+			containerStats["service"],
+			containerStats["status"],
+			containerStats["image"],
+			dVersion,
+			cVersion,
+		}
+
+		rows = append(rows, row)
 	}
 
-	table.Render()
+	if internal.MachineReadable {
+		rendered, err := json.Marshal(allContainerStats)
+		if err != nil {
+			log.Fatal("unable to convert to JSON: %s", err)
+		}
+
+		fmt.Println(string(rendered))
+
+		return
+	}
+
+	table, err := formatter.CreateTable()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	headers := []string{
+		"SERVICE",
+		"STATUS",
+		"IMAGE",
+		"VERSION",
+		"CHAOS",
+	}
+
+	table.
+		Headers(headers...).
+		Rows(rows...)
+
+	if err := formatter.PrintTable(table); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func init() {
+	AppPsCommand.Flags().BoolVarP(
+		&internal.MachineReadable,
+		"machine",
+		"m",
+		false,
+		"print machine-readable output",
+	)
+
+	AppPsCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
 }

cli/app/remove.go

@@ -3,28 +3,23 @@ package app
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/log"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/volume"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appRemoveCommand = cli.Command{
-	Name:      "remove",
-