fix: fetch recipe for "app list -S"

This was detected while debugging #534

.dockerignore

@@ -0,0 +1,8 @@
+*.swo
+*.swp
+.dockerignore
+Dockerfile
+abra
+dist
+kadabra
+tags

.drone.yml

@@ -3,35 +3,20 @@ kind: pipeline
 name: coopcloud.tech/abra
 steps:
   - name: make check
-    image: golang:1.18
+    image: golang:1.24
     commands:
       - make check
 
-  - name: make build
-    image: golang:1.18
-    commands:
-      - make build
-
   - name: make test
-    image: golang:1.18
+    image: golang:1.24
+    environment:
+      CATL_URL: https://git.coopcloud.tech/toolshed/recipes-catalogue-json.git
     commands:
+      - mkdir -p $HOME/.abra
+      - git clone $CATL_URL $HOME/.abra/catalogue
       - make test
-
-  - name: notify on failure
-    image: plugins/matrix
-    settings:
-      homeserver: https://matrix.autonomic.zone
-      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
-      userid: "@autono-bot:autonomic.zone"
-      accesstoken:
-        from_secret: autono_bot_access_token
     depends_on:
       - make check
-      - make build
-      - make test
-    when:
-      status:
-        - failure
 
   - name: fetch
     image: docker:git
@@ -39,13 +24,12 @@ steps:
       - git fetch --tags
     depends_on:
       - make check
-      - make build
       - make test
     when:
       event: tag
 
   - name: release
-    image: golang:1.18
+    image: goreleaser/goreleaser:v2.5.1
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token
@@ -53,12 +37,82 @@ steps:
       - name: deps
         path: /go
     commands:
-      - curl -sL https://git.io/goreleaser | bash
+      - goreleaser release
     depends_on:
       - fetch
     when:
       event: tag
 
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: abra-bot
+      password:
+        from_secret: git_coopcloud_tech_token_abra_bot
+      repo: git.coopcloud.tech/toolshed/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      branch:
+        - main
+    depends_on:
+      - make check
+      - make test
+
+  - name: on-demand integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      request_pty: true
+      script:
+        - |
+          wget https://git.coopcloud.tech/toolshed/abra/raw/branch/main/scripts/tests/run-ci-int -O run-ci-int
+          chmod +x run-ci-int
+          sh run-ci-int
+    when:
+      ref:
+        - refs/heads/int-*
+    depends_on:
+      - make check
+      - make test
+
+  - name: nightly integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      request_pty: true
+      script:
+        - |
+          wget https://git.coopcloud.tech/toolshed/abra/raw/branch/main/scripts/tests/run-ci-int -O run-ci-int
+          chmod +x run-ci-int
+          sh run-ci-int
+    when:
+      event:
+        - cron:
+      cron:
+        # @daily https://docs.drone.io/cron/
+        - integration 
+
 volumes:
   - name: deps
     temp: {}
+
+trigger:
+  action:
+    exclude:
+      - synchronized

.e2e.env.sample

@@ -1,4 +0,0 @@
-GANDI_TOKEN=...
-HCLOUD_TOKEN=...
-REGISTRY_PASSWORD=...
-REGISTRY_USERNAME=...

.envrc.sample

@@ -1,6 +1,7 @@
-go env -w GOPRIVATE=coopcloud.tech
+# integration test suite
+# export ABRA_DIR="$HOME/.abra_test"
+# export ABRA_TEST_DOMAIN=test.example.com
+# export ABRA_CI=1
 
-# export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
-# export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
-# export CAPSUL_TOKEN=...
-# export GITEA_TOKEN=...
+# release automation
+# export GITEA_TOKEN=

.gitea/ISSUE_TEMPLATE.md

@@ -1,8 +0,0 @@
----
-name: "Do not use this issue tracker"
-about: "Do not use this issue tracker"
-title: "Do not use this issue tracker"
-labels: []
----
-
-Please report your issue on [`coop-cloud/organising`](https://git.coopcloud.tech/coop-cloud/organising)

.gitignore

@@ -2,7 +2,7 @@
 .e2e.env
 .envrc
 .vscode/
-abra
+/abra
+/kadabra
 dist/
-tests/integration/.abra/catalogue
-vendor/
+tests/integration/.bats

.goreleaser.yml

@@ -1,16 +1,19 @@
 ---
-project_name: abra
 gitea_urls:
   api: https://git.coopcloud.tech/api/v1
   download: https://git.coopcloud.tech/
   skip_tls_verify: false
+
 before:
   hooks:
     - go mod tidy
+
 builds:
-  - env:
-      - CGO_ENABLED=0
+  - id: abra
+    binary: abra
     dir: cmd/abra
+    env:
+      - CGO_ENABLED=0
     goos:
       - linux
       - darwin
@@ -26,21 +29,48 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
-archives:
-  - replacements:
-      386: i386
-      amd64: x86_64
-    format: binary
+      - "-s"
+      - "-w"
+
+  - id: kadabra
+    binary: kadabra
+    dir: cmd/kadabra
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    gcflags:
+      - "all=-l -B"
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+      - "-s"
+      - "-w"
+
 checksum:
   name_template: "checksums.txt"
+
 snapshot:
   name_template: "{{ incpatch .Version }}-next"
+
 changelog:
   sort: desc
   filters:
     exclude:
+      - "^Merge"
+      - "^Revert"
       - "^WIP:"
+      - "^chore(deps):"
       - "^style:"
       - "^test:"
       - "^tests:"
-      - "^Revert"

AUTHORS.md

@@ -1,10 +1,22 @@
 # authors
 
-> If you're looking at this and you hack on Abra and you're not listed here,
-> please do add yourself! This is a community project, let's show
+> If you're looking at this and you hack on `abra` and you're not listed here,
+> please do add yourself! This is a community project, let's show some 💞
 
 - 3wordchant
+- ammaratef45
+- cassowary
+- codegod100
 - decentral1se
+- fauno
+- frando
 - kawaiipunk
 - knoflook
+- moritz
+- p4u1
+- rix
 - roxxers
+- vera
+- yksflip
+- basebuilder
+- mayel

Dockerfile

@@ -0,0 +1,30 @@
+# Build image
+FROM golang:1.24-alpine AS build
+
+ENV GOPRIVATE=coopcloud.tech
+
+RUN apk add --no-cache \
+  gcc \
+  git \
+  make \
+  musl-dev
+
+COPY . /app
+
+WORKDIR /app
+
+RUN CGO_ENABLED=0 make build
+
+# Release image ("slim")
+FROM alpine:3.19.1
+
+RUN apk add --no-cache \
+  ca-certificates \
+  git \
+  openssh
+
+RUN update-ca-certificates
+
+COPY --from=build /app/abra /abra
+
+ENTRYPOINT ["/abra"]

LICENSE

@@ -1,5 +1,5 @@
 Abra: The Co-op Cloud utility belt
-Copyright (C) 2022  Co-op Cloud <helo@coopcloud.tech>
+Copyright (C) 2022 Co-op Cloud <helo@coopcloud.tech>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Makefile

@@ -1,32 +1,55 @@
-ABRA    := ./cmd/abra
-COMMIT  := $(shell git rev-list -1 HEAD)
-GOPATH  := $(shell go env GOPATH)
-LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
+ABRA         := ./cmd/abra
+KADABRA      := ./cmd/kadabra
+COMMIT       := $(shell git rev-list -1 HEAD)
+GOPATH       := $(shell go env GOPATH)
+GOVERSION    := 1.24
+LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+GCFLAGS      := "all=-l -B"
+
 export GOPRIVATE=coopcloud.tech
 
-all: format check build test
+# NOTE(d1): default `make` optimised for Abra hacking
+all: format check build-abra test
 
-run:
-	@go run -ldflags=$(LDFLAGS) $(ABRA)
+run-abra:
+	@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
 
-install:
-	@go install -ldflags=$(LDFLAGS) $(ABRA)
+run-kadabra:
+	@go run -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(KADABRA)
 
-build-dev:
-	@go build -ldflags=$(LDFLAGS) $(ABRA)
+install-abra:
+	@go install -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(ABRA)
 
-build:
-	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
+install-kadabra:
+	@go install -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) $(KADABRA)
+
+install: install-abra install-kadabra
+
+build-abra:
+	@go build -v -gcflags=$(GCFLAGS) -ldflags=$(DIST_LDFLAGS) $(ABRA)
+
+build-kadabra:
+	@go build -v -gcflags=$(GCFLAGS) -ldflags=$(DIST_LDFLAGS) $(KADABRA)
+
+build: build-abra build-kadabra
+
+build-docker-abra:
+	@docker run -it -v $(PWD):/abra golang:$(GOVERSION) \
+		bash -c 'cd /abra; ./scripts/docker/build.sh'
+
+build-docker: build-docker-abra
 
 clean:
 	@rm '$(GOPATH)/bin/abra'
+	@rm '$(GOPATH)/bin/kadabra'
 
 format:
-	@gofmt -s -w .
+	@gofmt -s -w $$(find . -type f -name '*.go' | grep -v "/vendor/")
 
 check:
-	@test -z $$(gofmt -l .) || (echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+	@test -z $$(gofmt -l $$(find . -type f -name '*.go' | grep -v "/vendor/")) || \
+	(echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
 
 test:
 	@go test ./... -cover -v
@@ -34,9 +57,5 @@ test:
 loc:
 	@find . -name "*.go" | xargs wc -l
 
-loc-author:
-	@git ls-files -z | \
-		xargs -0rn 1 -P "$$(nproc)" -I{} sh -c 'git blame -w -M -C -C --line-porcelain -- {} | grep -I --line-buffered "^author "' | \
-   	sort -f | \
-   	uniq -ic | \
-   	sort -n
+deps:
+	@go get -t -u ./...

README.md

@@ -1,12 +1,13 @@
 # `abra`
 
-[![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
-[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
+[![Build Status](https://build.coopcloud.tech/api/badges/toolshed/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/toolshed/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/toolshed/abra)](https://goreportcard.com/report/git.coopcloud.tech/toolshed/abra)
+[![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
 
 The Co-op Cloud utility belt 🎩🐇
 
 <a href="https://github.com/egonelbre/gophers"><img align="right" width="150" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-`abra` is our flagship client & command-line tool which has been developed specifically in the context of the Co-op Cloud project for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community :heart:
+`abra` is the flagship client & command-line tool for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of [operators](https://docs.coopcloud.tech/operators/) and [maintainers](https://docs.coopcloud.tech/maintainers/) pleasant & convenient. It is libre software, written in [Go](https://go.dev) and maintained and extended by the community 💖
 
 Please see [docs.coopcloud.tech/abra](https://docs.coopcloud.tech/abra) for help on install, upgrade, hacking, troubleshooting & more!

cli/app/app.go

@@ -1,36 +1,11 @@
 package app
 
 import (
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var AppCommand = cli.Command{
-	Name:        "app",
-	Aliases:     []string{"a"},
-	Usage:       "Manage apps",
-	ArgsUsage:   "<domain>",
-	Description: "This command provides functionality for managing the life cycle of your apps",
-	Subcommands: []cli.Command{
-		appNewCommand,
-		appConfigCommand,
-		appRestartCommand,
-		appDeployCommand,
-		appUpgradeCommand,
-		appUndeployCommand,
-		appRemoveCommand,
-		appCheckCommand,
-		appListCommand,
-		appPsCommand,
-		appLogsCommand,
-		appCpCommand,
-		appRunCommand,
-		appRollbackCommand,
-		appSecretCommand,
-		appVolumeCommand,
-		appVersionCommand,
-		appErrorsCommand,
-		appCmdCommand,
-		appBackupCommand,
-		appRestoreCommand,
-	},
+var AppCommand = &cobra.Command{
+	Use:     "app [cmd] [args] [flags]",
+	Aliases: []string{"a"},
+	Short:   "Manage apps",
 }

cli/app/backup.go

@@ -1,389 +1,307 @@
 package app
 
 import (
-	"archive/tar"
-	"context"
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/system"
-	"github.com/klauspost/pgzip"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
 )
 
-type backupConfig struct {
-	preHookCmd  string
-	postHookCmd string
-	backupPaths []string
-}
-
-var appBackupCommand = cli.Command{
-	Name:      "backup",
-	Aliases:   []string{"bk"},
-	Usage:     "Run app backup",
-	ArgsUsage: "<domain> [<service>]",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
+var AppBackupListCommand = &cobra.Command{
+	Use:     "list <domain> [flags]",
+	Aliases: []string{"ls"},
+	Short:   "List the contents of a snapshot",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
 	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-This command runs an app backup.
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		recipe, err := recipe.Get(app.Recipe)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		backupConfigs := make(map[string]backupConfig)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName != "" {
-			backupConfig, ok := backupConfigs[serviceName]
-			if !ok {
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
-
-			logrus.Infof("running backup for the %s service", serviceName)
-
-			if err := runBackup(app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
-
-				if err := runBackup(app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
 		}
 
-		return nil
+		if snapshot != "" {
+			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if showAllPaths {
+			log.Debugf("including SHOW_ALL=%v in backupbot exec invocation", showAllPaths)
+			execEnv = append(execEnv, fmt.Sprintf("SHOW_ALL=%v", showAllPaths))
+		}
+
+		if timestamps {
+			log.Debugf("including TIMESTAMPS=%v in backupbot exec invocation", timestamps)
+			execEnv = append(execEnv, fmt.Sprintf("TIMESTAMPS=%v", timestamps))
+		}
+
+		if _, err = internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
 	},
 }
 
-// runBackup does the actual backup logic.
-func runBackup(app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
+var AppBackupDownloadCommand = &cobra.Command{
+	Use:     "download <domain> [flags]",
+	Aliases: []string{"d"},
+	Short:   "Download a snapshot",
+	Long: `Downloads a backup.tar.gz to the current working directory.
 
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
+"--volumes/-v" includes data contained in volumes alongide paths specified in
+"backupbot.backup.path" labels.`,
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		timestamp := strconv.Itoa(time.Now().Nanosecond())
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-		}
-		defer content.Close()
-
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			log.Fatal(err)
 		}
 
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
-	}
-
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
-
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
 		}
 
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
+		if snapshot != "" {
+			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
 
-	return nil
+		if includePath != "" {
+			log.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
+
+		if includeSecrets {
+			log.Debugf("including SECRETS=%v in backupbot exec invocation", includeSecrets)
+			execEnv = append(execEnv, fmt.Sprintf("SECRETS=%v", includeSecrets))
+		}
+
+		if includeVolumes {
+			log.Debugf("including VOLUMES=%v in backupbot exec invocation", includeVolumes)
+			execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%v", includeVolumes))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+
+		remoteBackupDir := "/tmp/backup.tar.gz"
+		currentWorkingDir := "."
+		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
+var AppBackupCreateCommand = &cobra.Command{
+	Use:     "create <domain> [flags]",
+	Aliases: []string{"c"},
+	Short:   "Create a new snapshot",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	return nil
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
+		if retries != "" {
+			log.Debugf("including RETRIES=%s in backupbot exec invocation", retries)
+			execEnv = append(execEnv, fmt.Sprintf("RETRIES=%s", retries))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
+var AppBackupSnapshotsCommand = &cobra.Command{
+	Use:     "snapshots <domain> [flags]",
+	Aliases: []string{"s"},
+	Short:   "List all snapshots",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-	timestamp := strconv.Itoa(time.Now().Nanosecond())
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, timestamp))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
 		}
-	}
 
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
 		}
-	}
 
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
 
-	return nil
+		if _, err = internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
-
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
-
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
-			}
-			break
-		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
-		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
+var AppBackupCommand = &cobra.Command{
+	Use:     "backup [cmd] [args] [flags]",
+	Aliases: []string{"b"},
+	Short:   "Manage app backups",
 }
 
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
+var (
+	snapshot       string
+	retries        string
+	includePath    string
+	showAllPaths   bool
+	timestamps     bool
+	includeSecrets bool
+	includeVolumes bool
+)
 
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
+func init() {
+	AppBackupListCommand.Flags().StringVarP(
+		&snapshot,
+		"snapshot",
+		"s",
+		"",
+		"list specific snapshot",
+	)
 
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
+	AppBackupListCommand.Flags().BoolVarP(
+		&showAllPaths,
+		"all",
+		"a",
+		false,
+		"show all paths",
+	)
 
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
+	AppBackupListCommand.Flags().BoolVarP(
+		&timestamps,
+		"timestamps",
+		"t",
+		false,
+		"include timestamps",
+	)
 
-	rc = fin
-	tr = tar.NewReader(rc)
+	AppBackupDownloadCommand.Flags().StringVarP(
+		&snapshot,
+		"snapshot",
+		"s",
+		"",
+		"list specific snapshot",
+	)
 
-	return tr, rc, nil
+	AppBackupDownloadCommand.Flags().StringVarP(
+		&includePath,
+		"path",
+		"p",
+		"",
+		"volumes path",
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&includeSecrets,
+		"secrets",
+		"S",
+		false,
+		"include secrets",
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&includeVolumes,
+		"volumes",
+		"v",
+		false,
+		"include volumes",
+	)
+
+	AppBackupDownloadCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+
+	AppBackupCreateCommand.Flags().StringVarP(
+		&retries,
+		"retries",
+		"r",
+		"1",
+		"number of retry attempts",
+	)
+
+	AppBackupCreateCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
 }

cli/app/check.go

@@ -1,57 +1,91 @@
 package app
 
 import (
-	"os"
-	"path"
-	"strings"
+	"fmt"
 
 	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/spf13/cobra"
 )
 
-var appCheckCommand = cli.Command{
-	Name:      "check",
-	Aliases:   []string{"chk"},
-	Usage:     "Check if app is configured correctly",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-	},
-	Before: internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+var AppCheckCommand = &cobra.Command{
+	Use:     "check <domain> [flags]",
+	Aliases: []string{"chk"},
+	Short:   "Ensure an app is well configured",
+	Long: `Compare env vars in both the app ".env" and recipe ".env.sample" file.
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
-		if _, err := os.Stat(envSamplePath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
-			}
-			logrus.Fatal(err)
+The goal is to ensure that recipe ".env.sample" env vars are defined in your
+app ".env" file. Only env var definitions in the ".env.sample" which are
+uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include
+these env vars, then "check" will complain.
+
+Recipe maintainers may or may not provide defaults for env vars within their
+recipes regardless of commenting or not (e.g. through the use of
+${FOO:<default>} syntax). "check" does not confirm or deny this for you.`,
+	Args: cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		envSample, err := config.ReadEnv(envSamplePath)
+		table, err := formatter.CreateTable()
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
-		var missing []string
-		for k := range envSample {
-			if _, ok := app.Env[k]; !ok {
-				missing = append(missing, k)
+		table.
+			Headers(
+				fmt.Sprintf("%s .env.sample", app.Recipe.Name),
+				fmt.Sprintf("%s.env", app.Name),
+			).
+			StyleFunc(func(row, col int) lipgloss.Style {
+				switch {
+				case col == 1:
+					return lipgloss.NewStyle().Padding(0, 1, 0, 1).Align(lipgloss.Center)
+				default:
+					return lipgloss.NewStyle().Padding(0, 1, 0, 1)
+				}
+			})
+
+		envVars, err := appPkg.CheckEnv(app)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if envVar.Present {
+				val := []string{envVar.Name, "✅"}
+				table.Row(val...)
+			} else {
+				val := []string{envVar.Name, "❌"}
+				table.Row(val...)
 			}
 		}
 
-		if len(missing) > 0 {
-			missingEnvVars := strings.Join(missing, ", ")
-			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
+		if err := formatter.PrintTable(table); err != nil {
+			log.Fatal(err)
 		}
-
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
-
-		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
+}
+
+func init() {
+	AppCheckCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
 }

cli/app/cmd.go

@@ -1,233 +1,276 @@
 package app
 
 import (
-	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
-	"path"
+	"slices"
+	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"coopcloud.tech/abra/pkg/log"
+	"github.com/spf13/cobra"
 )
 
-var localCmd bool
-var localCmdFlag = &cli.BoolFlag{
-	Name:        "local, l",
-	Usage:       "Run command locally",
-	Destination: &localCmd,
-}
-
-var remoteUser string
-var remoteUserFlag = &cli.StringFlag{
-	Name:        "user, u",
-	Value:       "",
-	Usage:       "User to run command within a service context",
-	Destination: &remoteUser,
-}
-
-var appCmdCommand = cli.Command{
-	Name:    "command",
+var AppCmdCommand = &cobra.Command{
+	Use:     "command <domain> [service | --local] <cmd> [[args] [flags] | [flags] -- [args]]",
 	Aliases: []string{"cmd"},
-	Usage:   "Run app commands",
-	Description: `
-This command runs app specific commands.
+	Short:   "Run app commands",
+	Long: `Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
-work station by passing "--local". Arguments can be passed into these functions
-using the "-- <args>" syntax.
+work station by passing "--local/-l".
 
-Example:
+N.B. If using the "--" style to pass arguments, flags (e.g. "--local/-l") must
+be passed *before* the "--". It is possible to pass arguments without the "--"
+as long as no dashes are present (i.e. "foo" works without "--", "-foo"
+does not).`,
+	Example: `  # pass <cmd> args/flags without "--"
+  abra app cmd 1312.net app my_cmd_arg foo --user bar
 
-  abra app cmd example.com app create_user -- me@example.com
-`,
-	ArgsUsage: "<domain> [<service>] <command>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		localCmdFlag,
-		remoteUserFlag,
-	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
+  # pass <cmd> args/flags with "--"
+  abra app cmd 1312.net app my_cmd_args --user bar -- foo -vvv
 
-		if localCmd && remoteUser != "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & <user> together"))
-		}
-
-		abraSh := path.Join(config.RECIPES_DIR, app.Recipe, "abra.sh")
-		if _, err := os.Stat(abraSh); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist for %s?", abraSh, app.Name)
-			}
-			logrus.Fatal(err)
-		}
-
-		var parsedCmdArgs string
-		var cmdArgsIdx int
-		var hasCmdArgs bool
-		for idx, arg := range c.Args() {
-			if arg == "--" {
-				cmdArgsIdx = idx
-				hasCmdArgs = true
+  # drop the [service] arg if using "--local/-l"
+  abra app cmd 1312.net my_cmd --local`,
+	Args: func(cmd *cobra.Command, args []string) error {
+		if local {
+			if !(len(args) >= 2) {
+				return errors.New("requires at least 2 arguments with --local/-l")
 			}
 
-			if hasCmdArgs && idx > cmdArgsIdx {
-				parsedCmdArgs += fmt.Sprintf("%s ", c.Args().Get(idx))
-			}
-		}
-
-		if localCmd {
-			cmdName := c.Args().Get(1)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
-				logrus.Fatal(err)
-			}
-
-			logrus.Debugf("--local detected, running %s on local work station", cmdName)
-
-			var sourceAndExec string
-			if hasCmdArgs {
-				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; . %s; %s %s", app.StackName(), abraSh, cmdName, parsedCmdArgs)
-			} else {
-				logrus.Debug("did not detect any command arguments")
-				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; . %s; %s", app.StackName(), abraSh, cmdName)
-			}
-
-			cmd := exec.Command("/bin/sh", "-c", sourceAndExec)
-
-			if err := internal.RunCmd(cmd); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			targetServiceName := c.Args().Get(1)
-
-			cmdName := c.Args().Get(2)
-			if err := ensureCommand(abraSh, app.Recipe, cmdName); err != nil {
-				logrus.Fatal(err)
-			}
-
-			serviceNames, err := config.GetAppServiceNames(app.Name)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			matchingServiceName := false
-			for _, serviceName := range serviceNames {
-				if serviceName == targetServiceName {
-					matchingServiceName = true
+			if slices.Contains(os.Args, "--") {
+				if cmd.ArgsLenAtDash() > 2 {
+					return errors.New("accepts at most 2 args with --local/-l")
 				}
 			}
 
-			if !matchingServiceName {
-				logrus.Fatalf("no service %s for %s?", targetServiceName, app.Name)
-			}
+			// NOTE(d1): it is unclear how to correctly validate this case
+			//
+			// abra app cmd 1312.net app test_cmd_args foo --local
+			// FATAL <recipe> doesn't have a app function
+			//
+			// "app" should not be there, but there is no reliable way to detect arg
+			// count when the user can pass an arbitrary amount of recipe command
+			// arguments
+			return nil
+		}
 
-			logrus.Debugf("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName)
-
-			if hasCmdArgs {
-				logrus.Debugf("parsed following command arguments: %s", parsedCmdArgs)
-			} else {
-				logrus.Debug("did not detect any command arguments")
-			}
-
-			if err := runCmdRemote(app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
-				logrus.Fatal(err)
-			}
+		if !(len(args) >= 3) {
+			return errors.New("requires at least 3 arguments")
 		}
 
 		return nil
 	},
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		case 1:
+			if !local {
+				return autocomplete.ServiceNameComplete(args[0])
+			}
+			return autocomplete.CommandNameComplete(args[0])
+		case 2:
+			if !local {
+				return autocomplete.CommandNameComplete(args[0])
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		default:
+			return nil, cobra.ShellCompDirectiveError
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
+
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
+
+		if local && remoteUser != "" {
+			log.Fatal("cannot use --local & --user together")
+		}
+
+		hasCmdArgs, parsedCmdArgs := parseCmdArgs(args, local)
+
+		if _, err := os.Stat(app.Recipe.AbraShPath); err != nil {
+			if os.IsNotExist(err) {
+				log.Fatalf("%s does not exist for %s?", app.Recipe.AbraShPath, app.Name)
+			}
+			log.Fatal(err)
+		}
+
+		if local {
+			cmdName := args[1]
+			if err := internal.EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+				log.Fatal(err)
+			}
+
+			log.Debugf("--local detected, running %s on local work station", cmdName)
+
+			var exportEnv string
+			for k, v := range app.Env {
+				exportEnv = exportEnv + fmt.Sprintf("%s='%s'; ", k, v)
+			}
+
+			var sourceAndExec string
+			if hasCmdArgs {
+				log.Debugf("parsed following command arguments: %s", parsedCmdArgs)
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s %s", app.Name, app.StackName(), exportEnv, app.Recipe.AbraShPath, cmdName, parsedCmdArgs)
+			} else {
+				log.Debug("did not detect any command arguments")
+				sourceAndExec = fmt.Sprintf("TARGET=local; APP_NAME=%s; STACK_NAME=%s; %s . %s; %s", app.Name, app.StackName(), exportEnv, app.Recipe.AbraShPath, cmdName)
+			}
+
+			shell := "/bin/bash"
+			if _, err := os.Stat(shell); errors.Is(err, os.ErrNotExist) {
+				log.Debugf("%s does not exist locally, use /bin/sh as fallback", shell)
+				shell = "/bin/sh"
+			}
+			cmd := exec.Command(shell, "-c", sourceAndExec)
+
+			if err := internal.RunCmd(cmd); err != nil {
+				log.Fatal(err)
+			}
+
+			return
+		}
+
+		cmdName := args[2]
+		if err := internal.EnsureCommand(app.Recipe.AbraShPath, app.Recipe.Name, cmdName); err != nil {
+			log.Fatal(err)
+		}
+
+		serviceNames, err := appPkg.GetAppServiceNames(app.Name)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		matchingServiceName := false
+		targetServiceName := args[1]
+		for _, serviceName := range serviceNames {
+			if serviceName == targetServiceName {
+				matchingServiceName = true
+			}
+		}
+
+		if !matchingServiceName {
+			log.Fatalf("no service %s for %s?", targetServiceName, app.Name)
+		}
+
+		log.Debugf("running command %s within the context of %s_%s", cmdName, app.StackName(), targetServiceName)
+
+		if hasCmdArgs {
+			log.Debugf("parsed following command arguments: %s", parsedCmdArgs)
+		} else {
+			log.Debug("did not detect any command arguments")
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := internal.RunCmdRemote(
+			cl,
+			app,
+			requestTTY,
+			app.Recipe.AbraShPath,
+			targetServiceName, cmdName, parsedCmdArgs, remoteUser); err != nil {
+			log.Fatal(err)
+		}
+	},
 }
 
-func ensureCommand(abraSh, recipeName, execCmd string) error {
-	bytes, err := ioutil.ReadFile(abraSh)
-	if err != nil {
-		return err
-	}
+var AppCmdListCommand = &cobra.Command{
+	Use:     "list <domain> [flags]",
+	Aliases: []string{"ls"},
+	Short:   "List all available commands",
+	Args:    cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-	if !strings.Contains(string(bytes), execCmd) {
-		return fmt.Errorf("%s doesn't have a %s function", recipeName, execCmd)
-	}
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
+		}
 
-	return nil
+		cmdNames, err := appPkg.ReadAbraShCmdNames(app.Recipe.AbraShPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		sort.Strings(cmdNames)
+
+		for _, cmdName := range cmdNames {
+			fmt.Println(cmdName)
+		}
+	},
 }
 
-func runCmdRemote(app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
-	cl, err := client.New(app.Server)
-	if err != nil {
-		return err
-	}
+func parseCmdArgs(args []string, isLocal bool) (bool, string) {
+	var (
+		parsedCmdArgs string
+		hasCmdArgs    bool
+	)
 
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(targetContainer.ID), app.Server)
-
-	toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-	content, err := archive.TarWithOptions(abraSh, toTarOpts)
-	if err != nil {
-		return err
-	}
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, "/tmp", content, copyOpts); err != nil {
-		return err
-	}
-
-	var cmd []string
-	if cmdArgs != "" {
-		cmd = []string{"/bin/sh", "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; . /tmp/abra.sh; %s %s", serviceName, app.StackName(), cmdName, cmdArgs)}
+	if isLocal {
+		if len(args) > 2 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[2:], " "))
+		}
 	} else {
-		cmd = []string{"/bin/sh", "-c", fmt.Sprintf("TARGET=%s; APP_NAME=%s; . /tmp/abra.sh; %s", serviceName, app.StackName(), cmdName)}
+		if len(args) > 3 {
+			return true, fmt.Sprintf("%s ", strings.Join(args[3:], " "))
+		}
 	}
 
-	logrus.Debugf("running command: %s", strings.Join(cmd, " "))
-
-	execCreateOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          cmd,
-		Detach:       false,
-		Tty:          true,
-	}
-
-	if remoteUser != "" {
-		logrus.Debugf("running command with user %s", remoteUser)
-		execCreateOpts.User = remoteUser
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
-		return err
-	}
-
-	return nil
+	return hasCmdArgs, parsedCmdArgs
+}
+
+var (
+	local      bool
+	remoteUser string
+	requestTTY bool
+)
+
+func init() {
+	AppCmdCommand.Flags().BoolVarP(
+		&local,
+		"local",
+		"l",
+		false,
+		"run command locally",
+	)
+
+	AppCmdCommand.Flags().StringVarP(
+		&remoteUser,
+		"user",
+		"u",
+		"",
+		"request remote user",
+	)
+
+	AppCmdCommand.Flags().BoolVarP(
+		&requestTTY,
+		"tty",
+		"T",
+		false,
+		"request remote TTY",
+	)
+
+	AppCmdCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
 }

cli/app/cmd_test.go

@@ -0,0 +1,31 @@
+package app
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseCmdArgs(t *testing.T) {
+	tests := []struct {
+		input          []string
+		shouldParse    bool
+		expectedOutput string
+	}{
+		// `--` is not parsed when passed in from the command-line e.g. -- foo bar baz
+		// so we need to eumlate that as missing when testing if bash args are passed in
+		// see https://git.coopcloud.tech/toolshed/organising/issues/336 for more
+		{[]string{"foo.com", "app", "test"}, false, ""},
+		{[]string{"foo.com", "app", "test", "foo"}, true, "foo "},
+		{[]string{"foo.com", "app", "test", "foo", "bar", "baz"}, true, "foo bar baz "},
+	}
+
+	for _, test := range tests {
+		ok, parsed := parseCmdArgs(test.input, false)
+		if ok != test.shouldParse {
+			t.Fatalf("[%s] should not parse", strings.Join(test.input, " "))
+		}
+		if parsed != test.expectedOutput {
+			t.Fatalf("%s does not match %s", parsed, test.expectedOutput)
+		}
+	}
+}

cli/app/config.go

@@ -1,64 +1,57 @@
 package app
 
 import (
-	"errors"
 	"os"
 	"os/exec"
 
-	"coopcloud.tech/abra/cli/internal"
+	appPkg "coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/log"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appConfigCommand = cli.Command{
-	Name:      "config",
-	Aliases:   []string{"cfg"},
-	Usage:     "Edit app config",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
+var AppConfigCommand = &cobra.Command{
+	Use:     "config <domain> [flags]",
+	Aliases: []string{"cfg"},
+	Short:   "Edit app config",
+	Example: "  abra config 1312.net",
+	Args:    cobra.ExactArgs(1),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		return autocomplete.AppNameComplete()
 	},
-	Before: internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		appName := c.Args().First()
-
-		if appName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("no app provided"))
-		}
-
-		files, err := config.LoadAppFiles("")
+	Run: func(cmd *cobra.Command, args []string) {
+		files, err := appPkg.LoadAppFiles("")
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
 
+		appName := args[0]
 		appFile, exists := files[appName]
 		if !exists {
-			logrus.Fatalf("cannot find app with name %s", appName)
+			log.Fatalf("cannot find app with name %s", appName)
 		}
 
 		ed, ok := os.LookupEnv("EDITOR")
 		if !ok {
 			edPrompt := &survey.Select{
-				Message: "Which editor do you wish to use?",
+				Message: "which editor do you wish to use?",
 				Options: []string{"vi", "vim", "nvim", "nano", "pico", "emacs"},
 			}
 			if err := survey.AskOne(edPrompt, &ed); err != nil {
-				logrus.Fatal(err)
+				log.Fatal(err)
 			}
 		}
 
-		cmd := exec.Command(ed, appFile.Path)
-		cmd.Stdin = os.Stdin
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if err := cmd.Run(); err != nil {
-			logrus.Fatal(err)
+		c := exec.Command(ed, appFile.Path)
+		c.Stdin = os.Stdin
+		c.Stdout = os.Stdout
+		c.Stderr = os.Stderr
+		if err := c.Run(); err != nil {
+			log.Fatal(err)
 		}
-
-		return nil
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }

cli/app/cp.go

@@ -2,150 +2,382 @@ package app
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"os"
+	"path"
+	"path/filepath"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/container"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
+	"coopcloud.tech/abra/pkg/log"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	containertypes "github.com/docker/docker/api/types/container"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/spf13/cobra"
 )
 
-var appCpCommand = cli.Command{
-	Name:      "cp",
-	Aliases:   []string{"c"},
-	ArgsUsage: "<domain> <src> <dst>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
+var AppCpCommand = &cobra.Command{
+	Use:     "cp <domain> <src> <dst> [flags]",
+	Aliases: []string{"c"},
+	Short:   "Copy files to/from a deployed app service",
+	Example: `  # copy myfile.txt to the root of the app service
+  abra app cp 1312.net myfile.txt app:/
+
+  # copy that file back to your current working directory locally
+  abra app cp 1312.net app:/myfile.txt`,
+	Args: cobra.ExactArgs(3),
+	ValidArgsFunction: func(
+		cmd *cobra.Command,
+		args []string,
+		toComplete string) ([]string, cobra.ShellCompDirective) {
+		switch l := len(args); l {
+		case 0:
+			return autocomplete.AppNameComplete()
+		default:
+			return nil, cobra.ShellCompDirectiveDefault
+		}
 	},
-	Before: internal.SubCommandBefore,
-	Usage:  "Copy files to/from a running app service",
-	Description: `
-This command supports copying files to and from any app service file system.
+	Run: func(cmd *cobra.Command, args []string) {
+		app := internal.ValidateApp(args)
 
-If you want to copy a myfile.txt to the root of the app service:
-
-    abra app cp <domain> myfile.txt app:/
-
-And if you want to copy that file back to your current working directory locally:
-
-		abra app cp <domain> app:/myfile.txt .
-`,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		src := c.Args().Get(1)
-		dst := c.Args().Get(2)
-		if src == "" {
-			logrus.Fatal("missing <src> argument")
-		} else if dst == "" {
-			logrus.Fatal("missing <dest> argument")
+		if err := app.Recipe.Ensure(internal.GetEnsureContext()); err != nil {
+			log.Fatal(err)
 		}
 
-		parsedSrc := strings.SplitN(src, ":", 2)
-		parsedDst := strings.SplitN(dst, ":", 2)
-		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
-		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
-			logrus.Fatal(errorMsg)
-		} else if len(parsedSrc) != 2 {
-			if len(parsedDst) != 2 {
-				logrus.Fatal(errorMsg)
-			}
-		} else if len(parsedDst) != 2 {
-			if len(parsedSrc) != 2 {
-				logrus.Fatal(errorMsg)
-			}
-		}
-
-		var service string
-		var srcPath string
-		var dstPath string
-		isToContainer := false // <container:src> <dst>
-		if len(parsedSrc) == 2 {
-			service = parsedSrc[0]
-			srcPath = parsedSrc[1]
-			dstPath = dst
-			logrus.Debugf("assuming transfer is coming FROM the container")
-		} else if len(parsedDst) == 2 {
-			service = parsedDst[0]
-			dstPath = parsedDst[1]
-			srcPath = src
-			isToContainer = true // <src> <container:dst>
-			logrus.Debugf("assuming transfer is going TO the container")
-		}
-
-		if !isToContainer {
-			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist locally?", dstPath)
-			}
-		}
-		err := configureAndCp(c, app, srcPath, dstPath, service, isToContainer)
+		src := args[1]
+		dst := args[2]
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
 		if err != nil {
-			logrus.Fatal(err)
+			log.Fatal(err)
 		}
-		return nil
 
+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+
+		if toContainer {
+			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
+		} else {
+			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
 	},
-	BashComplete: autocomplete.AppNameComplete,
 }
 
-func configureAndCp(
-	c *cli.Context,
-	app config.App,
-	srcPath string,
-	dstPath string,
-	service string,
-	isToContainer bool) error {
-	cl, err := client.New(app.Server)
+var errServiceMissing = errors.New("one of <src>/<dest> arguments must take $SERVICE:$PATH form")
+
+// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH
+func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) {
+	parsedSrc := strings.SplitN(src, ":", 2)
+	parsedDst := strings.SplitN(dst, ":", 2)
+	if len(parsedSrc)+len(parsedDst) != 3 {
+		return "", "", "", false, errServiceMissing
+	}
+	if len(parsedSrc) == 2 {
+		return parsedSrc[1], dst, parsedSrc[0], false, nil
+	}
+	if len(parsedDst) == 2 {
+		return src, parsedDst[1], parsedDst[0], true, nil
+	}
+	return "", "", "", false, errServiceMissing
+}
+
+// CopyToContainer copies a file or directory from the local file system to the container.
+// See the possible copy modes and their documentation.
+func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := os.Stat(srcPath)
 	if err != nil {
-		logrus.Fatal(err)
+		return fmt.Errorf("local %s ", err)
 	}
 
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
-
-	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
+	dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath)
+	dstExists := true
 	if err != nil {
-		logrus.Fatal(err)
+		if errdefs.IsNotFound(err) {
+			dstExists = false
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
 	}
 
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists)
+	if err != nil {
+		return err
+	}
+	movePath := ""
+	switch mode {
+	case CopyModeDirToDir:
+		// Add the src directory to the destination path
+		_, srcDir := path.Split(srcPath)
+		dstPath = path.Join(dstPath, srcDir)
 
-	if isToContainer {
-		if _, err := os.Stat(srcPath); err != nil {
-			logrus.Fatalf("%s does not exist?", srcPath)
-		}
-
-		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		content, err := archive.TarWithOptions(srcPath, toTarOpts)
+		// Make sure the dst directory exits.
+		dcli, err := command.NewDockerCli()
 		if err != nil {
-			logrus.Fatal(err)
+			return err
 		}
+		if _, err := container.RunExec(dcli, cl, containerID, &containertypes.ExecOptions{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mkdir", "-p", dstPath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return fmt.Errorf("create remote directory: %s", err)
+		}
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		movePath = dstPath
+		dstPath = "/tmp"
+	}
 
-		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	} else {
-		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
+	toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(srcPath, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("copy %s from local to %s on container", srcPath, dstPath)
+	copyOpts := containertypes.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
+		return err
+	}
+
+	if movePath != "" {
+		_, srcFile := path.Split(srcPath)
+		dcli, err := command.NewDockerCli()
 		if err != nil {
-			logrus.Fatal(err)
+			return err
 		}
-		defer content.Close()
-		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-			logrus.Fatal(err)
+		if _, err := container.RunExec(dcli, cl, containerID, &containertypes.ExecOptions{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mv", path.Join("/tmp", srcFile), movePath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return fmt.Errorf("create remote directory: %s", err)
 		}
 	}
 
 	return nil
 }
+
+// CopyFromContainer copies a file or directory from the given container to the local file system.
+// See the possible copy modes and their documentation.
+func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return fmt.Errorf("remote: %s does not exist", srcPath)
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	}
+
+	dstStat, err := os.Stat(dstPath)
+	dstExists := true
+	var dstMode os.FileMode
+	if err != nil {
+		if os.IsNotExist(err) {
+			dstExists = false
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	} else {
+		dstMode = dstStat.Mode()
+	}
+
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists)
+	if err != nil {
+		return err
+	}
+
+	moveDstDir := ""
+	moveDstFile := ""
+	switch mode {
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		moveDstFile = dstPath
+		dstPath = "/tmp"
+	case CopyModeFilesToDir:
+		// Copy the directory to the temp directory and move it to its
+		// dstPath afterwards.
+		moveDstDir = path.Join(dstPath, "/")
+		dstPath = "/tmp"
+
+		// Make sure the temp directory always gets removed
+		defer os.Remove(path.Join("/tmp"))
+	}
+
+	content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath)
+	if err != nil {
+		return fmt.Errorf("copy: %s", err)
+	}
+	defer content.Close()
+	if err := archive.Untar(content, dstPath, &archive.TarOptions{
+		NoOverwriteDirNonDir: true,
+		Compression:          archive.Gzip,
+		NoLchown:             true,
+	}); err != nil {
+		return fmt.Errorf("untar: %s", err)
+	}
+
+	if moveDstFile != "" {
+		_, srcFile := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil {
+			return err
+		}
+	}
+	if moveDstDir != "" {
+		_, srcDir := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var (
+	ErrCopyDirToFile  = fmt.Errorf("can't copy dir to file")
+	ErrDstDirNotExist = fmt.Errorf("destination directory does not exist")
+)
+
+type CopyMode int
+
+const (
+	// Copy a src file to a dest file. The src and dest file names are the same.
+	//  <dir_src>/<file> + <dir_dst>/<file> -> <dir_dst>/<file>
+	CopyModeFileToFile = CopyMode(iota)
+	// Copy a src file to a dest file. The src and dest file names are  not the same.
+	//  <dir_src>/<file_src> + <dir_dst>/<file_dst> -> <dir_dst>/<file_dst>
+	CopyModeFileToFileRename
+	// Copy a src file to dest directory. The dest file gets created in the dest
+	// folder with the src filename.
+	//  <dir_src>/<file> + <dir_dst> -> <dir_dst>/<file>
+	CopyModeFileToDir
+	// Copy a src directory to dest directory.
+	//  <dir_src> + <dir_dst> -> <dir_dst>/<dir_src>
+	CopyModeDirToDir
+	// Copy all files in the src directory to the dest directory. This works recursively.
+	//  <dir_src>/ + <dir_dst> -> <dir_dst>/<files_from_dir_src>
+	CopyModeFilesToDir
+)
+
+// copyMode takes a src and dest path and file mode to determine the copy mode.
+// See the possible copy modes and their documentation.
+func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) {
+	_, srcFile := path.Split(srcPath)
+	_, dstFile := path.Split(dstPath)
+	if srcMode.IsDir() {
+		if !dstExists {
+			return -1, ErrDstDirNotExist
+		}
+		if dstMode.IsDir() {
+			if strings.HasSuffix(srcPath, "/") {
+				return CopyModeFilesToDir, nil
+			}
+			return CopyModeDirToDir, nil
+		}
+		return -1, ErrCopyDirToFile
+	}
+
+	if dstMode.IsDir() {
+		return CopyModeFileToDir, nil
+	}
+
+	if srcFile != dstFile {
+		return CopyModeFileToFileRename, nil
+	}
+
+	return CopyModeFileToFile, nil
+}
+
+// moveDir moves all files from a source path to the destination path recursively.
+func moveDir(sourcePath, destPath string) error {
+	return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath))
+		if info.IsDir() {
+			err := os.Mkdir(newPath, info.Mode())
+			if err != nil {
+				if os.IsExist(err) {
+					return nil
+				}
+				return err
+			}
+		}
+		if info.Mode().IsRegular() {
+			return moveFile(p, newPath)
+		}
+		return nil
+	})
+}
+
+// moveFile moves a file from a source path to a destination path.
+func moveFile(sourcePath, destPath string) error {
+	inputFile, err := os.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	outputFile, err := os.Create(destPath)
+	if err != nil {
+		inputFile.Close()
+		return err
+	}
+	defer outputFile.Close()
+	_, err = io.Copy(outputFile, inputFile)
+	inputFile.Close()
+	if err != nil {
+		return err
+	}
+
+	// Remove file after succesfull copy.
+	err = os.Remove(sourcePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func init() {
+	AppCpCommand.Flags().BoolVarP(
+		&internal.Chaos,
+		"chaos",
+		"C",
+		false,
+		"ignore uncommitted recipes changes",
+	)
+}

cli/app/cp_test.go

@@ -0,0 +1,113 @@
+package app
+
+import (
+	"os"
+	"testing"
+)
+
+func TestParse(t