WIP: backup revolution [ci skip]

See coop-cloud/organising#485

.dockerignore

@@ -1,4 +1,8 @@
-Dockerfile
-.dockerignore
-*.swp
 *.swo
+*.swp
+.dockerignore
+Dockerfile
+abra
+dist
+kadabra
+tags

.drone.yml

@@ -7,16 +7,13 @@ steps:
     commands:
       - make check
 
-  - name: make build
-    image: golang:1.21
-    commands:
-      - make build
-    depends_on:
-      - make check
-
   - name: make test
     image: golang:1.21
+    environment:
+      ABRA_DIR: "/root/.abra"
     commands:
+      - make build-abra
+      - ./abra help # show version, initialise $ABRA_DIR
       - make test
     depends_on:
       - make check
@@ -27,7 +24,6 @@ steps:
       - git fetch --tags
     depends_on:
       - make check
-      - make build
       - make test
     when:
       event: tag
@@ -47,22 +43,22 @@ steps:
     when:
       event: tag
 
-  # - name: publish image
-  #   image: plugins/docker
-  #   settings:
-  #     auto_tag: true
-  #     username: 3wordchant
-  #     password:
-  #       from_secret: git_coopcloud_tech_token_3wc
-  #     repo: git.coopcloud.tech/coop-cloud/abra
-  #     tags: dev
-  #     registry: git.coopcloud.tech
-  #   when:
-  #     event:
-  #       exclude:
-  #         - pull_request
-  #   depends_on:
-  #     - make check
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: 3wordchant
+      password:
+        from_secret: git_coopcloud_tech_token_3wc
+      repo: git.coopcloud.tech/coop-cloud/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      event:
+        exclude:
+          - pull_request
+    depends_on:
+      - make check
 
 volumes:
   - name: deps

Dockerfile

@@ -2,7 +2,14 @@ FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
-RUN apk add --no-cache make git gcc musl-dev
+RUN apk add --no-cache \
+  ca-certificates \
+  gcc \
+  git \
+  make \
+  musl-dev
+
+RUN update-ca-certificates
 
 COPY . /app
 

Makefile

@@ -7,7 +7,8 @@ DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 
 export GOPRIVATE=coopcloud.tech
 
-all: format check build test
+# NOTE(d1): default `make` optimised for Abra hacking
+all: format check build-abra test
 
 run-abra:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)
@@ -18,19 +19,17 @@ run-kadabra:
 install-abra:
 	@go install -ldflags=$(LDFLAGS) $(ABRA)
 
-install-kadaabra:
+install-kadabra:
 	@go install -ldflags=$(LDFLAGS) $(KADABRA)
 
 build-abra:
-	@go build -v -ldflags=$(LDFLAGS) $(ABRA)
+	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
 
 build-kadabra:
-	@go build -v -ldflags=$(LDFLAGS) $(KADABRA)
-
-build:
-	@go build -v -ldflags=$(DIST_LDFLAGS) $(ABRA)
 	@go build -v -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
+build: build-abra build-kadabra
+
 clean:
 	@rm '$(GOPATH)/bin/abra'
 	@rm '$(GOPATH)/bin/kadabra'

cli/app/backup.go

@@ -1,414 +1,422 @@
 package app
 
 import (
-	"archive/tar"
 	"context"
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type backupConfig struct {
-	preHookCmd  string
-	postHookCmd string
-	backupPaths []string
+var snapshot string
+var snapshotFlag = &cli.StringFlag{
+	Name:        "snapshot, s",
+	Usage:       "Lists specific snapshot",
+	Destination: &snapshot,
 }
 
-var appBackupCommand = cli.Command{
-	Name:      "backup",
-	Aliases:   []string{"bk"},
-	Usage:     "Run app backup",
-	ArgsUsage: "<domain> [<service>]",
+var includePath string
+var includePathFlag = &cli.StringFlag{
+	Name:        "path, p",
+	Usage:       "Include path",
+	Destination: &includePath,
+}
+
+var resticRepo string
+var resticRepoFlag = &cli.StringFlag{
+	Name:        "repo, r",
+	Usage:       "Restic repository",
+	Destination: &resticRepo,
+}
+
+var appBackupListCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		snapshotFlag,
+		includePathFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	Usage:        "List all backups",
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app backup.
-
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		backupConfigs := make(map[string]backupConfig)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName != "" {
-			backupConfig, ok := backupConfigs[serviceName]
-			if !ok {
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-			logrus.Infof("running backup for the %s service", serviceName)
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
 
-			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if len(backupConfigs) == 0 {
-				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
-			}
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
 
-				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "ls"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
 
-// TimeStamp generates a file name friendly timestamp.
-func TimeStamp() string {
-	ts := time.Now().UTC().Format(time.RFC3339)
-	return strings.Replace(ts, ":", "-", -1)
-}
+var appBackupDownloadCommand = cli.Command{
+	Name:    "download",
+	Aliases: []string{"d"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+		includePathFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Download a backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-// runBackup does the actual backup logic.
-func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-		}
-		defer content.Close()
-
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			logrus.Fatal(err)
 		}
 
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
 		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
-	}
 
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
 
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
 
-		postHookExecOpts := types.ExecConfig{
+		execBackupListOpts := types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
-			Cmd:          splitCmd,
+			Cmd:          []string{"/usr/bin/backup", "--", "download"},
 			Detach:       false,
+			Env:          execEnv,
 			Tty:          true,
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
 		}
 
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
-
-	return nil
-}
-
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
-
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
-
-	return nil
+		return nil
+	},
 }
 
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
+var appBackupCreateCommand = cli.Command{
+	Name:    "create",
+	Aliases: []string{"c"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		resticRepoFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Create a new backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
-	}
 
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
-		}
-	}
-
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
-
-	return nil
-}
-
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
-
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
-
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
 			}
-			break
 		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
 		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
+
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if resticRepo != "" {
+			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
+			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
+		}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "create"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
 }
 
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
+var appBackupSnapshotsCommand = cli.Command{
+	Name:    "snapshots",
+	Aliases: []string{"s"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "List backup snapshots",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 
-	rc = fin
-	tr = tar.NewReader(rc)
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
-	return tr, rc, nil
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "snapshots"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+var appBackupCommand = cli.Command{
+	Name:      "backup",
+	Aliases:   []string{"b"},
+	Usage:     "Manage app backups",
+	ArgsUsage: "<domain>",
+	Subcommands: []cli.Command{
+		appBackupListCommand,
+		appBackupSnapshotsCommand,
+		appBackupDownloadCommand,
+		appBackupCreateCommand,
+	},
 }

cli/app/check.go

@@ -1,13 +1,10 @@
 package app
 
 import (
-	"os"
-	"path"
-	"strings"
-
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
@@ -15,9 +12,21 @@ import (
 )
 
 var appCheckCommand = cli.Command{
-	Name:      "check",
-	Aliases:   []string{"chk"},
-	Usage:     "Check if an app is configured correctly",
+	Name:    "check",
+	Aliases: []string{"chk"},
+	Usage:   "Ensure an app is well configured",
+	Description: `
+This command compares env vars in both the app ".env" and recipe ".env.sample"
+file.
+
+The goal is to ensure that recipe ".env.sample" env vars are defined in your
+app ".env" file. Only env var definitions in the ".env.sample" which are
+uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include
+these env vars, then "check" will complain.
+
+Recipe maintainers may or may not provide defaults for env vars within their
+recipes regardless of commenting or not (e.g. through the use of
+${FOO:<default>} syntax). "check" does not confirm or deny this for you.`,
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
@@ -49,32 +58,23 @@ var appCheckCommand = cli.Command{
 			}
 		}
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
-		if _, err := os.Stat(envSamplePath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
-			}
-			logrus.Fatal(err)
-		}
+		tableCol := []string{"recipe env sample", "app env"}
+		table := formatter.CreateTable(tableCol)
 
-		envSample, err := config.ReadEnv(envSamplePath)
+		envVars, err := config.CheckEnv(app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		var missing []string
-		for k := range envSample {
-			if _, ok := app.Env[k]; !ok {
-				missing = append(missing, k)
+		for _, envVar := range envVars {
+			if envVar.Present {
+				table.Append([]string{envVar.Name, "✅"})
+			} else {
+				table.Append([]string{envVar.Name, "❌"})
 			}
 		}
 
-		if len(missing) > 0 {
-			missingEnvVars := strings.Join(missing, ", ")
-			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
-		}
-
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+		table.Render()
 
 		return nil
 	},

cli/app/deploy.go

@@ -6,6 +6,7 @@ import (
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/secret"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -50,6 +51,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
@@ -91,6 +97,17 @@ recipes.
 			logrus.Fatal(err)
 		}
 
+		secStats, err := secret.PollSecretsStatus(cl, app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, secStat := range secStats {
+			if !secStat.CreatedOnRemote {
+				logrus.Fatalf("unable to deploy, secrets not generated (%s)?", secStat.LocalName)
+			}
+		}
+
 		if isDeployed {
 			if internal.Force || internal.Chaos {
 				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
@@ -100,7 +117,6 @@ recipes.
 		}
 
 		version := deployedVersion
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			version = specificVersion
 			logrus.Debugf("choosing %s as version to deploy", version)
@@ -188,6 +204,17 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, version)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
 			logrus.Fatal(err)
 		}

cli/app/logs.go

@@ -11,6 +11,7 @@ import (
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
@@ -86,6 +87,10 @@ var appLogsCommand = cli.Command{
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)

cli/app/new.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"fmt"
+	"path"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -96,7 +97,7 @@ var appNewCommand = cli.Command{
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
-			sampleEnv, err := recipe.SampleEnv()
+			sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -106,7 +107,8 @@ var appNewCommand = cli.Command{
 				logrus.Fatal(err)
 			}
 
-			secretsConfig, err := secret.ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 			if err != nil {
 				return err
 			}

cli/app/restore.go

@@ -2,9 +2,7 @@ package app
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -12,212 +10,112 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type restoreConfig struct {
-	preHookCmd  string
-	postHookCmd string
+var targetPath string
+var targetPathFlag = &cli.StringFlag{
+	Name:        "target, t",
+	Usage:       "Target path",
+	Destination: &targetPath,
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Run app restore",
-	ArgsUsage: "<domain> <service> <file>",
+	Usage:     "Restore an app backup",
+	ArgsUsage: "<domain> <service>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		targetPathFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app restore.
-
-Pre/post hook commands are defined in the recipe configuration. Abra reads this
-configuration and run the comands in the context of the service before
-restoring the backup.
-
-Unlike "abra app backup", restore must be run on a per-service basis. You can
-not restore all services in one go. Backup files produced by Abra are
-compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic, i.e. the backup will be restored to
-the path it was originally backed up from.
-
-Example:
-
-    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipe.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
-		}
-
-		backupPath := c.Args().Get(2)
-		if backupPath == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
-		}
-
-		if _, err := os.Stat(backupPath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s doesn't exist?", backupPath)
-			}
-		}
-
-		restoreConfigs := make(map[string]restoreConfig)
-		for _, service := range recipe.Config.Services {
-			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
-				if restoreEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					rsConfig := restoreConfig{}
-
-					logrus.Debugf("restore config detected for %s", fullServiceName)
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						rsConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						rsConfig.postHookCmd = postHookCmd
-					}
-
-					restoreConfigs[service.Name] = rsConfig
-				}
-			}
-		}
-
-		rsConfig, ok := restoreConfigs[serviceName]
-		if !ok {
-			rsConfig = restoreConfig{}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if targetPath != "" {
+			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
+			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
+		}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "restore"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
-
-// runRestore does the actual restore logic.
-func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if rsConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
-	}
-
-	backupReader, err := os.Open(backupPath)
-	if err != nil {
-		return err
-	}
-
-	content, err := archive.DecompressStream(backupReader)
-	if err != nil {
-		return err
-	}
-
-	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
-	// files according to the paths set in the compressed archive
-	restorePath := "/"
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
-		return err
-	}
-
-	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
-
-	if rsConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
-	}
-
-	return nil
-}

cli/app/rollback.go

@@ -51,6 +51,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
@@ -125,7 +130,6 @@ recipes.
 			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {

cli/app/secret.go

@@ -87,7 +87,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -276,7 +276,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -356,6 +356,7 @@ var appSecretLsCommand = cli.Command{
 		internal.DebugFlag,
 		internal.OfflineFlag,
 		internal.ChaosFlag,
+		internal.MachineReadableFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "List all secrets",
@@ -383,12 +384,7 @@ var appSecretLsCommand = cli.Command{
 			}
 		}
 
-		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -396,42 +392,27 @@ var appSecretLsCommand = cli.Command{
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
 		table := formatter.CreateTable(tableCol)
 
-		cl, err := client.New(app.Server)
+		secStats, err := secret.PollSecretsStatus(cl, app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, false)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteSecretNames := make(map[string]bool)
-		for _, cont := range secretList {
-			remoteSecretNames[cont.Spec.Annotations.Name] = true
-		}
-
-		for secretName, secretValue := range secretsConfig {
-			createdRemote := false
-			val, err := secret.ParseSecretValue(secretValue)
-			if err != nil {
-				logrus.Fatal(err)
+		for _, secStat := range secStats {
+			tableRow := []string{
+				secStat.LocalName,
+				secStat.Version,
+				secStat.RemoteName,
+				strconv.FormatBool(secStat.CreatedOnRemote),
 			}
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
-			if _, ok := remoteSecretNames[secretRemoteName]; ok {
-				createdRemote = true
-			}
-			tableRow := []string{secretName, val.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
 			table.Append(tableRow)
 		}
 
 		if table.NumLines() > 0 {
-			table.Render()
+			if internal.MachineReadable {
+				table.JSONRender()
+			} else {
+				table.Render()
+			}
 		} else {
 			logrus.Warnf("no secrets stored for %s", app.Name)
 		}

cli/app/upgrade.go

@@ -56,6 +56,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if !internal.Chaos {
 			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
@@ -126,7 +131,6 @@ recipes.
 			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {
@@ -254,6 +258,17 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}

cli/cli.go

@@ -14,8 +14,8 @@ import (
 	"coopcloud.tech/abra/cli/recipe"
 	"coopcloud.tech/abra/cli/server"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	cataloguePkg "coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -184,12 +184,8 @@ func newAbraApp(version, commit string) *cli.App {
 			}
 		}
 
-		if _, err := os.Stat(config.CATALOGUE_DIR); os.IsNotExist(err) {
-			url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
-			logrus.Warnf("local recipe catalogue is missing, retrieving now")
-			if err := git.Clone(config.CATALOGUE_DIR, url); err != nil {
-				logrus.Fatal(err)
-			}
+		if err := cataloguePkg.EnsureCatalogue(); err != nil {
+			logrus.Fatal(err)
 		}
 
 		logrus.Debugf("abra version %s, commit %s", version, commit)

cli/internal/cli.go

@@ -54,7 +54,7 @@ var Chaos bool
 // ChaosFlag turns on/off chaos functionality.
 var ChaosFlag = &cli.BoolFlag{
 	Name:        "chaos, C",
-	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Usage:       "Proceed with uncommitted recipes changes. Use with care!",
 	Destination: &Chaos,
 }
 

cli/recipe/lint.go

@@ -68,7 +68,7 @@ var recipeLintCommand = cli.Command{
 
 				skippedOutput := "-"
 				if skipped {
-					skippedOutput = "yes"
+					skippedOutput = "✅"
 				}
 
 				satisfied := false
@@ -87,9 +87,9 @@ var recipeLintCommand = cli.Command{
 					}
 				}
 
-				satisfiedOutput := "yes"
+				satisfiedOutput := "✅"
 				if !satisfied {
-					satisfiedOutput = "NO"
+					satisfiedOutput = "❌"
 					if skipped {
 						satisfiedOutput = "-"
 					}

go.mod

@@ -1,6 +1,6 @@
 module coopcloud.tech/abra
 
-go 1.18
+go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
@@ -8,7 +8,7 @@ require (
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/docker/cli v24.0.6+incompatible
-	github.com/docker/distribution v2.8.2+incompatible
+	github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/docker v24.0.6+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/go-git/go-git/v5 v5.9.0
@@ -35,6 +35,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
@@ -115,5 +116,5 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/sys v0.12.0
+	golang.org/x/sys v0.13.0
 )

go.sum

@@ -124,6 +124,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -132,6 +133,7 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -313,6 +315,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
@@ -332,6 +335,8 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
+github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
@@ -339,8 +344,8 @@ github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvM
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
-github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
+github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
 github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
@@ -369,6 +374,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -406,11 +412,13 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
 github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
 github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -663,6 +671,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -782,6 +791,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -881,6 +891,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -1304,8 +1315,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1573,6 +1584,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

pkg/catalogue/catalogue.go

@@ -14,48 +14,49 @@ import (
 
 // CatalogueSkipList is all the repos that are not recipes.
 var CatalogueSkipList = map[string]bool{
-	"abra":                         true,
-	"abra-apps":                    true,
-	"abra-aur":                     true,
-	"abra-bash":                    true,
-	"abra-capsul":                  true,
-	"abra-gandi":                   true,
-	"abra-hetzner":                 true,
-	"abra-integration-test-recipe": true,
-	"apps":                         true,
-	"aur-abra-git":                 true,
-	"auto-mirror":                  true,
-	"auto-recipes-catalogue-json":  true,
-	"backup-bot":                   true,
-	"backup-bot-two":               true,
-	"beta.coopcloud.tech":          true,
-	"comrade-renovate-bot":         true,
-	"coopcloud.tech":               true,
-	"coturn":                       true,
-	"docker-cp-deploy":             true,
-	"docker-dind-bats-kcov":        true,
-	"docs.coopcloud.tech":          true,
-	"drone-abra":                   true,
-	"example":                      true,
-	"gardening":                    true,
-	"go-abra":                      true,
-	"organising":                   true,
-	"pyabra":                       true,
-	"radicle-seed-node":            true,
-	"recipes-catalogue-json":       true,
-	"recipes-wishlist":             true,
-	"recipes.coopcloud.tech":       true,
-	"stack-ssh-deploy":             true,
-	"swarm-cronjob":                true,
-	"tagcmp":                       true,
-	"traefik-cert-dumper":          true,
-	"tyop":                         true,
+	"abra":                        true,
+	"abra-apps":                   true,
+	"abra-aur":                    true,
+	"abra-bash":                   true,
+	"abra-capsul":                 true,
+	"abra-gandi":                  true,
+	"abra-hetzner":                true,
+	"abra-test-recipe":            true,
+	"apps":                        true,
+	"aur-abra-git":                true,
+	"auto-mirror":                 true,
+	"auto-recipes-catalogue-json": true,
+	"backup-bot":                  true,
+	"backup-bot-two":              true,
+	"beta.coopcloud.tech":         true,
+	"comrade-renovate-bot":        true,
+	"coopcloud.tech":              true,
+	"coturn":                      true,
+	"docker-cp-deploy":            true,
+	"docker-dind-bats-kcov":       true,
+	"docs.coopcloud.tech":         true,
+	"drone-abra":                  true,
+	"example":                     true,
+	"gardening":                   true,
+	"go-abra":                     true,
+	"organising":                  true,
+	"pyabra":                      true,
+	"radicle-seed-node":           true,
+	"recipes-catalogue-json":      true,
+	"recipes-wishlist":            true,
+	"recipes.coopcloud.tech":      true,
+	"stack-ssh-deploy":            true,
+	"swarm-cronjob":               true,
+	"tagcmp":                      true,
+	"traefik-cert-dumper":         true,
+	"tyop":                        true,
 }
 
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
+		logrus.Warnf("local recipe catalogue is missing, retrieving now")
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
 		if err := gitPkg.Clone(catalogueDir, url); err != nil {
 			return err

pkg/compose/compose.go

@@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return err
 		}

pkg/config/app.go

@@ -149,15 +149,15 @@ func (a ByName) Less(i, j int) bool {
 	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
 }
 
-func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path)
+func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
+	env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
 
 	logrus.Debugf("read env %s from %s", env, appFile.Path)
 
-	app, err := newApp(env, name, appFile)
+	app, err := NewApp(env, name, appFile)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
 	}
@@ -165,8 +165,8 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	return app, nil
 }
 
-// newApp creates new App object
-func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+// NewApp creates new App object
+func NewApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	domain := env["DOMAIN"]
 
 	recipe, exists := env["RECIPE"]
@@ -232,7 +232,7 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
 		return App{}, fmt.Errorf("cannot find app with name %s", name)
 	}
 
-	app, err := readAppEnvFile(appFile, name)
+	app, err := ReadAppEnvFile(appFile, name)
 	if err != nil {
 		return App{}, err
 	}
@@ -437,27 +437,56 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
 	return statuses, nil
 }
 
+// ensurePathExists ensures that a path exists.
+func ensurePathExists(path string) error {
+	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
 // GetComposeFiles gets the list of compose files for an app (or recipe if you
 // don't already have an app) which should be merged into a composetypes.Config
 // while respecting the COMPOSE_FILE env var.
 func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
 	var composeFiles []string
 
-	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
-		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
+	composeFileEnvVar, ok := appEnv["COMPOSE_FILE"]
+	if !ok {
 		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path)
 		composeFiles = append(composeFiles, path)
 		return composeFiles, nil
 	}
 
-	composeFileEnvVar := appEnv["COMPOSE_FILE"]
-	envVars := strings.Split(composeFileEnvVar, ":")
-	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
-	for _, file := range strings.Split(composeFileEnvVar, ":") {
+	if !strings.Contains(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("COMPOSE_FILE detected, loading %s", path)
+		composeFiles = append(composeFiles, path)
+		return composeFiles, nil
+	}
+
+	numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1
+	envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles)
+	if len(envVars) != numComposeFiles {
+		return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar)
+	}
+
+	for _, file := range envVars {
 		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
 		composeFiles = append(composeFiles, path)
 	}
 
+	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
 	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
 
 	return composeFiles, nil

pkg/config/app_test.go

@@ -1,36 +1,108 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestNewApp(t *testing.T) {
-	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestReadAppEnvFile(t *testing.T) {
-	app, err := readAppEnvFile(expectedAppFile, appName)
+	app, err := config.ReadAppEnvFile(ExpectedAppFile, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestGetApp(t *testing.T) {
-	app, err := GetApp(expectedAppFiles, appName)
+	app, err := config.GetApp(ExpectedAppFiles, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
+	}
+}
+
+func TestGetComposeFiles(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		appEnv       map[string]string
+		composeFiles []string
+	}{
+		{
+			map[string]string{},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml:compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		composeFiles, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, composeFiles, test.composeFiles)
+	}
+}
+
+func TestGetComposeFilesError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct{ appEnv map[string]string }{
+		{map[string]string{"COMPOSE_FILE": "compose.yml::compose.foo.yml"}},
+		{map[string]string{"COMPOSE_FILE": "doesnt.exist.yml"}},
+	}
+
+	for _, test := range tests {
+		_, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err == nil {
+			t.Fatalf("should have failed: %v", test.appEnv)
+		}
 	}
 }

pkg/config/env.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/Autonomic-Cooperative/godotenv"
@@ -34,6 +36,13 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+var BackupbotLabel = "coop-cloud.backupbot.enabled"
+
+// envVarModifiers is a list of env var modifier strings. These are added to
+// env vars as comments and modify their processing by Abra, e.g. determining
+// how long secrets should be.
+var envVarModifiers = []string{"length"}
+
 // GetServers retrieves all servers.
 func GetServers() ([]string, error) {
 	var servers []string
@@ -48,18 +57,43 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
-// ReadEnv loads an app envivornment into a map.
-func ReadEnv(filePath string) (AppEnv, error) {
-	var envFile AppEnv
+// ReadEnvOptions modifies the ReadEnv processing of env vars.
+type ReadEnvOptions struct {
+	IncludeModifiers bool
+}
 
-	envFile, err := godotenv.Read(filePath)
+// ContainsEnvVarModifier determines if an env var contains a modifier.
+func ContainsEnvVarModifier(envVar string) bool {
+	for _, mod := range envVarModifiers {
+		if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
+			return true
+		}
+	}
+	return false
+}
+
+// ReadEnv loads an app envivornment into a map.
+func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
+	var envVars AppEnv
+
+	envVars, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
-	logrus.Debugf("read %s from %s", envFile, filePath)
+	for idx, envVar := range envVars {
+		if strings.Contains(envVar, "#") {
+			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
+				continue
+			}
+			vals := strings.Split(envVar, "#")
+			envVars[idx] = strings.TrimSpace(vals[0])
+		}
+	}
 
-	return envFile, nil
+	logrus.Debugf("read %s from %s", envVars, filePath)
+
+	return envVars, nil
 }
 
 // ReadServerNames retrieves all server names.
@@ -149,22 +183,71 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 		}
 		return envVars, err
 	}
+	defer file.Close()
+
+	exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`)
+	if err != nil {
+		return envVars, err
+	}
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, "export") {
-			splitVals := strings.Split(line, "export ")
+		txt := scanner.Text()
+		if exportRegex.MatchString(txt) {
+			splitVals := strings.Split(txt, "export ")
 			envVarDef := splitVals[len(splitVals)-1]
 			keyVal := strings.Split(envVarDef, "=")
 			if len(keyVal) != 2 {
-				return envVars, fmt.Errorf("couldn't parse %s", line)
+				return envVars, fmt.Errorf("couldn't parse %s", txt)
 			}
 			envVars[keyVal[0]] = keyVal[1]
 		}
 	}
 
-	logrus.Debugf("read %s from %s", envVars, abraSh)
+	if len(envVars) > 0 {
+		logrus.Debugf("read %s from %s", envVars, abraSh)
+	} else {
+		logrus.Debugf("read 0 env var exports from %s", abraSh)
+	}
+
+	return envVars, nil
+}
+
+type EnvVar struct {
+	Name    string
+	Present bool
+}
+
+func CheckEnv(app App) ([]EnvVar, error) {
+	var envVars []EnvVar
+
+	envSamplePath := path.Join(RECIPES_DIR, app.Recipe, ".env.sample")
+	if _, err := os.Stat(envSamplePath); err != nil {
+		if os.IsNotExist(err) {
+			return envVars, fmt.Errorf("%s does not exist?", envSamplePath)
+		}
+		return envVars, err
+	}
+
+	envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
+	if err != nil {
+		return envVars, err
+	}
+
+	var keys []string
+	for key := range envSample {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		if _, ok := app.Env[key]; ok {
+			envVars = append(envVars, EnvVar{Name: key, Present: true})
+		} else {
+			envVars = append(envVars, EnvVar{Name: key, Present: false})
+		}
+	}
 
 	return envVars, nil
 }

pkg/config/env_test.go

@@ -1,60 +1,62 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"reflect"
 	"strings"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
 
 // make sure these are in alphabetical order
-var tFolders = []string{"folder1", "folder2"}
-var tFiles = []string{"bar.env", "foo.env"}
+var TFolders = []string{"folder1", "folder2"}
+var TFiles = []string{"bar.env", "foo.env"}
 
-var appName = "ecloud"
-var serverName = "evil.corp"
+var AppName = "ecloud"
+var ServerName = "evil.corp"
 
-var expectedAppEnv = AppEnv{
+var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
 	"RECIPE": "ecloud",
 }
 
-var expectedApp = App{
-	Name:   appName,
-	Recipe: expectedAppEnv["RECIPE"],
-	Domain: expectedAppEnv["DOMAIN"],
-	Env:    expectedAppEnv,
-	Path:   expectedAppFile.Path,
-	Server: expectedAppFile.Server,
+var ExpectedApp = config.App{
+	Name:   AppName,
+	Recipe: ExpectedAppEnv["RECIPE"],
+	Domain: ExpectedAppEnv["DOMAIN"],
+	Env:    ExpectedAppEnv,
+	Path:   ExpectedAppFile.Path,
+	Server: ExpectedAppFile.Server,
 }
 
-var expectedAppFile = AppFile{
-	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
-	Server: serverName,
+var ExpectedAppFile = config.AppFile{
+	Path:   path.Join(ValidAbraConf, "servers", ServerName, AppName+".env"),
+	Server: ServerName,
 }
 
-var expectedAppFiles = map[string]AppFile{
-	appName: expectedAppFile,
+var ExpectedAppFiles = map[string]config.AppFile{
+	AppName: ExpectedAppFile,
 }
 
-// var expectedServerNames = []string{"evil.corp"}
-
 func TestGetAllFoldersInDirectory(t *testing.T) {
-	folders, err := GetAllFoldersInDirectory(testFolder)
+	folders, err := config.GetAllFoldersInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(folders, tFolders) {
-		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	if !reflect.DeepEqual(folders, TFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(TFolders, ","), strings.Join(folders, ","))
 	}
 }
 
 func TestGetAllFilesInDirectory(t *testing.T) {
-	files, err := GetAllFilesInDirectory(testFolder)
+	files, err := config.GetAllFilesInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,23 +64,183 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 	for _, file := range files {
 		fileNames = append(fileNames, file.Name())
 	}
-	if !reflect.DeepEqual(fileNames, tFiles) {
-		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	if !reflect.DeepEqual(fileNames, TFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(TFiles, ","), strings.Join(fileNames, ","))
 	}
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := ReadEnv(expectedAppFile.Path)
+	env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(env, expectedAppEnv) {
+	if !reflect.DeepEqual(env, ExpectedAppEnv) {
 		t.Fatalf(
 			"did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s",
-			expectedAppEnv["DOMAIN"],
-			expectedAppEnv["RECIPE"],
+			ExpectedAppEnv["DOMAIN"],
+			ExpectedAppEnv["RECIPE"],
 			env["DOMAIN"],
 			env["RECIPE"],
 		)
 	}
 }
+
+func TestReadAbraShEnvVars(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(abraShEnv) == 0 {
+		t.Error("at least one env var should be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_FOO"]; ok {
+		t.Error("INNER_FOO should not be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_BAZ"]; ok {
+		t.Error("INNER_BAZ should not be exported")
+	}
+
+	if _, ok := abraShEnv["OUTER_FOO"]; !ok {
+		t.Error("OUTER_FOO should be exported")
+	}
+}
+
+func TestCheckEnv(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if !envVar.Present {
+			t.Fatalf("%s should be present", envVar.Name)
+		}
+	}
+}
+
+func TestCheckEnvError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	delete(envSample, "DOMAIN")
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if envVar.Name == "DOMAIN" && envVar.Present {
+			t.Fatalf("%s should not be present", envVar.Name)
+		}
+	}
+}
+
+func TestContainsEnvVarModifier(t *testing.T) {
+	if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
+		t.Fatal("FOO contains no env var modifier")
+	}
+
+	if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
+		t.Fatal("FOO contains an env var modifier (length)")
+	}
+}
+
+func TestEnvVarCommentsRemoved(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envVar, exists := envSample["WITH_COMMENT"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "should be removed") {
+		t.Fatalf("comment from '%s' should be removed", envVar)
+	}
+
+	envVar, exists = envSample["SECRET_TEST_PASS_TWO_VERSION"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should have been removed")
+	}
+}
+
+func TestEnvVarModifiersIncluded(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
+	}
+}

pkg/container/container.go

@@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) != 1 {
+	if len(containers) > 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")

pkg/dns/dns.go

@@ -7,11 +7,16 @@ import (
 
 // EnsureIPv4 ensures that an ipv4 address is set for a domain name
 func EnsureIPv4(domainName string) (string, error) {
-	ipv4, err := net.ResolveIPAddr("ip", domainName)
+	ipv4, err := net.ResolveIPAddr("ip4", domainName)
 	if err != nil {
 		return "", err
 	}
 
+	// NOTE(d1): e.g. when there is only an ipv6 record available
+	if ipv4 == nil {
+		return "", fmt.Errorf("unable to resolve ipv4 address for %s", domainName)
+	}
+
 	return ipv4.String(), nil
 }
 

pkg/dns/dns_test.go

@@ -0,0 +1,64 @@
+package dns
+
+import (
+	"fmt"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
+	tests := []struct {
+		domainName     string
+		serverName     string
+		shouldValidate bool
+	}{
+		// NOTE(d1): DNS records get checked, so use something that is maintained
+		// within the federation. if you're here because of a failing test, try
+		// `dig +short <domain>` to ensure stuff matches first! If flakyness
+		// becomes an issue we can look into mocking
+		{"docs.coopcloud.tech", "coopcloud.tech", true},
+		{"docs.coopcloud.tech", "swarm.autonomic.zone", true},
+
+		// NOTE(d1): special case handling for "--local"
+		{"", "default", true},
+		{"", "local", true},
+
+		{"", "", false},
+		{"123", "", false},
+	}
+
+	for _, test := range tests {
+		_, err := EnsureDomainsResolveSameIPv4(test.domainName, test.serverName)
+		if err != nil && test.shouldValidate {
+			t.Fatal(err)
+		}
+
+		if err == nil && !test.shouldValidate {
+			t.Fatal(fmt.Errorf("should have failed but did not: %v", test))
+		}
+	}
+}
+
+func TestEnsureIpv4(t *testing.T) {
+	// NOTE(d1): DNS records get checked, so use something that is maintained
+	// within the federation. if you're here because of a failing test, try `dig
+	// +short <domain>` to ensure stuff matches first! If flakyness becomes an
+	// issue we can look into mocking
+	domainName := "collabora.ostrom.collective.tools"
+	serverName := "ostrom.collective.tools"
+
+	for i := 0; i < 15; i++ {
+		domainIpv4, err := EnsureIPv4(domainName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		serverIpv4, err := EnsureIPv4(serverName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, domainIpv4, serverIpv4)
+	}
+}

pkg/jsontable/jsontable.go

@@ -3,6 +3,7 @@ package jsontable
 import (
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/olekukonko/tablewriter"
 )
@@ -109,6 +110,9 @@ func (t *JSONTable) _JSONRenderInner() {
 		}
 		writeChar(t.out, '{')
 		for keyidx, key := range t.keys {
+			key := strings.ToLower(key)
+			key = strings.ReplaceAll(key, " ", "-")
+
 			value := "nil"
 			if keyidx < len(row) {
 				value = row[keyidx]
@@ -138,10 +142,8 @@ func (t *JSONTable) JSONRender() {
 
 		if t.hasCaption {
 			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
-
 		}
 		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
-
 	}
 
 	// write list

pkg/lint/recipe.go

@@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}

pkg/recipe/recipe.go

@@ -221,7 +221,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -249,9 +249,9 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}, nil
 }
 
-func (r Recipe) SampleEnv() (map[string]string, error) {
+func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, opts)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}

pkg/secret/secret.go

@@ -4,6 +4,7 @@
 package secret
 
 import (
+	"context"
 	"fmt"
 	"slices"
 	"strconv"
@@ -11,9 +12,11 @@ import (
 	"sync"
 
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/decentral1se/passgen"
+	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
@@ -66,9 +69,14 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeName string) (map[string]string, error) {
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
 	secretConfigs := make(map[string]string)
 
+	appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
+	if err != nil {
+		return secretConfigs, err
+	}
+
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
@@ -93,7 +101,7 @@ func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeNa
 		}
 
 		if !(slices.Contains(enabledSecrets, secretId)) {
-			logrus.Warnf("%s not enabled in recipe config, not generating", secretId)
+			logrus.Warnf("%s not enabled in recipe config, skipping", secretId)
 			continue
 		}
 
@@ -209,3 +217,66 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 	return secrets, nil
 }
+
+type secretStatus struct {
+	LocalName       string
+	RemoteName      string
+	Version         string
+	CreatedOnRemote bool
+}
+
+type secretStatuses []secretStatus
+
+// PollSecretsStatus checks status of secrets by comparing the local recipe
+// config and deploymend server state.
+func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, error) {
+	var secStats secretStatuses
+
+	composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+	if err != nil {
+		return secStats, err
+	}
+
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+	if err != nil {
+		return secStats, err
+	}
+
+	remoteSecretNames := make(map[string]bool)
+	for _, cont := range secretList {
+		remoteSecretNames[cont.Spec.Annotations.Name] = true
+	}
+
+	for secretName, secretValue := range secretsConfig {
+		createdRemote := false
+
+		val, err := ParseSecretValue(secretValue)
+		if err != nil {
+			return secStats, err
+		}
+
+		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+		if _, ok := remoteSecretNames[secretRemoteName]; ok {
+			createdRemote = true
+		}
+
+		secStats = append(secStats, secretStatus{
+			LocalName:       secretName,
+			RemoteName:      secretRemoteName,
+			Version:         val.Version,
+			CreatedOnRemote: createdRemote,
+		})
+	}
+
+	return secStats, nil
+}

pkg/secret/secret_test.go

@@ -18,13 +18,14 @@ func TestReadSecretsConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sampleEnv, err := recipe.SampleEnv()
+	sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
-	secretsFromConfig, err := ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/service/service.go

@@ -14,6 +14,69 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// GetService retrieves a service container based on a label. If prompt is true
+// and the retrievd count of service containers does not match 1, then a prompt
+// is presented to let the user choose. A count of 0 is handled gracefully.
+func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
+	services, err := cl.ServiceList(c, types.ServiceListOptions{})
+	if err != nil {
+		return swarm.Service{}, err
+	}
+
+	if len(services) == 0 {
+		return swarm.Service{}, fmt.Errorf("no services deployed?")
+	}
+
+	var backupServices []swarm.Service
+	for _, service := range services {
+		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
+			backupServices = append(backupServices, service)
+		}
+	}
+
+	if len(backupServices) == 0 {
+		return swarm.Service{}, fmt.Errorf("no backup services deployed?")
+	}
+
+	if len(backupServices) > 1 {
+		var servicesRaw []string
+		for _, service := range backupServices {
+			serviceName := service.Spec.Name
+			created := formatter.HumanDuration(service.CreatedAt.Unix())
+			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
+		}
+
+		if !prompt {
+			err := fmt.Errorf("expected 1 service but found %v: %s", len(backupServices), strings.Join(servicesRaw, " "))
+			return swarm.Service{}, err
+		}
+
+		logrus.Warnf("ambiguous service list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which service are you looking for?",
+			Options: servicesRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return swarm.Service{}, err
+		}
+
+		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, service := range backupServices {
+			serviceName := strings.ToLower(service.Spec.Name)
+			if serviceName == chosenService {
+				return service, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen service")
+	}
+
+	return backupServices[0], nil
+}
+
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.

pkg/upstream/convert/service.go

@@ -420,7 +420,7 @@ func convertServiceSecrets(
 			return nil, err
 		}
 
-		// NOTE(d1): strip # length=... modifiers
+		// NOTE(d1): strip all comments
 		if strings.Contains(obj.Name, "#") {
 			vals := strings.Split(obj.Name, "#")
 			obj.Name = strings.TrimSpace(vals[0])

scripts/installer/installer

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.8.0-beta"
+ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.0-beta"
+RC_VERSION="0.8.1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do

tests/integration/app_check.bats

@@ -35,7 +35,7 @@ setup(){
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
-  assert_output --partial 'all necessary environment variables defined'
+  refute_output --partial '❌'
 
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 }
@@ -111,12 +111,16 @@ setup(){
 }
 
 @test "error if missing env var" {
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+  refute_output --partial '❌'
+
   run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
   assert_success
 
   run $ABRA app check "$TEST_APP_DOMAIN" --chaos
-  assert_failure
-  assert_output --partial "NEW_VAR"
+  assert_success
+  assert_output --partial '❌'
 
   _checkout_recipe
 }

tests/integration/app_deploy.bats

@@ -272,6 +272,21 @@ teardown(){
   assert_success
 }
 
+@test "ensure domain is checked" {
+  appDomain="custom-html.DOESNTEXIST"
+
+  run $ABRA app new custom-html \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$appDomain"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$appDomain.env"
+
+  run $ABRA app deploy "$appDomain" --no-input
+  assert_failure
+  assert_output --partial 'no such host'
+}
+
 # bats test_tags=slow
 @test "skip domain check when requested" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
@@ -296,3 +311,54 @@ teardown(){
 
   _undeploy_app
 }
+
+@test "bail out if specific version and chaos" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
+# bats test_tags=slow
+@test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
+  _reset_recipe
+
+  run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  # NOTE(d1): --chaos used to bypass versions and access compose.extra_env.yml
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial "compose.yml"
+  assert_output --partial "compose.extra_env.yml"
+
+  _undeploy_app
+  _reset_app
+}
+
+@test "error if no secrets generated" {
+  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'unable to deploy, secrets not generated'
+
+  _reset_app
+}
+
+@test "recipe config comments not present in values" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app env
+  assert_success
+  refute_output --partial 'should be removed'
+}

tests/integration/app_logs.bats

@@ -32,3 +32,15 @@ setup(){
   assert_failure
   assert_output --partial 'is not deployed'
 }
+
+@test "retrieve recipe if missing" {
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+
+  run $ABRA app logs "$TEST_APP_DOMAIN"
+  assert_failure
+  assert_output --partial 'is not deployed'
+
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
+}

tests/integration/app_rollback.bats

@@ -194,6 +194,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app rollback "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "rollback to previous version" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks

tests/integration/app_secret.bats

@@ -26,19 +26,6 @@ teardown(){
   fi
 }
 
-_reset_app(){
-  run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-  assert_success
-  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  run $ABRA app new "$TEST_RECIPE" \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-  assert_success
-  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-}
-
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
@@ -123,10 +110,11 @@ setup(){
   assert_success
   assert_output --partial 'test_pass_one'
 
-  run docker -c "$TEST_SERVER" secret ls
+  run bash -c '$ABRA app secret ls $TEST_APP_DOMAIN --machine | \
+    jq -r ".[] | select(.name==\"test_pass_one\") | .version"'
   assert_success
-  assert_output --regexp ".*_test_pass_one_v2"
-  refute_output --regexp ".*_test_pass_one_v1"
+  assert_output --partial 'v2'
+  refute_output --partial 'v1'
 
   run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
   assert_success
@@ -338,6 +326,22 @@ setup(){
   assert_success
 }
 
+@test "ls: show secrets as machine readable" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN" --machine
+  assert_success
+  assert_output --partial '"created-on-server":"true"'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+}
+
 @test "ls: bail if unstaged changes and no --chaos" {
   run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success

tests/integration/app_upgrade.bats

@@ -47,6 +47,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "no upgrade if lint error" {
   _deploy_app

tests/integration/helpers/app.bash

@@ -40,3 +40,16 @@ _rm_app() {
     run $ABRA app remove "$TEST_APP_DOMAIN" --no-input
   fi
 }
+
+_reset_app(){
+  run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN" \
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+}

tests/integration/helpers/common.bash

@@ -15,5 +15,5 @@ _common_setup() {
 
   export TEST_APP_NAME="$(basename "${BATS_TEST_FILENAME//./_}")"
   export TEST_APP_DOMAIN="$TEST_APP_NAME.$TEST_SERVER"
-  export TEST_RECIPE="abra-integration-test-recipe"
+  export TEST_RECIPE="abra-test-recipe"
 }