reset

AUTHORS.md

@@ -11,7 +11,6 @@
 - kawaiipunk
 - knoflook
 - moritz
-- p4u1
 - rix
 - roxxers
 - vera

Makefile

@@ -2,7 +2,6 @@ ABRA         := ./cmd/abra
 KADABRA      := ./cmd/kadabra
 COMMIT       := $(shell git rev-list -1 HEAD)
 GOPATH       := $(shell go env GOPATH)
-GOVERSION    := 1.21
 LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 
@@ -31,12 +30,6 @@ build-kadabra:
 
 build: build-abra build-kadabra
 
-build-docker-abra:
-	@docker run -it -v $(PWD):/abra golang:$(GOVERSION) \
-		bash -c 'cd /abra; ./scripts/docker/build.sh'
-
-build-docker: build-docker-abra
-
 clean:
 	@rm '$(GOPATH)/bin/abra'
 	@rm '$(GOPATH)/bin/kadabra'

cli/app/backup.go

@@ -1,296 +1,414 @@
 package app
 
 import (
+	"archive/tar"
+	"context"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var snapshot string
+type backupConfig struct {
-var snapshotFlag = &cli.StringFlag{
+	preHookCmd  string
-	Name:        "snapshot, s",
+	postHookCmd string
-	Usage:       "Lists specific snapshot",
+	backupPaths []string
-	Destination: &snapshot,
-}
-
-var includePath string
-var includePathFlag = &cli.StringFlag{
-	Name:        "path, p",
-	Usage:       "Include path",
-	Destination: &includePath,
-}
-
-var resticRepo string
-var resticRepoFlag = &cli.StringFlag{
-	Name:        "repo, r",
-	Usage:       "Restic repository",
-	Destination: &resticRepo,
-}
-
-var appBackupListCommand = cli.Command{
-	Name:    "list",
-	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List all backups",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}
-
-var appBackupDownloadCommand = cli.Command{
-	Name:    "download",
-	Aliases: []string{"d"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "Download a backup",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteBackupDir := "/tmp/backup.tar.gz"
-		currentWorkingDir := "."
-		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
-			logrus.Fatal(err)
-		}
-
-		fmt.Println("backup successfully downloaded to current working directory")
-
-		return nil
-	},
-}
-
-var appBackupCreateCommand = cli.Command{
-	Name:    "create",
-	Aliases: []string{"c"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		resticRepoFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "Create a new backup",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if resticRepo != "" {
-			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
-			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}
-
-var appBackupSnapshotsCommand = cli.Command{
-	Name:    "snapshots",
-	Aliases: []string{"s"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List backup snapshots",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
 }
 
 var appBackupCommand = cli.Command{
 	Name:      "backup",
-	Aliases:   []string{"b"},
+	Aliases:   []string{"bk"},
-	Usage:     "Manage app backups",
+	Usage:     "Run app backup",
-	ArgsUsage: "<domain>",
+	ArgsUsage: "<domain> [<service>]",
-	Subcommands: []cli.Command{
+	Flags: []cli.Flag{
-		appBackupListCommand,
+		internal.DebugFlag,
-		appBackupSnapshotsCommand,
+		internal.OfflineFlag,
-		appBackupDownloadCommand,
+		internal.ChaosFlag,
-		appBackupCreateCommand,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app backup.
+
+A backup command and pre/post hook commands are defined in the recipe
+configuration. Abra reads this configuration and run the comands in the context
+of the deployed services. Pass <service> if you only want to back up a single
+service. All backups are placed in the ~/.abra/backups directory.
+
+A single backup file is produced for all backup paths specified for a service.
+If we have the following backup configuration:
+
+    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
+
+And we run "abra app backup example.com app", Abra will produce a file that
+looks like:
+
+    ~/.abra/backups/example_com_app_609341138.tar.gz
+
+This file is a compressed archive which contains all backup paths. To see paths, run:
+
+    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
+
+(Make sure to change the name of the backup file)
+
+This single file can be used to restore your app. See "abra app restore" for more.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		backupConfigs := make(map[string]backupConfig)
+		for _, service := range recipe.Config.Services {
+			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
+				if backupsEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					bkConfig := backupConfig{}
+
+					logrus.Debugf("backup config detected for %s", fullServiceName)
+
+					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
+						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
+						bkConfig.backupPaths = strings.Split(paths, ",")
+					}
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						bkConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						bkConfig.postHookCmd = postHookCmd
+					}
+
+					backupConfigs[service.Name] = bkConfig
+				}
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName != "" {
+			backupConfig, ok := backupConfigs[serviceName]
+			if !ok {
+				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
+			}
+
+			logrus.Infof("running backup for the %s service", serviceName)
+
+			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if len(backupConfigs) == 0 {
+				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
+			}
+
+			for serviceName, backupConfig := range backupConfigs {
+				logrus.Infof("running backup for the %s service", serviceName)
+
+				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
 	},
 }
+
+// TimeStamp generates a file name friendly timestamp.
+func TimeStamp() string {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	return strings.Replace(ts, ":", "-", -1)
+}
+
+// runBackup does the actual backup logic.
+func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
+	if len(bkConfig.backupPaths) == 0 {
+		return fmt.Errorf("backup paths are empty for %s?", serviceName)
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if bkConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
+	}
+
+	var tempBackupPaths []string
+	for _, remoteBackupPath := range bkConfig.backupPaths {
+		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
+		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
+
+		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
+
+		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		if err != nil {
+			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+		}
+		defer content.Close()
+
+		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
+		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
+		if err := copyToFile(localBackupPath, preArchive); err != nil {
+			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+		}
+
+		tempBackupPaths = append(tempBackupPaths, localBackupPath)
+	}
+
+	logrus.Infof("compressing and merging archives...")
+
+	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
+		logrus.Debugf("failed to merge archive files: %s", err.Error())
+		if err := cleanupTempArchives(tempBackupPaths); err != nil {
+			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+		}
+		return fmt.Errorf("failed to merge archive files: %s", err.Error())
+	}
+
+	if err := cleanupTempArchives(tempBackupPaths); err != nil {
+		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+	}
+
+	if bkConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
+	}
+
+	return nil
+}
+
+func copyToFile(outfile string, r io.Reader) error {
+	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
+	if err != nil {
+		return err
+	}
+
+	tmpPath := tmpFile.Name()
+
+	_, err = io.Copy(tmpFile, r)
+	tmpFile.Close()
+
+	if err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	if err = os.Rename(tmpPath, outfile); err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	return nil
+}
+
+func cleanupTempArchives(tarPaths []string) error {
+	for _, tarPath := range tarPaths {
+		if err := os.RemoveAll(tarPath); err != nil {
+			return err
+		}
+
+		logrus.Debugf("remove temporary archive file %s", tarPath)
+	}
+
+	return nil
+}
+
+func mergeArchives(tarPaths []string, serviceName string) error {
+	var out io.Writer
+	var cout *pgzip.Writer
+
+	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
+
+	fout, err := os.Create(localBackupPath)
+	if err != nil {
+		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
+	}
+
+	defer fout.Close()
+	out = fout
+
+	cout = pgzip.NewWriter(out)
+	out = cout
+
+	tw := tar.NewWriter(out)
+
+	for _, tarPath := range tarPaths {
+		if err := addTar(tw, tarPath); err != nil {
+			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		return fmt.Errorf("failed to close tar writer %v", err)
+	}
+
+	if cout != nil {
+		if err := cout.Flush(); err != nil {
+			return fmt.Errorf("failed to flush: %s", err)
+		} else if err = cout.Close(); err != nil {
+			return fmt.Errorf("failed to close compressed writer: %s", err)
+		}
+	}
+
+	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
+
+	return nil
+}
+
+func addTar(tw *tar.Writer, pth string) (err error) {
+	var tr *tar.Reader
+	var rc io.ReadCloser
+	var hdr *tar.Header
+
+	if tr, rc, err = openTarFile(pth); err != nil {
+		return
+	}
+
+	for {
+		if hdr, err = tr.Next(); err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			break
+		}
+		if err = tw.WriteHeader(hdr); err != nil {
+			break
+		} else if _, err = io.Copy(tw, tr); err != nil {
+			break
+		}
+	}
+	if err == nil {
+		err = rc.Close()
+	} else {
+		rc.Close()
+	}
+	return
+}
+
+func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
+	var fin *os.File
+	var n int
+	buff := make([]byte, 1024)
+
+	if fin, err = os.Open(pth); err != nil {
+		return
+	}
+
+	if n, err = fin.Read(buff); err != nil {
+		fin.Close()
+		return
+	} else if n == 0 {
+		fin.Close()
+		err = fmt.Errorf("%s is empty", pth)
+		return
+	}
+
+	if _, err = fin.Seek(0, 0); err != nil {
+		fin.Close()
+		return
+	}
+
+	rc = fin
+	tr = tar.NewReader(rc)
+
+	return tr, rc, nil
+}

cli/app/cmd.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -46,17 +45,6 @@ Example:
 	},
 	Before:      internal.SubCommandBefore,
 	Subcommands: []cli.Command{appCmdListCommand},
-	BashComplete: func(ctx *cli.Context) {
-		args := ctx.Args()
-		switch len(args) {
-		case 0:
-			autocomplete.AppNameComplete(ctx)
-		case 1:
-			autocomplete.ServiceNameComplete(args.Get(0))
-		case 2:
-			cmdNameComplete(args.Get(0))
-		}
-	},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -199,20 +187,6 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 	return hasCmdArgs, parsedCmdArgs
 }
 
-func cmdNameComplete(appName string) {
-	app, err := app.Get(appName)
-	if err != nil {
-		return
-	}
-	cmdNames, _ := getShCmdNames(app)
-	if err != nil {
-		return
-	}
-	for _, n := range cmdNames {
-		fmt.Println(n)
-	}
-}
-
 var appCmdListCommand = cli.Command{
 	Name:      "list",
 	Aliases:   []string{"ls"},
@@ -248,11 +222,13 @@ var appCmdListCommand = cli.Command{
 			}
 		}
 
-		cmdNames, err := getShCmdNames(app)
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		sort.Strings(cmdNames)
 		for _, cmdName := range cmdNames {
 			fmt.Println(cmdName)
 		}
@@ -260,14 +236,3 @@ var appCmdListCommand = cli.Command{
 		return nil
 	},
 }
-
-func getShCmdNames(app config.App) ([]string, error) {
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
-	if err != nil {
-		return nil, err
-	}
-
-	sort.Strings(cmdNames)
-	return cmdNames, nil
-}

cli/app/cp.go

@@ -2,24 +2,19 @@ package app
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"io"
 	"os"
-	"path"
-	"path/filepath"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -54,14 +49,46 @@ And if you want to copy that file back to your current working directory locally
 		dst := c.Args().Get(2)
 		if src == "" {
 			logrus.Fatal("missing <src> argument")
-		}
+		} else if dst == "" {
-		if dst == "" {
 			logrus.Fatal("missing <dest> argument")
 		}
 
-		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
+		parsedSrc := strings.SplitN(src, ":", 2)
-		if err != nil {
+		parsedDst := strings.SplitN(dst, ":", 2)
-			logrus.Fatal(err)
+		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
+		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
+			logrus.Fatal(errorMsg)
+		} else if len(parsedSrc) != 2 {
+			if len(parsedDst) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		} else if len(parsedDst) != 2 {
+			if len(parsedSrc) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		}
+
+		var service string
+		var srcPath string
+		var dstPath string
+		isToContainer := false // <container:src> <dst>
+		if len(parsedSrc) == 2 {
+			service = parsedSrc[0]
+			srcPath = parsedSrc[1]
+			dstPath = dst
+			logrus.Debugf("assuming transfer is coming FROM the container")
+		} else if len(parsedDst) == 2 {
+			service = parsedDst[0]
+			dstPath = parsedDst[1]
+			srcPath = src
+			isToContainer = true // <src> <container:dst>
+			logrus.Debugf("assuming transfer is going TO the container")
+		}
+
+		if isToContainer {
+			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", srcPath)
+			}
 		}
 
 		cl, err := client.New(app.Server)
@@ -69,18 +96,7 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Fatal(err)
 		}
 
-		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
+		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
-
-		if toContainer {
-			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
-		} else {
-			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
-		}
-		if err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -88,292 +104,46 @@ And if you want to copy that file back to your current working directory locally
 	},
 }
 
-var errServiceMissing = errors.New("one of <src>/<dest> arguments must take $SERVICE:$PATH form")
+func configureAndCp(
+	c *cli.Context,
+	cl *dockerClient.Client,
+	app config.App,
+	srcPath string,
+	dstPath string,
+	service string,
+	isToContainer bool) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
 
-// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH
+	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
-func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) {
-	parsedSrc := strings.SplitN(src, ":", 2)
-	parsedDst := strings.SplitN(dst, ":", 2)
-	if len(parsedSrc)+len(parsedDst) != 3 {
-		return "", "", "", false, errServiceMissing
-	}
-	if len(parsedSrc) == 2 {
-		return parsedSrc[1], dst, parsedSrc[0], false, nil
-	}
-	if len(parsedDst) == 2 {
-		return src, parsedDst[1], parsedDst[0], true, nil
-	}
-	return "", "", "", false, errServiceMissing
-}
-
-// CopyToContainer copies a file or directory from the local file system to the container.
-// See the possible copy modes and their documentation.
-func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
-	srcStat, err := os.Stat(srcPath)
 	if err != nil {
-		return fmt.Errorf("local %s ", err)
+		logrus.Fatal(err)
 	}
 
-	dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath)
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
-	dstExists := true
-	if err != nil {
-		if errdefs.IsNotFound(err) {
-			dstExists = false
-		} else {
-			return fmt.Errorf("remote path: %s", err)
-		}
-	}
 
-	mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists)
+	if isToContainer {
-	if err != nil {
+		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		return err
-	}
-	movePath := ""
-	switch mode {
-	case CopyModeDirToDir:
-		// Add the src directory to the destination path
-		_, srcDir := path.Split(srcPath)
-		dstPath = path.Join(dstPath, srcDir)
-
-		// Make sure the dst directory exits.
-		dcli, err := command.NewDockerCli()
-		if err != nil {
-			return err
-		}
-		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          []string{"mkdir", "-p", dstPath},
-			Detach:       false,
-			Tty:          true,
-		}); err != nil {
-			return fmt.Errorf("create remote directory: %s", err)
-		}
-	case CopyModeFileToFile:
-		// Remove the file component from the path, since docker can only copy
-		// to a directory.
-		dstPath, _ = path.Split(dstPath)
-	case CopyModeFileToFileRename:
-		// Copy the file to the temp directory and move it to its dstPath
-		// afterwards.
-		movePath = dstPath
-		dstPath = "/tmp"
-	}
-
-	toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip}
 		content, err := archive.TarWithOptions(srcPath, toTarOpts)
 		if err != nil {
-		return err
+			logrus.Fatal(err)
 		}
 
-	logrus.Debugf("copy %s from local to %s on container", srcPath, dstPath)
 		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
+		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
-		return err
+			logrus.Fatal(err)
-	}
-
-	if movePath != "" {
-		_, srcFile := path.Split(srcPath)
-		dcli, err := command.NewDockerCli()
-		if err != nil {
-			return err
-		}
-		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          []string{"mv", path.Join("/tmp", srcFile), movePath},
-			Detach:       false,
-			Tty:          true,
-		}); err != nil {
-			return fmt.Errorf("create remote directory: %s", err)
-		}
-	}
-
-	return nil
-}
-
-// CopyFromContainer copies a file or directory from the given container to the local file system.
-// See the possible copy modes and their documentation.
-func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
-	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
-	if err != nil {
-		if errdefs.IsNotFound(err) {
-			return fmt.Errorf("remote: %s does not exist", srcPath)
-		} else {
-			return fmt.Errorf("remote path: %s", err)
-		}
-	}
-
-	dstStat, err := os.Stat(dstPath)
-	dstExists := true
-	var dstMode os.FileMode
-	if err != nil {
-		if os.IsNotExist(err) {
-			dstExists = false
-		} else {
-			return fmt.Errorf("remote path: %s", err)
 		}
 	} else {
-		dstMode = dstStat.Mode()
+		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
-	}
-
-	mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists)
 		if err != nil {
-		return err
+			logrus.Fatal(err)
-	}
-
-	moveDstDir := ""
-	moveDstFile := ""
-	switch mode {
-	case CopyModeFileToFile:
-		// Remove the file component from the path, since docker can only copy
-		// to a directory.
-		dstPath, _ = path.Split(dstPath)
-	case CopyModeFileToFileRename:
-		// Copy the file to the temp directory and move it to its dstPath
-		// afterwards.
-		moveDstFile = dstPath
-		dstPath = "/tmp"
-	case CopyModeFilesToDir:
-		// Copy the directory to the temp directory and move it to its
-		// dstPath afterwards.
-		moveDstDir = path.Join(dstPath, "/")
-		dstPath = "/tmp"
-
-		// Make sure the temp directory always gets removed
-		defer os.Remove(path.Join("/tmp"))
-	}
-
-	content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath)
-	if err != nil {
-		return fmt.Errorf("copy: %s", err)
 		}
 		defer content.Close()
-	if err := archive.Untar(content, dstPath, &archive.TarOptions{
+		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		NoOverwriteDirNonDir: true,
+		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-		Compression:          archive.Gzip,
+			logrus.Fatal(err)
-		NoLchown:             true,
+		}
-	}); err != nil {
-		return fmt.Errorf("untar: %s", err)
 	}
 
-	if moveDstFile != "" {
-		_, srcFile := path.Split(strings.TrimSuffix(srcPath, "/"))
-		if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil {
-			return err
-		}
-	}
-	if moveDstDir != "" {
-		_, srcDir := path.Split(strings.TrimSuffix(srcPath, "/"))
-		if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-var (
-	ErrCopyDirToFile  = fmt.Errorf("can't copy dir to file")
-	ErrDstDirNotExist = fmt.Errorf("destination directory does not exist")
-)
-
-type CopyMode int
-
-const (
-	// Copy a src file to a dest file. The src and dest file names are the same.
-	//  <dir_src>/<file> + <dir_dst>/<file> -> <dir_dst>/<file>
-	CopyModeFileToFile = CopyMode(iota)
-	// Copy a src file to a dest file. The src and dest file names are  not the same.
-	//  <dir_src>/<file_src> + <dir_dst>/<file_dst> -> <dir_dst>/<file_dst>
-	CopyModeFileToFileRename
-	// Copy a src file to dest directory. The dest file gets created in the dest
-	// folder with the src filename.
-	//  <dir_src>/<file> + <dir_dst> -> <dir_dst>/<file>
-	CopyModeFileToDir
-	// Copy a src directory to dest directory.
-	//  <dir_src> + <dir_dst> -> <dir_dst>/<dir_src>
-	CopyModeDirToDir
-	// Copy all files in the src directory to the dest directory. This works recursively.
-	//  <dir_src>/ + <dir_dst> -> <dir_dst>/<files_from_dir_src>
-	CopyModeFilesToDir
-)
-
-// copyMode takes a src and dest path and file mode to determine the copy mode.
-// See the possible copy modes and their documentation.
-func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) {
-	_, srcFile := path.Split(srcPath)
-	_, dstFile := path.Split(dstPath)
-	if srcMode.IsDir() {
-		if !dstExists {
-			return -1, ErrDstDirNotExist
-		}
-		if dstMode.IsDir() {
-			if strings.HasSuffix(srcPath, "/") {
-				return CopyModeFilesToDir, nil
-			}
-			return CopyModeDirToDir, nil
-		}
-		return -1, ErrCopyDirToFile
-	}
-
-	if dstMode.IsDir() {
-		return CopyModeFileToDir, nil
-	}
-
-	if srcFile != dstFile {
-		return CopyModeFileToFileRename, nil
-	}
-
-	return CopyModeFileToFile, nil
-}
-
-// moveDir moves all files from a source path to the destination path recursively.
-func moveDir(sourcePath, destPath string) error {
-	return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath))
-		if info.IsDir() {
-			err := os.Mkdir(newPath, info.Mode())
-			if err != nil {
-				if os.IsExist(err) {
-					return nil
-				}
-				return err
-			}
-		}
-		if info.Mode().IsRegular() {
-			return moveFile(p, newPath)
-		}
-		return nil
-	})
-}
-
-// moveFile moves a file from a source path to a destination path.
-func moveFile(sourcePath, destPath string) error {
-	inputFile, err := os.Open(sourcePath)
-	if err != nil {
-		return err
-	}
-	outputFile, err := os.Create(destPath)
-	if err != nil {
-		inputFile.Close()
-		return err
-	}
-	defer outputFile.Close()
-	_, err = io.Copy(outputFile, inputFile)
-	inputFile.Close()
-	if err != nil {
-		return err
-	}
-
-	// Remove file after succesfull copy.
-	err = os.Remove(sourcePath)
-	if err != nil {
-		return err
-	}
 	return nil
 }

cli/app/cp_test.go

@@ -1,113 +0,0 @@
-package app
-
-import (
-	"os"
-	"testing"
-)
-
-func TestParse(t *testing.T) {
-	tests := []struct {
-		src         string
-		dst         string
-		srcPath     string
-		dstPath     string
-		service     string
-		toContainer bool
-		err         error
-	}{
-		{src: "foo", dst: "bar", err: errServiceMissing},
-		{src: "app:foo", dst: "app:bar", err: errServiceMissing},
-		{src: "app:foo", dst: "bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: false},
-		{src: "foo", dst: "app:bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: true},
-	}
-
-	for i, tc := range tests {
-		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(tc.src, tc.dst)
-		if srcPath != tc.srcPath {
-			t.Errorf("[%d] srcPath: want (%s), got(%s)", i, tc.srcPath, srcPath)
-		}
-		if dstPath != tc.dstPath {
-			t.Errorf("[%d] dstPath: want (%s), got(%s)", i, tc.dstPath, dstPath)
-		}
-		if service != tc.service {
-			t.Errorf("[%d] service: want (%s), got(%s)", i, tc.service, service)
-		}
-		if toContainer != tc.toContainer {
-			t.Errorf("[%d] toConainer: want (%t), got(%t)", i, tc.toContainer, toContainer)
-		}
-		if err == nil && tc.err != nil && err.Error() != tc.err.Error() {
-			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
-		}
-	}
-}
-
-func TestCopyMode(t *testing.T) {
-	tests := []struct {
-		srcPath   string
-		dstPath   string
-		srcMode   os.FileMode
-		dstMode   os.FileMode
-		dstExists bool
-		mode      CopyMode
-		err       error
-	}{
-		{
-			srcPath:   "foo.txt",
-			dstPath:   "foo.txt",
-			srcMode:   os.ModePerm,
-			dstMode:   os.ModePerm,
-			dstExists: true,
-			mode:      CopyModeFileToFile,
-		},
-		{
-			srcPath:   "foo.txt",
-			dstPath:   "bar.txt",
-			srcMode:   os.ModePerm,
-			dstExists: true,
-			mode:      CopyModeFileToFileRename,
-		},
-		{
-			srcPath:   "foo",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstMode:   os.ModeDir,
-			dstExists: true,
-			mode:      CopyModeDirToDir,
-		},
-		{
-			srcPath:   "foo/",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstMode:   os.ModeDir,
-			dstExists: true,
-			mode:      CopyModeFilesToDir,
-		},
-		{
-			srcPath:   "foo",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstExists: false,
-			mode:      -1,
-			err:       ErrDstDirNotExist,
-		},
-		{
-			srcPath:   "foo",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstMode:   os.ModePerm,
-			dstExists: true,
-			mode:      -1,
-			err:       ErrCopyDirToFile,
-		},
-	}
-
-	for i, tc := range tests {
-		mode, err := copyMode(tc.srcPath, tc.dstPath, tc.srcMode, tc.dstMode, tc.dstExists)
-		if mode != tc.mode {
-			t.Errorf("[%d] mode: want (%d), got(%d)", i, tc.mode, mode)
-		}
-		if err != tc.err {
-			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
-		}
-	}
-}

cli/app/logs.go

@@ -2,26 +2,75 @@ package app
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
-	"slices"
 	"sync"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+var logOpts = types.ContainerLogsOptions{
+	ShowStderr: true,
+	ShowStdout: true,
+	Since:      "",
+	Until:      "",
+	Timestamps: true,
+	Follow:     true,
+	Tail:       "20",
+	Details:    false,
+}
+
+// stackLogs lists logs for all stack services
+func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
+	filters, err := app.Filters(true, false)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(context.Background(), serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			if internal.StdErrOnly {
+				logOpts.ShowStdout = false
+			}
+
+			logs, err := client.ServiceLogs(context.Background(), s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+
+	wg.Wait()
+
+	os.Exit(0)
+}
+
 var appLogsCommand = cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
@@ -56,70 +105,37 @@ var appLogsCommand = cli.Command{
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
+		logOpts.Since = internal.SinceLogs
+
 		serviceName := c.Args().Get(1)
-		serviceNames := []string{}
+		if serviceName == "" {
-		if serviceName != "" {
+			logrus.Debugf("tailing logs for all %s services", app.Recipe)
-			serviceNames = []string{serviceName}
+			stackLogs(c, app, cl)
-		}
+		} else {
-		err = tailLogs(cl, app, serviceNames)
+			logrus.Debugf("tailing logs for %s", serviceName)
-		if err != nil {
+			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
 				logrus.Fatal(err)
 			}
+		}
 
 		return nil
 	},
 }
 
-// tailLogs prints logs for the given app with optional service names to be
+func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
-// filtered on. It also checks if the latest task is not runnning and then
-// prints the past tasks.
-func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) error {
-	f, err := app.Filters(true, false, serviceNames...)
-	if err != nil {
-		return err
-	}
-
-	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: f})
-	if err != nil {
-		return err
-	}
-
-	var wg sync.WaitGroup
-	for _, service := range services {
 	filters := filters.NewArgs()
-		filters.Add("name", service.Spec.Name)
+	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
-		tasks, err := cl.TaskList(context.Background(), types.TaskListOptions{Filters: f})
+
+	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
 	if err != nil {
-			return err
+		logrus.Fatal(err)
-		}
-		if len(tasks) > 0 {
-			// Need to sort the tasks by the CreatedAt field in the inverse order.
-			// Otherwise they are in the reversed order and not sorted properly.
-			slices.SortFunc[[]swarm.Task](tasks, func(t1, t2 swarm.Task) int {
-				return int(t2.Meta.CreatedAt.Unix() - t1.Meta.CreatedAt.Unix())
-			})
-			lastTask := tasks[0].Status
-			if lastTask.State != swarm.TaskStateRunning {
-				for _, task := range tasks {
-					logrus.Errorf("[%s] %s State %s: %s", service.Spec.Name, task.Meta.CreatedAt.Format(time.RFC3339), task.Status.State, task.Status.Err)
-				}
-			}
 	}
 
-		// Collect the logs in a go routine, so the logs from all services are
+	if internal.StdErrOnly {
-		// collected in parallel.
+		logOpts.ShowStdout = false
-		wg.Add(1)
+	}
-		go func(serviceID string) {
+
-			logs, err := cl.ServiceLogs(context.Background(), serviceID, types.ContainerLogsOptions{
+	logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts)
-				ShowStderr: true,
-				ShowStdout: !internal.StdErrOnly,
-				Since:      internal.SinceLogs,
-				Until:      "",
-				Timestamps: true,
-				Follow:     true,
-				Tail:       "20",
-				Details:    false,
-			})
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -129,11 +145,6 @@ func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) er
 	if err != nil && err != io.EOF {
 		logrus.Fatal(err)
 	}
-		}(service.ID)
-	}
-
-	// Wait for all log streams to be closed.
-	wg.Wait()
 
 	return nil
 }

cli/app/new.go

@@ -55,16 +55,8 @@ var appNewCommand = cli.Command{
 		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
-	ArgsUsage: "[<recipe>] [<version>]",
+	ArgsUsage:    "[<recipe>]",
-	BashComplete: func(ctx *cli.Context) {
+	BashComplete: autocomplete.RecipeNameComplete,
-		args := ctx.Args()
-		switch len(args) {
-		case 0:
-			autocomplete.RecipeNameComplete(ctx)
-		case 1:
-			autocomplete.RecipeVersionComplete(ctx.Args().Get(0))
-		}
-	},
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
@@ -77,15 +69,9 @@ var appNewCommand = cli.Command{
 					logrus.Fatal(err)
 				}
 			}
-			if c.Args().Get(1) == "" {
 			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
 				logrus.Fatal(err)
 			}
-			} else {
-				if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil {
-					logrus.Fatal(err)
-				}
-			}
 		}
 
 		if err := ensureServerFlag(); err != nil {
@@ -111,7 +97,7 @@ var appNewCommand = cli.Command{
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
-			sampleEnv, err := recipe.SampleEnv()
+			sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -122,7 +108,7 @@ var appNewCommand = cli.Command{
 			}
 
 			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, config.StackName(internal.Domain))
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 			if err != nil {
 				return err
 			}
@@ -182,8 +168,14 @@ var appNewCommand = cli.Command{
 type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) {
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) {
-	secrets, err := secret.GenerateSecrets(cl, secretsConfig, internal.NewAppServer)
+	// NOTE(d1): trim to match app.StackName() implementation
+	if len(sanitisedAppName) > 45 {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
+		sanitisedAppName = sanitisedAppName[:45]
+	}
+
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
 	if err != nil {
 		return nil, err
 	}
@@ -225,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 }
 
 // promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(recipeName string, secretsConfig map[string]secret.Secret) error {
+func promptForSecrets(recipeName string, secretsConfig map[string]string) error {
 	if len(secretsConfig) == 0 {
 		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
 		return nil

cli/app/remove.go

@@ -3,7 +3,6 @@ package app
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -12,6 +11,7 @@ import (
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -110,19 +110,26 @@ flag.
 			logrus.Fatal(err)
 		}
 
-		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, fs)
+		volumeListOptions := volume.ListOptions{fs}
+		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
+		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		volumeNames := client.GetVolumeNames(volumeList)
 
-		if len(volumeNames) > 0 {
+		var vols []string
-			err := client.RemoveVolumes(cl, context.Background(), volumeNames, internal.Force, 5)
+		for _, vol := range volumeList {
-			if err != nil {
+			vols = append(vols, vol.Name)
-				log.Fatalf("removing volumes failed: %s", err)
 		}
 
-			logrus.Infof("%d volumes removed successfully", len(volumeNames))
+		if len(vols) > 0 {
+			for _, vol := range vols {
+				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("volume %s removed", vol))
+			}
 		} else {
 			logrus.Info("no volumes to remove")
 		}

cli/app/restore.go

@@ -1,82 +1,223 @@
 package app
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var targetPath string
+type restoreConfig struct {
-var targetPathFlag = &cli.StringFlag{
+	preHookCmd  string
-	Name:        "target, t",
+	postHookCmd string
-	Usage:       "Target path",
-	Destination: &targetPath,
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Restore an app backup",
+	Usage:     "Run app restore",
-	ArgsUsage: "<domain> <service>",
+	ArgsUsage: "<domain> <service> <file>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		targetPathFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app restore.
+
+Pre/post hook commands are defined in the recipe configuration. Abra reads this
+configuration and run the comands in the context of the service before
+restoring the backup.
+
+Unlike "abra app backup", restore must be run on a per-service basis. You can
+not restore all services in one go. Backup files produced by Abra are
+compressed archives which use absolute paths. This allows Abra to restore
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
+
+Example:
+
+    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
+`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
+		}
+
+		backupPath := c.Args().Get(2)
+		if backupPath == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
+		}
+
+		if _, err := os.Stat(backupPath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s doesn't exist?", backupPath)
+			}
+		}
+
+		restoreConfigs := make(map[string]restoreConfig)
+		for _, service := range recipe.Config.Services {
+			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
+				if restoreEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					rsConfig := restoreConfig{}
+
+					logrus.Debugf("restore config detected for %s", fullServiceName)
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						rsConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						rsConfig.postHookCmd = postHookCmd
+					}
+
+					restoreConfigs[service.Name] = rsConfig
+				}
+			}
+		}
+
+		rsConfig, ok := restoreConfigs[serviceName]
+		if !ok {
+			rsConfig = restoreConfig{}
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if targetPath != "" {
-			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
-			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
+
+// runRestore does the actual restore logic.
+func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if rsConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
+	}
+
+	backupReader, err := os.Open(backupPath)
+	if err != nil {
+		return err
+	}
+
+	content, err := archive.DecompressStream(backupReader)
+	if err != nil {
+		return err
+	}
+
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
+	restorePath := "/"
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
+		return err
+	}
+
+	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
+
+	if rsConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
+	}
+
+	return nil
+}

cli/app/run.go

@@ -91,7 +91,7 @@ var appRunCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/secret.go

@@ -20,23 +20,19 @@ import (
 	"github.com/urfave/cli"
 )
 
-var (
+var allSecrets bool
-	allSecrets     bool
+var allSecretsFlag = &cli.BoolFlag{
-	allSecretsFlag = &cli.BoolFlag{
 	Name:        "all, a",
 	Destination: &allSecrets,
 	Usage:       "Generate all secrets",
-	}
+}
-)
 
-var (
+var rmAllSecrets bool
-	rmAllSecrets     bool
+var rmAllSecretsFlag = &cli.BoolFlag{
-	rmAllSecretsFlag = &cli.BoolFlag{
 	Name:        "all, a",
 	Destination: &rmAllSecrets,
 	Usage:       "Remove all secrets",
-	}
+}
-)
 
 var appSecretGenerateCommand = cli.Command{
 	Name:      "generate",
@@ -91,21 +87,27 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if !allSecrets {
+		secretsToCreate := make(map[string]string)
+		if allSecrets {
+			secretsToCreate = secretsConfig
+		} else {
 			secretName := c.Args().Get(1)
 			secretVersion := c.Args().Get(2)
-			s, ok := secrets[secretName]
+			matches := false
-			if !ok {
+			for name := range secretsConfig {
-				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
+				if secretName == name {
+					secretsToCreate[name] = secretVersion
+					matches = true
 				}
-			s.Version = secretVersion
+			}
-			secrets = map[string]secret.Secret{
+
-				secretName: s,
+			if !matches {
+				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
 		}
 
@@ -114,7 +116,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secrets, app.Server)
+		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -274,7 +276,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -309,7 +311,12 @@ Example:
 
 		match := false
 		secretToRm := c.Args().Get(1)
-		for secretName, val := range secrets {
+		for secretName, secretValue := range secretsConfig {
+			val, err := secret.ParseSecretValue(secretValue)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
 			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 			if _, ok := remoteSecretNames[secretRemoteName]; ok {
 				if secretToRm != "" {

cli/app/volume.go

@@ -2,7 +2,6 @@ package app
 
 import (
 	"context"
-	"log"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -132,12 +131,12 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		}
 
 		if len(volumesToRemove) > 0 {
-			err := client.RemoveVolumes(cl, context.Background(), volumesToRemove, internal.Force, 5)
+			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
 			if err != nil {
-				log.Fatalf("removing volumes failed: %s", err)
+				logrus.Fatal(err)
 			}
 
-			logrus.Infof("%d volumes removed successfully", len(volumesToRemove))
+			logrus.Info("volumes removed successfully")
 		} else {
 			logrus.Info("no volumes removed")
 		}

cli/internal/backup.go

@@ -1,67 +1,35 @@
 package internal
 
 import (
-	"context"
+	"strings"
-
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/service"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
 )
 
-// RetrieveBackupBotContainer gets the deployed backupbot container.
+// SafeSplit splits up a string into a list of commands safely.
-func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) {
+func SafeSplit(s string) []string {
-	ctx := context.Background()
+	split := strings.Split(s, " ")
-	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
+
-	if err != nil {
+	var result []string
-		return types.Container{}, err
+	var inquote string
+	var block string
+	for _, i := range split {
+		if inquote == "" {
+			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
+				inquote = string(i[0])
+				block = strings.TrimPrefix(i, inquote) + " "
+			} else {
+				result = append(result, i)
+			}
+		} else {
+			if !strings.HasSuffix(i, inquote) {
+				block += i + " "
+			} else {
+				block += strings.TrimSuffix(i, inquote)
+				inquote = ""
+				result = append(result, block)
+				block = ""
+			}
+		}
 	}
 
-	logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+	return result
-
-	filters := filters.NewArgs()
-	filters.Add("name", chosenService.Spec.Name)
-	targetContainer, err := containerPkg.GetContainer(
-		ctx,
-		cl,
-		filters,
-		NoInput,
-	)
-	if err != nil {
-		return types.Container{}, err
-	}
-
-	return targetContainer, nil
-}
-
-// RunBackupCmdRemote runs a backup related command on a remote backupbot container.
-func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error {
-	execBackupListOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          []string{"/usr/bin/backup", "--", backupCmd},
-		Detach:       false,
-		Env:          execEnv,
-		Tty:          true,
-	}
-
-	logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts)
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil {
-		return err
-	}
-
-	return nil
 }

cli/internal/command.go

@@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		Tty:          false,
 	}
 
-	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
 		shell = "/bin/sh"
 	}
@@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		execCreateOpts.Tty = false
 	}
 
-	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		return err
 	}
 

cli/recipe/fetch.go

@@ -3,7 +3,6 @@ package recipe
 import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -18,31 +17,26 @@ var recipeFetchCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
+
 		if recipeName != "" {
 			internal.ValidateRecipe(c)
-			if err := recipe.Ensure(recipeName); err != nil {
-				logrus.Fatal(err)
-			}
-			return nil
 		}
 
-		catalogue, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err := recipe.EnsureExists(recipeName); err != nil {
-		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		catlBar := formatter.CreateProgressbar(len(catalogue), "fetching latest recipes...")
+		if err := recipe.EnsureUpToDate(recipeName); err != nil {
-		for recipeName := range catalogue {
+			logrus.Fatal(err)
-			if err := recipe.Ensure(recipeName); err != nil {
-				logrus.Error(err)
 		}
-			catlBar.Add(1)
+
+		if err := recipe.EnsureLatest(recipeName); err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil

cli/recipe/release.go

@@ -1,9 +1,7 @@
 package recipe
 
 import (
-	"errors"
 	"fmt"
-	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -142,7 +140,7 @@ your SSH keys configured on your account.
 
 // getImageVersions retrieves image versions for a recipe
 func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
-	services := make(map[string]string)
+	var services = make(map[string]string)
 
 	missingTag := false
 	for _, service := range recipe.Config.Services {
@@ -209,10 +207,6 @@ func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string
 		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
 	}
 
-	if err := addReleaseNotes(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
 	if err := commitRelease(recipe, tagString); err != nil {
 		logrus.Fatal(err)
 	}
@@ -243,82 +237,6 @@ func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
 	return git.CreateTagOptions{Message: msg}, nil
 }
 
-// addReleaseNotes checks if the release/next release note exists and moves the
-// file to release/<tag>.
-func addReleaseNotes(recipe recipe.Recipe, tag string) error {
-	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	tagReleaseNotePath := path.Join(repoPath, "release", tag)
-	if _, err := os.Stat(tagReleaseNotePath); err == nil {
-		// Release note for current tag already exist exists.
-		return nil
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-
-	nextReleaseNotePath := path.Join(repoPath, "release", "next")
-	if _, err := os.Stat(nextReleaseNotePath); err == nil {
-		// release/next note exists. Move it to release/<tag>
-		if internal.Dry {
-			logrus.Debugf("dry run: move release note from 'next' to %s", tag)
-			return nil
-		}
-		if !internal.NoInput {
-			prompt := &survey.Input{
-				Message: "Use release note in release/next?",
-			}
-			var addReleaseNote bool
-			if err := survey.AskOne(prompt, &addReleaseNote); err != nil {
-				return err
-			}
-			if !addReleaseNote {
-				return nil
-			}
-		}
-		err := os.Rename(nextReleaseNotePath, tagReleaseNotePath)
-		if err != nil {
-			return err
-		}
-		err = gitPkg.Add(repoPath, path.Join("release", "next"), internal.Dry)
-		if err != nil {
-			return err
-		}
-		err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry)
-		if err != nil {
-			return err
-		}
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-
-	// No release note exists for the current release.
-	if internal.NoInput {
-		return nil
-	}
-
-	prompt := &survey.Input{
-		Message: "Release Note (leave empty for no release note)",
-	}
-	var releaseNote string
-	if err := survey.AskOne(prompt, &releaseNote); err != nil {
-		return err
-	}
-
-	if releaseNote == "" {
-		return nil
-	}
-
-	err := os.WriteFile(tagReleaseNotePath, []byte(releaseNote), 0o644)
-	if err != nil {
-		return err
-	}
-	err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func commitRelease(recipe recipe.Recipe, tag string) error {
 	if internal.Dry {
 		logrus.Debugf("dry run: no changes committed")
@@ -486,10 +404,6 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip
 		}
 	}
 
-	if err := addReleaseNotes(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
 	if err := commitRelease(recipe, tagString); err != nil {
 		logrus.Fatalf("failed to commit changes: %s", err.Error())
 	}

go.mod

@@ -4,20 +4,19 @@ go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
-	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355
 	github.com/AlecAivazis/survey/v2 v2.3.7
+	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/docker/cli v24.0.7+incompatible
 	github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/go-git/go-git/v5 v5.10.0
-	github.com/google/go-cmp v0.5.9
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.14.1
+	github.com/schollz/progressbar/v3 v3.14.0
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )
@@ -48,6 +47,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -56,7 +56,7 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
@@ -71,7 +71,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -82,7 +82,7 @@ require (
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.13.0 // indirect

go.sum

@@ -51,12 +51,12 @@ coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355 h1:tCv2B4qoN6RMheKDnCzIafOkWS5BB1h7hwhmo+9bVeE=
-git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
 github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
+github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg=
+github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -590,8 +590,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
-github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -705,8 +705,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -885,9 +885,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -901,8 +900,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI=
+github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
-github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E=
+github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
@@ -1311,20 +1310,21 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/autocomplete/autocomplete.go

@@ -25,16 +25,6 @@ func AppNameComplete(c *cli.Context) {
 	}
 }
 
-func ServiceNameComplete(appName string) {
-	serviceNames, err := config.GetAppServiceNames(appName)
-	if err != nil {
-		return
-	}
-	for _, s := range serviceNames {
-		fmt.Println(s)
-	}
-}
-
 // RecipeNameComplete completes recipe names.
 func RecipeNameComplete(c *cli.Context) {
 	catl, err := recipe.ReadRecipeCatalogue(false)
@@ -51,20 +41,6 @@ func RecipeNameComplete(c *cli.Context) {
 	}
 }
 
-// RecipeVersionComplete completes versions for the recipe.
-func RecipeVersionComplete(recipeName string) {
-	catl, err := recipe.ReadRecipeCatalogue(false)
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	for _, v := range catl[recipeName].Versions {
-		for v2 := range v {
-			fmt.Println(v2)
-		}
-	}
-}
-
 // ServerNameComplete completes server names.
 func ServerNameComplete(c *cli.Context) {
 	files, err := config.LoadAppFiles("")

pkg/client/volumes.go

@@ -2,17 +2,15 @@ package client
 
 import (
 	"context"
-	"fmt"
-	"time"
 
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
 )
 
 func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
-	volumeListOKBody, err := cl.VolumeList(ctx, volume.ListOptions{Filters: fs})
+	volumeListOptions := volume.ListOptions{fs}
+	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
 		return volumeList, err
@@ -31,32 +29,13 @@ func GetVolumeNames(volumes []*volume.Volume) []string {
 	return volumeNames
 }
 
-func RemoveVolumes(cl *client.Client, ctx context.Context, volumeNames []string, force bool, retries int) error {
+func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
 	for _, volName := range volumeNames {
-		err := retryFunc(5, func() error {
+		err := cl.VolumeRemove(ctx, volName, force)
-			return cl.VolumeRemove(context.Background(), volName, force)
-		})
 		if err != nil {
-			return fmt.Errorf("volume %s: %s", volName, err)
+			return err
 		}
 	}
-	return nil
-}
 
-// retryFunc retries the given function for the given retries. After the nth
-// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
-// It returns an error if the function still failed after the last retry.
-func retryFunc(retries int, fn func() error) error {
-	for i := 0; i < retries; i++ {
-		err := fn()
-		if err == nil {
 	return nil
-		}
-		if i+1 < retries {
-			sleep := time.Duration(i+1) * time.Duration(i+1)
-			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
-			time.Sleep(sleep * time.Second)
-		}
-	}
-	return fmt.Errorf("%d retries failed", retries)
 }

pkg/client/volumes_test.go

@@ -1,26 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestRetryFunc(t *testing.T) {
-	err := retryFunc(1, func() error { return nil })
-	if err != nil {
-		t.Errorf("should not return an error: %s", err)
-	}
-
-	i := 0
-	fn := func() error {
-		i++
-		return fmt.Errorf("oh no, something went wrong!")
-	}
-	err = retryFunc(2, fn)
-	if err == nil {
-		t.Error("should return an error")
-	}
-	if i != 2 {
-		t.Errorf("The function should have been called 1 times, got %d", i)
-	}
-}

pkg/compose/compose.go

@@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return err
 		}

pkg/config/app.go

@@ -25,9 +25,6 @@ import (
 // AppEnv is a map of the values in an apps env config
 type AppEnv = map[string]string
 
-// AppModifiers is a map of modifiers in an apps env config
-type AppModifiers = map[string]map[string]string
-
 // AppName is AppName
 type AppName = string
 
@@ -50,61 +47,34 @@ type App struct {
 	Path   string
 }
 
-// See documentation of config.StackName
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
 	}
 
-	stackName := StackName(a.Name)
+	stackName := SanitiseAppName(a.Name)
-
-	a.Env["STACK_NAME"] = stackName
-
-	return stackName
-}
-
-// StackName gets whatever the docker safe (uses the right delimiting
-// character, e.g. "_") stack name is for the app. In general, you don't want
-// to use this to show anything to end-users, you want use a.Name instead.
-func StackName(appName string) string {
-	stackName := SanitiseAppName(appName)
 
 	if len(stackName) > 45 {
 		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:45])
 		stackName = stackName[:45]
 	}
 
+	a.Env["STACK_NAME"] = stackName
+
 	return stackName
 }
 
-// Filters retrieves app filters for querying the container runtime. By default
+// Filters retrieves exact app filters for querying the container runtime. Due
-// it filters on all services in the app. It is also possible to pass an
+// to upstream issues, filtering works different depending on what you're
-// otional list of service names, which get filtered instead.
-//
-// Due to upstream issues, filtering works different depending on what you're
 // querying. So, for example, secrets don't work with regex! The caller needs
 // to implement their own validation that the right secrets are matched. In
 // order to handle these cases, we provide the `appendServiceNames` /
 // `exactMatch` modifiers.
-func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) {
+func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) {
 	filters := filters.NewArgs()
-	if len(services) > 0 {
-		for _, serviceName := range services {
-			filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch))
-		}
-		return filters, nil
-	}
-
-	// When not appending the service name, just add one filter for the whole
-	// stack.
-	if !appendServiceNames {
-		f := fmt.Sprintf("%s", a.StackName())
-		if exactMatch {
-			f = fmt.Sprintf("^%s", f)
-		}
-		filters.Add("name", f)
-		return filters, nil
-	}
 
 	composeFiles, err := GetComposeFiles(a.Recipe, a.Env)
 	if err != nil {
@@ -118,23 +88,28 @@ func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (f
 	}
 
 	for _, service := range compose.Services {
-		f := ServiceFilter(a.StackName(), service.Name, exactMatch)
+		var filter string
-		filters.Add("name", f)
+
+		if appendServiceNames {
+			if exactMatch {
+				filter = fmt.Sprintf("^%s_%s", a.StackName(), service.Name)
+			} else {
+				filter = fmt.Sprintf("%s_%s", a.StackName(), service.Name)
+			}
+		} else {
+			if exactMatch {
+				filter = fmt.Sprintf("^%s", a.StackName())
+			} else {
+				filter = fmt.Sprintf("%s", a.StackName())
+			}
+		}
+
+		filters.Add("name", filter)
 	}
 
 	return filters, nil
 }
 
-// ServiceFilter creates a filter string for filtering a service in the docker
-// container runtime. When exact match is true, it uses regex to match the
-// string exactly.
-func ServiceFilter(stack, service string, exact bool) string {
-	if exact {
-		return fmt.Sprintf("^%s_%s", stack, service)
-	}
-	return fmt.Sprintf("%s_%s", stack, service)
-}
-
 // ByServer sort a slice of Apps
 type ByServer []App
 
@@ -175,7 +150,7 @@ func (a ByName) Less(i, j int) bool {
 }
 
 func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path)
+	env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
@@ -355,7 +330,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
-	err = ioutil.WriteFile(appEnvPath, envSample, 0o664)
+	err = ioutil.WriteFile(appEnvPath, envSample, 0664)
 	if err != nil {
 		return err
 	}
@@ -617,7 +592,7 @@ func GetLabel(compose *composetypes.Config, stackName string, label string) stri
 
 // GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value
 func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
-	timeout := 50 // Default Timeout
+	var timeout = 50 // Default Timeout
 	var err error = nil
 	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
 		logrus.Debugf("timeout label: %s", timeoutLabel)

pkg/config/app_test.go

@@ -1,15 +1,12 @@
 package config_test
 
 import (
-	"encoding/json"
 	"fmt"
 	"reflect"
 	"testing"
 
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -109,89 +106,3 @@ func TestGetComposeFilesError(t *testing.T) {
 		}
 	}
 }
-
-func TestFilters(t *testing.T) {
-	oldDir := config.RECIPES_DIR
-	config.RECIPES_DIR = "./testdir"
-	defer func() {
-		config.RECIPES_DIR = oldDir
-	}()
-
-	app, err := config.NewApp(config.AppEnv{
-		"DOMAIN": "test.example.com",
-		"RECIPE": "test-recipe",
-	}, "test_example_com", config.AppFile{
-		Path:   "./testdir/filtertest.end",
-		Server: "local",
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	f, err := app.Filters(false, false)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f, map[string]map[string]bool{
-		"name": {
-			"test_example_com": true,
-		},
-	})
-
-	f2, err := app.Filters(false, true)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f2, map[string]map[string]bool{
-		"name": {
-			"^test_example_com": true,
-		},
-	})
-
-	f3, err := app.Filters(true, false)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f3, map[string]map[string]bool{
-		"name": {
-			"test_example_com_bar": true,
-			"test_example_com_foo": true,
-		},
-	})
-
-	f4, err := app.Filters(true, true)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f4, map[string]map[string]bool{
-		"name": {
-			"^test_example_com_bar": true,
-			"^test_example_com_foo": true,
-		},
-	})
-
-	f5, err := app.Filters(false, false, "foo")
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f5, map[string]map[string]bool{
-		"name": {
-			"test_example_com_foo": true,
-		},
-	})
-}
-
-func compareFilter(t *testing.T, f1 filters.Args, f2 map[string]map[string]bool) {
-	t.Helper()
-	j1, err := f1.MarshalJSON()
-	if err != nil {
-		t.Error(err)
-	}
-	j2, err := json.Marshal(f2)
-	if err != nil {
-		t.Error(err)
-	}
-	if diff := cmp.Diff(string(j2), string(j1)); diff != "" {
-		t.Errorf("filters mismatch (-want +got):\n%s", diff)
-	}
-}

pkg/config/env.go

@@ -12,7 +12,7 @@ import (
 	"sort"
 	"strings"
 
-	"git.coopcloud.tech/coop-cloud/godotenv"
+	"github.com/Autonomic-Cooperative/godotenv"
 	"github.com/sirupsen/logrus"
 )
 
@@ -36,8 +36,6 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
-var BackupbotLabel = "coop-cloud.backupbot.enabled"
-
 // envVarModifiers is a list of env var modifier strings. These are added to
 // env vars as comments and modify their processing by Abra, e.g. determining
 // how long secrets should be.
@@ -57,34 +55,45 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
+// ReadEnvOptions modifies the ReadEnv processing of env vars.
+type ReadEnvOptions struct {
+	IncludeModifiers bool
+}
+
+// ContainsEnvVarModifier determines if an env var contains a modifier.
+func ContainsEnvVarModifier(envVar string) bool {
+	for _, mod := range envVarModifiers {
+		if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
+			return true
+		}
+	}
+	return false
+}
+
 // ReadEnv loads an app envivornment into a map.
-func ReadEnv(filePath string) (AppEnv, error) {
+func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
 	var envVars AppEnv
 
-	envVars, _, err := godotenv.Read(filePath)
+	envVars, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
+	//	for idx, envVar := range envVars {
+	//		if strings.Contains(envVar, "#") {
+	//			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
+	//				continue
+	//			}
+	//			vals := strings.Split(envVar, "#")
+	//			envVars[idx] = strings.TrimSpace(vals[0])
+	//		}
+	//	}
+
 	logrus.Debugf("read %s from %s", envVars, filePath)
 
 	return envVars, nil
 }
 
-// ReadEnv loads an app envivornment and their modifiers in two different maps.
-func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) {
-	var envVars AppEnv
-
-	envVars, mods, err := godotenv.Read(filePath)
-	if err != nil {
-		return nil, mods, err
-	}
-
-	logrus.Debugf("read %s from %s", envVars, filePath)
-
-	return envVars, mods, nil
-}
-
 // ReadServerNames retrieves all server names.
 func ReadServerNames() ([]string, error) {
 	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
@@ -218,7 +227,7 @@ func CheckEnv(app App) ([]EnvVar, error) {
 		return envVars, err
 	}
 
-	envSample, err := ReadEnv(envSamplePath)
+	envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
 	if err != nil {
 		return envVars, err
 	}

pkg/config/env_test.go

@@ -13,21 +13,15 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var (
+var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-	TestFolder    = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
-	ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
-)
 
 // make sure these are in alphabetical order
-var (
+var TFolders = []string{"folder1", "folder2"}
-	TFolders = []string{"folder1", "folder2"}
+var TFiles = []string{"bar.env", "foo.env"}
-	TFiles   = []string{"bar.env", "foo.env"}
-)
 
-var (
+var AppName = "ecloud"
-	AppName    = "ecloud"
+var ServerName = "evil.corp"
-	ServerName = "evil.corp"
-)
 
 var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
@@ -77,7 +71,7 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := config.ReadEnv(ExpectedAppFile.Path)
+	env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +149,7 @@ func TestCheckEnv(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -189,7 +183,7 @@ func TestCheckEnvError(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -217,6 +211,16 @@ func TestCheckEnvError(t *testing.T) {
 	}
 }
 
+func TestContainsEnvVarModifier(t *testing.T) {
+	if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
+		t.Fatal("FOO contains no env var modifier")
+	}
+
+	if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
+		t.Fatal("FOO contains an env var modifier (length)")
+	}
+}
+
 func TestEnvVarCommentsRemoved(t *testing.T) {
 	offline := true
 	r, err := recipe.Get("abra-test-recipe", offline)
@@ -225,7 +229,7 @@ func TestEnvVarCommentsRemoved(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -257,19 +261,12 @@ func TestEnvVarModifiersIncluded(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "v1") {
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
-		t.Errorf("value should be 'v1', got: '%s'", envSample["SECRET_TEST_PASS_TWO_VERSION"])
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
-	}
-	if modifiers == nil || modifiers["SECRET_TEST_PASS_TWO_VERSION"] == nil {
-		t.Errorf("no modifiers included")
-	} else {
-		if modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"] != "10" {
-			t.Errorf("length modifier should be '10', got: '%s'", modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"])
-		}
 	}
 }

pkg/config/testdir/filtertest.env

@@ -1,2 +0,0 @@
-RECIPE=test-recipe
-DOMAIN=test.example.com

pkg/config/testdir/test-recipe/compose.yml

@@ -1,6 +0,0 @@
-version: "3.8"
-services:
-  foo:
-    image: debian
-  bar:
-    image: debian

pkg/container/container.go

@@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) > 1 {
+	if len(containers) != 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")
@@ -68,15 +68,3 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 
 	return containers[0], nil
 }
-
-// GetContainerFromStackAndService retrieves the container for the given stack and service.
-func GetContainerFromStackAndService(cl *client.Client, stack, service string) (types.Container, error) {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", stack, service))
-
-	container, err := GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return types.Container{}, err
-	}
-	return container, nil
-}

pkg/git/add.go

@@ -1,27 +0,0 @@
-package git
-
-import (
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-)
-
-// Add adds a file to the git index.
-func Add(repoPath, path string, dryRun bool) error {
-	repo, err := git.PlainOpen(repoPath)
-	if err != nil {
-		return err
-	}
-
-	worktree, err := repo.Worktree()
-	if err != nil {
-		return err
-	}
-
-	if dryRun {
-		logrus.Debugf("dry run: adding %s", path)
-	} else {
-		worktree.Add(path)
-	}
-
-	return nil
-}

pkg/lint/recipe.go

@@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}

pkg/recipe/recipe.go

@@ -227,7 +227,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -255,29 +255,15 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}, nil
 }
 
-func (r Recipe) SampleEnv() (map[string]string, error) {
+func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, opts)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}
 	return sampleEnv, nil
 }
 
-// Ensure makes sure the recipe exists, is up to date and has the latest version checked out.
-func Ensure(recipeName string) error {
-	if err := EnsureExists(recipeName); err != nil {
-		return err
-	}
-	if err := EnsureUpToDate(recipeName); err != nil {
-		return err
-	}
-	if err := EnsureLatest(recipeName); err != nil {
-		return err
-	}
-	return nil
-}
-
 // EnsureExists ensures that a recipe is locally cloned
 func EnsureExists(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)

pkg/secret/secret.go

@@ -21,24 +21,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// Secret represents a secret.
+// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
-type Secret struct {
+// secret definition.
-	// Version comes from the secret version environment variable.
+type secretValue struct {
-	// For example:
-	//  SECRET_FOO=v1
 	Version string
-	// Length comes from the length modifier at the secret version environment
-	// variable. For Example:
-	//   SECRET_FOO=v1 # length=12
 	Length  int
-	// RemoteName is the name of the secret on the server. For example:
-	//   name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
-	// With the following:
-	//   STACK_NAME=test_example_com
-	//   SECRET_TEST_PASS_TWO_VERSION=v2
-	// Will have this remote name:
-	//   test_example_com_test_pass_two_v2
-	RemoteName string
 }
 
 // GeneratePasswords generates passwords.
@@ -48,6 +35,7 @@ func GeneratePasswords(count, length uint) ([]string, error) {
 		length,
 		passgen.AlphabetDefault,
 	)
+
 	if err != nil {
 		return nil, err
 	}
@@ -66,6 +54,7 @@ func GeneratePassphrases(count uint) ([]string, error) {
 		passgen.PassphraseCasingDefault,
 		passgen.WordListDefault,
 	)
+
 	if err != nil {
 		return nil, err
 	}
@@ -80,23 +69,18 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) {
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
-	appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
+	secretConfigs := make(map[string]string)
+
+	appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
 	if err != nil {
-		return nil, err
+		return secretConfigs, err
 	}
-	// Set the STACK_NAME to be able to generate the remote name correctly.
-	appEnv["STACK_NAME"] = stackName
 
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
-		return nil, err
+		return secretConfigs, err
-	}
-	// Read the compose files without injecting environment variables.
-	configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
-	if err != nil {
-		return nil, err
 	}
 
 	var enabledSecrets []string
@@ -108,13 +92,12 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 
 	if len(enabledSecrets) == 0 {
 		logrus.Debugf("not generating app secrets, none enabled in recipe config")
-		return nil, nil
+		return secretConfigs, nil
 	}
 
-	secretValues := map[string]Secret{}
 	for secretId, secretConfig := range config.Secrets {
 		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
-			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
+			return secretConfigs, fmt.Errorf("missing version for secret? (%s)", secretId)
 		}
 
 		if !(slices.Contains(enabledSecrets, secretId)) {
@@ -124,58 +107,68 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 
 		lastIdx := strings.LastIndex(secretConfig.Name, "_")
 		secretVersion := secretConfig.Name[lastIdx+1:]
-		value := Secret{Version: secretVersion, RemoteName: secretConfig.Name}
+		secretConfigs[secretId] = secretVersion
-
-		// Check if the length modifier is set for this secret.
-		for envName, modifierValues := range appModifiers {
-			// configWithoutEnv contains the raw name as defined in the compose.yaml
-			// The name will look something like this:
-			//   name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
-			// To check if the current modifier is for the current secret we check
-			// if the raw name contains the env name (e.g. SECRET_TEST_PASS_TWO_VERSION).
-			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, envName) {
-				continue
 	}
-			lengthRaw, ok := modifierValues["length"]
+
-			if ok {
+	return secretConfigs, nil
-				length, err := strconv.Atoi(lengthRaw)
+}
+
+func ParseSecretValue(secret string) (secretValue, error) {
+	values := strings.Split(secret, "#")
+	if len(values) == 0 {
+		return secretValue{}, fmt.Errorf("unable to parse %s", secret)
+	}
+
+	if len(values) == 1 {
+		return secretValue{Version: values[0], Length: 0}, nil
+	}
+
+	split := strings.Split(values[1], "=")
+	parsed := split[len(split)-1]
+	stripped := strings.ReplaceAll(parsed, " ", "")
+	length, err := strconv.Atoi(stripped)
 	if err != nil {
-					return nil, err
+		return secretValue{}, err
-				}
-				value.Length = length
-			}
-			break
-		}
-		secretValues[secretId] = value
 	}
+	version := strings.ReplaceAll(values[0], " ", "")
 
-	return secretValues, nil
+	logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
+
+	return secretValue{Version: version, Length: length}, nil
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server string) (map[string]string, error) {
+func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, appName, server string) (map[string]string, error) {
-	secretsGenerated := map[string]string{}
+	secrets := make(map[string]string)
+
 	var mutex sync.Mutex
 	var wg sync.WaitGroup
-	ch := make(chan error, len(secrets))
+	ch := make(chan error, len(secretsFromConfig))
-	for n, v := range secrets {
+	for n, v := range secretsFromConfig {
 		wg.Add(1)
 
-		go func(secretName string, secret Secret) {
+		go func(secretName, secretValue string) {
 			defer wg.Done()
 
-			logrus.Debugf("attempting to generate and store %s on %s", secret.RemoteName, server)
+			parsedSecretValue, err := ParseSecretValue(secretValue)
-
-			if secret.Length > 0 {
-				passwords, err := GeneratePasswords(1, uint(secret.Length))
 			if err != nil {
 				ch <- err
 				return
 			}
 
-				if err := client.StoreSecret(cl, secret.RemoteName, passwords[0], server); err != nil {
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version)
+			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
+
+			if parsedSecretValue.Length > 0 {
+				passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length))
+				if err != nil {
+					ch <- err
+					return
+				}
+
+				if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secret.RemoteName)
+						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil
 					} else {
 						ch <- err
@@ -185,7 +178,7 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secretsGenerated[secretName] = passwords[0]
+				secrets[secretName] = passwords[0]
 			} else {
 				passphrases, err := GeneratePassphrases(1)
 				if err != nil {
@@ -193,9 +186,9 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 					return
 				}
 
-				if err := client.StoreSecret(cl, secret.RemoteName, passphrases[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secret.RemoteName)
+						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil
 					} else {
 						ch <- err
@@ -205,7 +198,7 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secretsGenerated[secretName] = passphrases[0]
+				secrets[secretName] = passphrases[0]
 			}
 			ch <- nil
 		}(n, v)
@@ -213,16 +206,16 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 
 	wg.Wait()
 
-	for range secrets {
+	for range secretsFromConfig {
 		err := <-ch
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	logrus.Debugf("generated and stored %v on %s", secrets, server)
+	logrus.Debugf("generated and stored %s on %s", secrets, server)
 
-	return secretsGenerated, nil
+	return secrets, nil
 }
 
 type secretStatus struct {
@@ -244,7 +237,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		return secStats, err
 	}
 
-	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 	if err != nil {
 		return secStats, err
 	}
@@ -264,9 +257,14 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		remoteSecretNames[cont.Spec.Annotations.Name] = true
 	}
 
-	for secretName, val := range secretsConfig {
+	for secretName, secretValue := range secretsConfig {
 		createdRemote := false
 
+		val, err := ParseSecretValue(secretValue)
+		if err != nil {
+			return secStats, err
+		}
+
 		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 		if _, ok := remoteSecretNames[secretRemoteName]; ok {
 			createdRemote = true

pkg/secret/secret_test.go

@@ -1,30 +1,42 @@
 package secret
 
 import (
+	"path"
 	"testing"
 
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestReadSecretsConfig(t *testing.T) {
-	composeFiles := []string{"./testdir/compose.yaml"}
+	offline := true
-	secretsFromConfig, err := ReadSecretsConfig("./testdir/.env.sample", composeFiles, "test_example_com")
+	recipe, err := recipe.Get("matrix-synapse", offline)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Simple secret
+	sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
-	assert.Equal(t, "test_example_com_test_pass_one_v2", secretsFromConfig["test_pass_one"].RemoteName)
+	if err != nil {
-	assert.Equal(t, "v2", secretsFromConfig["test_pass_one"].Version)
+		t.Fatal(err)
-	assert.Equal(t, 0, secretsFromConfig["test_pass_one"].Length)
+	}
 
-	// Has a length modifier
+	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
-	assert.Equal(t, "test_example_com_test_pass_two_v1", secretsFromConfig["test_pass_two"].RemoteName)
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	assert.Equal(t, "v1", secretsFromConfig["test_pass_two"].Version)
+	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
-	assert.Equal(t, 10, secretsFromConfig["test_pass_two"].Length)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	// Secret name does not include the secret id
+	opts := stack.Deploy{Composefiles: composeFiles}
-	assert.Equal(t, "test_example_com_pass_three_v2", secretsFromConfig["test_pass_three"].RemoteName)
+	config, err := loader.LoadComposefile(opts, sampleEnv)
-	assert.Equal(t, "v2", secretsFromConfig["test_pass_three"].Version)
+	if err != nil {
-	assert.Equal(t, 0, secretsFromConfig["test_pass_three"].Length)
+		t.Fatal(err)
+	}
+
+	for secretId := range config.Secrets {
+		assert.Contains(t, secretsFromConfig, secretId)
+	}
 }

pkg/secret/testdir/.env.sample

@@ -1,3 +0,0 @@
-SECRET_TEST_PASS_ONE_VERSION=v2
-SECRET_TEST_PASS_TWO_VERSION=v1 # length=10
-SECRET_TEST_PASS_THREE_VERSION=v2

pkg/secret/testdir/compose.yaml

@@ -1,21 +0,0 @@
----
-version: "3.8"
-
-services:
-  app:
-    image: nginx:1.21.0
-    secrets:
-      - test_pass_one
-      - test_pass_two
-      - test_pass_three
-
-secrets:
-  test_pass_one:
-    external: true
-    name: ${STACK_NAME}_test_pass_one_${SECRET_TEST_PASS_ONE_VERSION}  # should be removed
-  test_pass_two:
-    external: true
-    name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
-  test_pass_three:
-    external: true
-    name: ${STACK_NAME}_pass_three_${SECRET_TEST_PASS_THREE_VERSION} # secretId and name don't match

pkg/service/service.go

@@ -14,70 +14,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// GetService retrieves a service container based on a label. If prompt is true
-// and the retrievd count of service containers does not match 1, then a prompt
-// is presented to let the user choose. An error is returned when no service is
-// found.
-func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
-	services, err := cl.ServiceList(c, types.ServiceListOptions{})
-	if err != nil {
-		return swarm.Service{}, err
-	}
-
-	if len(services) == 0 {
-		return swarm.Service{}, fmt.Errorf("no services deployed?")
-	}
-
-	var matchingServices []swarm.Service
-	for _, service := range services {
-		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
-			matchingServices = append(matchingServices, service)
-		}
-	}
-
-	if len(matchingServices) == 0 {
-		return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label)
-	}
-
-	if len(matchingServices) > 1 {
-		var servicesRaw []string
-		for _, service := range matchingServices {
-			serviceName := service.Spec.Name
-			created := formatter.HumanDuration(service.CreatedAt.Unix())
-			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
-		}
-
-		if !prompt {
-			err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " "))
-			return swarm.Service{}, err
-		}
-
-		logrus.Warnf("ambiguous service list received, prompting for input")
-
-		var response string
-		prompt := &survey.Select{
-			Message: "which service are you looking for?",
-			Options: servicesRaw,
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return swarm.Service{}, err
-		}
-
-		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
-		for _, service := range matchingServices {
-			serviceName := strings.ToLower(service.Spec.Name)
-			if serviceName == chosenService {
-				return service, nil
-			}
-		}
-
-		logrus.Panic("failed to match chosen service")
-	}
-
-	return matchingServices[0], nil
-}
-
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.

pkg/upstream/commandconn/connection.go

@@ -18,7 +18,7 @@ import (
 //
 // ssh://<user>@<host> URL requires Docker 18.09 or later on the remote host.
 func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
-	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=60"})
+	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=5"})
 }
 
 func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) {

pkg/upstream/container/exec.go

@@ -13,10 +13,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// RunExec runs a command on a remote container. io.Writer corresponds to the
+func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error {
-// command output.
-func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string,
-	execConfig *types.ExecConfig) (io.Writer, error) {
 	ctx := context.Background()
 
 	// We need to check the tty _before_ we do the ContainerExecCreate, because
@@ -24,22 +21,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 	// there's no easy way to clean those up). But also in order to make "not
 	// exist" errors take precedence we do a dummy inspect first.
 	if _, err := client.ContainerInspect(ctx, containerID); err != nil {
-		return nil, err
+		return err
 	}
 	if !execConfig.Detach {
 		if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	response, err := client.ContainerExecCreate(ctx, containerID, *execConfig)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	execID := response.ID
 	if execID == "" {
-		return nil, errors.New("exec ID empty")
+		return errors.New("exec ID empty")
 	}
 
 	if execConfig.Detach {
@@ -47,13 +44,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 			Detach: execConfig.Detach,
 			Tty:    execConfig.Tty,
 		}
-		return nil, client.ContainerExecStart(ctx, execID, execStartCheck)
+		return client.ContainerExecStart(ctx, execID, execStartCheck)
 	}
 	return interactiveExec(ctx, dockerCli, client, execConfig, execID)
 }
 
 func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client,
-	execConfig *types.ExecConfig, execID string) (io.Writer, error) {
+	execConfig *types.ExecConfig, execID string) error {
 	// Interactive exec requested.
 	var (
 		out, stderr io.Writer
@@ -79,7 +76,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 	}
 	resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck)
 	if err != nil {
-		return out, err
+		return err
 	}
 	defer resp.Close()
 
@@ -110,10 +107,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 
 	if err := <-errCh; err != nil {
 		logrus.Debugf("Error hijack: %s", err)
-		return out, err
+		return err
 	}
 
-	return out, getExecExitStatus(ctx, client, execID)
+	return getExecExitStatus(ctx, client, execID)
 }
 
 func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {

pkg/upstream/stack/loader.go

@@ -18,24 +18,15 @@ func DontSkipValidation(opts *loader.Options) {
 	opts.SkipValidation = false
 }
 
-// SkipInterpolation skip interpolating environment variables.
-func SkipInterpolation(opts *loader.Options) {
-	opts.SkipInterpolation = true
-}
-
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
-func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
+func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
 	if err != nil {
 		return nil, err
 	}
 
-	if options == nil {
-		options = []func(*loader.Options){DontSkipValidation}
-	}
-
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails, options...)
+	config, err := loader.Load(configDetails, DontSkipValidation)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
 			return nil, fmt.Errorf("compose file contains unsupported options: %s",

scripts/docker/build.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-if [ ! -f .envrc ]; then
-    . .envrc.sample
-else
-    . .envrc
-fi
-
-git config --global --add safe.directory /abra  # work around funky file permissions
-
-make build

scripts/installer/installer

@@ -2,7 +2,7 @@
 
 ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.0-rc1-beta"
+RC_VERSION="0.8.1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do
@@ -65,19 +65,17 @@ function install_abra_release {
 
   checksums=$(wget -q -O- $checksums_url)
   checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p')
-  abra_download="/tmp/abra-download"
 
   echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..."
-
+  wget -q "$release_url" -O "$HOME/.local/bin/.abra-download"
-  wget -q "$release_url" -O $abra_download 
+  localsum=$(sha256sum $HOME/.local/bin/.abra-download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
-  localsum=$(sha256sum $abra_download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
   echo "checking if checksums match..."
   if [[ "$localsum" != "$checksum" ]]; then
       print_checksum_error
       exit 1
   fi
   echo "$(tput setaf 2)check successful!$(tput sgr0)"
-  mv "$abra_download" "$HOME/.local/bin/abra"
+  mv "$HOME/.local/bin/.abra-download" "$HOME/.local/bin/abra"
   chmod +x "$HOME/.local/bin/abra"
 
   x=$(echo $PATH | grep $HOME/.local/bin)

tests/integration/app_check.bats

@@ -70,13 +70,13 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }
@@ -86,7 +86,7 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
+  assert_output --partial 'behind 1'
 
   # NOTE(d1): we can't quite tell if this will fail or not in the future, so,
   # since it isn't an important part of what we're testing here, we don't check
@@ -94,7 +94,7 @@ setup(){
   run $ABRA app check "$TEST_APP_DOMAIN" --offline
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
+  assert_output --partial 'behind 1'
 
   _reset_recipe
 }

tests/integration/app_cmd.bats

@@ -25,24 +25,6 @@ teardown(){
   fi
 }
 
-# bats test_tags=slow
-@test "autocomplete" {
-  run $ABRA app cmd --generate-bash-completion
-  assert_success
-  assert_output "$TEST_APP_DOMAIN"
-
-  run $ABRA app cmd "$TEST_APP_DOMAIN" --generate-bash-completion
-  assert_success
-  assert_output "app"
-
-  run $ABRA app cmd "$TEST_APP_DOMAIN" app --generate-bash-completion
-  assert_success
-  assert_output "test_cmd
-test_cmd_arg
-test_cmd_args
-test_cmd_export"
-}
-
 @test "validate app argument" {
   run $ABRA app cmd
   assert_failure
@@ -58,7 +40,7 @@ test_cmd_export"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_success
   assert_output --partial 'baz'
 
@@ -70,7 +52,7 @@ test_cmd_export"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_failure
   assert_output --partial 'locally unstaged changes'
 
@@ -83,7 +65,7 @@ test_cmd_export"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
   assert_success
   assert_output --partial 'baz'
 
@@ -96,14 +78,14 @@ test_cmd_export"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_success
   assert_output --partial 'baz'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -113,14 +95,14 @@ test_cmd_export"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app cmd --local --offline "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --offline
   assert_success
   assert_output --partial 'baz'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -132,13 +114,13 @@ test_cmd_export"
 }
 
 @test "error if missing arguments when passing --local" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN"
+  run $ABRA app cmd "$TEST_APP_DOMAIN" --local
   assert_failure
   assert_output --partial 'missing arguments'
 }
 
 @test "cannot use --local and --user at same time" {
-  run $ABRA app cmd --local --user root "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --user root
   assert_failure
   assert_output --partial 'cannot use --local & --user together'
 }
@@ -147,7 +129,7 @@ test_cmd_export"
   run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh"
   assert_success
 
-  run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
   assert_failure
   assert_output --partial "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh does not exist"
 
@@ -155,25 +137,25 @@ test_cmd_export"
 }
 
 @test "error if missing command" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" doesnt_exist
+  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist --local
   assert_failure
   assert_output --partial "doesn't have a doesnt_exist function"
 }
 
 @test "run --local command" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_success
   assert_output --partial 'baz'
 }
 
 @test "run command with single arg" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_arg -- bing
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_arg --local -- bing
   assert_success
   assert_output --partial 'bing'
 }
 
 @test "run command with several args" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_args -- bong bang
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_args --local -- bong bang
   assert_success
   assert_output --partial 'bong bang'
 }

tests/integration/app_cp.bats

@@ -5,11 +5,9 @@ setup_file(){
   _common_setup
   _add_server
   _new_app
-  _deploy_app
 }
 
 teardown_file(){
-  _undeploy_app
   _rm_app
   _rm_server
 }
@@ -19,6 +17,13 @@ setup(){
   _common_setup
 }
 
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
 @test "validate app argument" {
   run $ABRA app cp
   assert_failure
@@ -49,120 +54,68 @@ setup(){
   assert_output --partial 'arguments must take $SERVICE:$PATH form'
 }
 
-@test "error if local file missing" {
+@test "detect 'coming FROM' syntax" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" thisfileshouldnotexist.txt app:/somewhere
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug
   assert_failure
-  assert_output --partial 'local stat thisfileshouldnotexist.txt: no such file or directory'
+  assert_output --partial 'coming FROM the container'
+}
+
+@test "detect 'going TO' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug
+  assert_failure
+  assert_output --partial 'going TO the container'
+}
+
+@test "error if local file missing" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere
+  assert_failure
+  assert_output --partial 'myfile.txt does not exist locally?'
 }
 
 # bats test_tags=slow
 @test "error if service doesn't exist" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+  _deploy_app
 
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ --debug
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/
   assert_failure
   assert_output --partial 'no containers matching'
 
-  _rm "$BATS_TMPDIR/myfile.txt"
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
 }
 
 # bats test_tags=slow
-@test "copy local file to container directory" {
+@test "copy to container" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
 
   run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
   assert_success
 
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
   assert_success
-  assert_output --partial "foo"
 
-  _rm "$BATS_TMPDIR/myfile.txt"
+  _undeploy_app
-  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
-@test "copy local file to container file (and override on remote)" {
+@test "copy from container" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+  _deploy_app
 
-  # create
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
   assert_success
 
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
-  assert_success
-  assert_output --partial "foo"
-
-  _mkfile "$BATS_TMPDIR/myfile.txt" "bar"
-
-  # override
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
   assert_success
 
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
-  assert_success
-  assert_output --partial "bar"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy local file to container file (and rename)" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
-
-  # rename
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile2.txt
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile2.txt
-  assert_success
-  assert_output --partial "foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
-}
-
-# bats test_tags=slow
-@test "copy local directory to container directory (and creates missing directory)" {
-  _mkdir "$BATS_TMPDIR/mydir"
-  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir" app:/etc
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/mydir
-  assert_success
-  assert_output --partial "myfile.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/mydir"
-}
-
-# bats test_tags=slow
-@test "copy local files to container directory" {
-  _mkdir "$BATS_TMPDIR/mydir"
-  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
-  _mkfile "$BATS_TMPDIR/mydir/myfile2.txt" "foo"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir/" app:/etc
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile.txt
-  assert_success
-  assert_output --partial "myfile.txt"
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile2.txt
-  assert_success
-  assert_output --partial "myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile*"
-}
-
-# bats test_tags=slow
-@test "copy container file to local directory" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
   assert_success
 
   run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR"
@@ -170,76 +123,8 @@ setup(){
   assert_exists "$BATS_TMPDIR/myfile.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
 
-  _rm "$BATS_TMPDIR/myfile.txt"
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
+  assert_success
-}
+
-
+  _undeploy_app
-# bats test_tags=slow
-@test "copy container file to local file" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/myfile.txt"
-  assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy container file to local file and rename" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile2.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/myfile2.txt"
-  assert bash -c "cat $BATS_TMPDIR/myfile2.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile2.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy container directory to local directory" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
-  assert_success
-
-  mkdir "$BATS_TMPDIR/mydir"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc "$BATS_TMPDIR/mydir"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/etc/myfile.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/etc/myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
-}
-
-# bats test_tags=slow
-@test "copy container files to local directory" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
-  assert_success
-
-  mkdir "$BATS_TMPDIR/mydir"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/ "$BATS_TMPDIR/mydir"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/myfile.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
 }

tests/integration/app_deploy.bats

@@ -16,7 +16,6 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-  _reset_recipe
 }
 
 teardown(){
@@ -83,13 +82,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
   _undeploy_app
@@ -101,7 +100,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
@@ -109,7 +108,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _undeploy_app
   _reset_recipe
@@ -117,9 +116,6 @@ teardown(){
 
 # bats test_tags=slow
 @test "deploy latest commit if no published versions and no --chaos" {
-  # TODO(d1): fix with a new test recipe which has no published versions?
-  skip "known issue, abra-test-recipe has published versions now"
-
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   _remove_tags
@@ -144,7 +140,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
@@ -277,10 +273,6 @@ teardown(){
 }
 
 @test "ensure domain is checked" {
-  if [[ "$TEST_SERVER" == "default" ]]; then
-      skip "domain checks are disabled for local server"
-  fi
-
   appDomain="custom-html.DOESNTEXIST"
 
   run $ABRA app new custom-html \

tests/integration/app_new.bats

@@ -18,24 +18,9 @@ setup(){
 }
 
 teardown(){
-  load "$PWD/tests/integration/helpers/common"
   _rm_app
 }
 
-@test "autocomplete" {
-  run $ABRA app new --generate-bash-completion
-  assert_success
-  assert_output --partial "traefik"
-  assert_output --partial "abra-test-recipe"
-
-  # Note: this test needs to be updated when a new version of the test recipe is published.
-  run $ABRA app new abra-test-recipe --generate-bash-completion
-  assert_success
-  assert_output "0.1.0+1.20.0
-0.1.1+1.20.2
-0.2.0+1.21.0"
-}
-
 @test "create new app" {
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -43,29 +28,10 @@ teardown(){
     --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
-}
-
-@test "create new app with version" {
-  run $ABRA app new "$TEST_RECIPE" 0.1.1+1.20.2 \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
-  assert_success
-  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" log -1
-  assert_output --partial "453db7121c0a56a7a8f15378f18fe3bf21ccfdef"
 }
 
 @test "does not overwrite existing env files" {
-  run $ABRA app new "$TEST_RECIPE" \
+  _new_app
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
-  assert_success
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -108,7 +74,8 @@ teardown(){
     --no-input \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
@@ -121,17 +88,18 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }
@@ -141,7 +109,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app new "$TEST_RECIPE" \
@@ -149,12 +117,13 @@ teardown(){
     --offline \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _reset_recipe
 }

tests/integration/app_remove.bats

@@ -104,7 +104,10 @@ teardown(){
 
   _undeploy_app
 
-  run $ABRA app volume rm "$TEST_APP_DOMAIN" --no-input
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
 
   run $ABRA app volume ls "$TEST_APP_DOMAIN"
@@ -129,6 +132,9 @@ teardown(){
 
   _undeploy_app
 
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
   assert_output --partial 'test-volume'

tests/integration/app_restore.bats

@@ -109,13 +109,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app restore "$TEST_APP_DOMAIN" app
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 }
 
 @test "ensure recipe not up to date if --offline" {
@@ -126,19 +126,19 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app restore "$TEST_APP_DOMAIN" app --offline
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST --offline
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "HEAD detached at $latestCommit"
+  refute_output --partial 'behind 3'
 }
 
 @test "error if missing service" {

tests/integration/app_rollback.bats

@@ -50,13 +50,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 }
 
 @test "ensure recipe not up to date if --offline" {
@@ -67,14 +67,14 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" \
     --no-input --no-converge-checks --offline
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
   assert_success
@@ -131,7 +131,7 @@ teardown(){
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
-    --no-input --chaos
+    --no-input --no-converge-checks --chaos
   assert_success
   assert_output --partial "$latestCommit"
   assert_output --partial 'chaos'

tests/integration/app_secret.bats

@@ -8,7 +8,7 @@ setup_file(){
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 }
@@ -19,6 +19,13 @@ teardown_file(){
   _reset_recipe
 }
 
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup

tests/integration/app_version.bats

@@ -59,8 +59,6 @@ teardown(){
 
 # bats test_tags=slow
 @test "error if not in catalogue" {
-  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
-
   _deploy_app
 
   run $ABRA app version "$TEST_APP_DOMAIN"
@@ -94,7 +92,7 @@ teardown(){
   assert_success
 
  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
+  sleep 3
 
   run $ABRA app volume remove "$appDomain" --no-input
   assert_success

tests/integration/app_volume.bats

@@ -78,6 +78,9 @@ teardown(){
 
   _undeploy_app
 
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'
@@ -89,6 +92,9 @@ teardown(){
 
   _undeploy_app
 
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'

tests/integration/helpers/app.bash

@@ -49,7 +49,7 @@ _reset_app(){
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 }

tests/integration/helpers/common.bash

@@ -5,7 +5,6 @@ _common_setup() {
   bats_load_library bats-assert
   bats_load_library bats-file
 
-  load "$PWD/tests/integration/helpers/file"
   load "$PWD/tests/integration/helpers/app"
   load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/recipe"

tests/integration/helpers/file.bash

@@ -1,24 +0,0 @@
-_mkfile() {
-  run bash -c "echo $2 > $1"
-  assert_success
-}
-
-_mkfile_remote() {
-  run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\""
-  assert_success
-}
-
-_mkdir() {
-  run bash -c "mkdir -p $1"
-  assert_success
-}
-
-_rm() {
-  run rm -rf "$1"
-  assert_success
-}
-
-_rm_remote() {
-  run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1"
-  assert_success
-}

tests/integration/helpers/git.bash

@@ -28,10 +28,3 @@ _reset_tags() {
   assert_success
   refute_output '0'
 }
-
-_set_git_author() {
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.email test@example.com
-  assert_success
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.name test
-  assert_success
-}

tests/integration/helpers/server.bash

@@ -11,11 +11,7 @@ _add_server() {
 }
 
 _rm_server() {
-  if [[ "$TEST_SERVER" == "default" ]]; then
-      run rm -rf "$ABRA_DIR/servers/default"
-  else
   run $ABRA server remove --no-input "$TEST_SERVER"
-  fi
   assert_success
   assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER"
 }

tests/integration/recipe_fetch.bats

@@ -5,17 +5,7 @@ setup() {
   _common_setup
 }
 
-@test "recipe fetch all" {
+@test "recipe fetch" {
-  run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
-  assert_success
-  assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"
-
-  run $ABRA recipe fetch
-  assert_success
-  assert_exists "$ABRA_DIR/recipes/matrix-synapse"
-}
-
-@test "recipe fetch single recipe" {
   run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"

tests/integration/recipe_lint.bats

@@ -66,13 +66,13 @@ setup() {
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA recipe lint "$TEST_RECIPE"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }
@@ -82,13 +82,13 @@ setup() {
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA recipe lint "$TEST_RECIPE" --offline
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _reset_recipe
 }

tests/integration/recipe_release.bats

@@ -15,11 +15,6 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-  _set_git_author
-}
-
-teardown() {
-  _reset_recipe
 }
 
 @test "validate recipe argument" {
@@ -56,6 +51,8 @@ teardown() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
   assert_output --partial '0.2.1+1.21.6'
+
+  _reset_recipe
 }
 
 # NOTE(d1): this test can't assert hardcoded versions since we upgrade a minor
@@ -84,6 +81,8 @@ teardown() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
   assert_output --regexp '0\.3\.0\+1\.2.*'
+
+  _reset_recipe "$TEST_RECIPE"
 }
 
 @test "unknown files not committed" {
@@ -101,21 +100,6 @@ teardown() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo
   assert_failure
   assert_output --partial "fatal: pathspec 'foo' did not match any files"
-}
+
-
+  _reset_recipe
-# NOTE: relies on 0.2.x being the last minor version
-@test "release with next release note" {
-  _mkfile "$ABRA_DIR/recipes/$TEST_RECIPE/release/next" "those are some release notes for the next release"
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add release/next
-  assert_success
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "added some release notes"
-  assert_success
-
-  run $ABRA recipe release  "$TEST_RECIPE" --no-input  --minor
-  assert_success
-  assert_output --partial 'no -p/--publish passed, not publishing'
-
-  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/next"
-  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0"
-  assert_file_contains "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0" "those are some release notes for the next release"
 }

tests/integration/recipe_upgrade.bats

@@ -61,14 +61,14 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input
   assert_success
   assert_output --partial 'can upgrade service: app'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }

tests/integration/recipe_version.bats

@@ -12,8 +12,6 @@ setup() {
 }
 
 @test "error if not present in catalogue" {
-  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
-
   run $ABRA recipe versions "$TEST_RECIPE"
   assert_failure
   assert_output --partial "is not published on the catalogue"