proper env modifiers support

This implements proper modifier support in the env file using this new fork of the godotenv library. The modifier implementation is quite basic for but can be improved later if needed. See this commit for the actual implementation.

Because we are now using proper modifer parsing, it does not affect the parsing of value, so this is possible again:
```
MY_VAR="#foo"
```
Closes coop-cloud/organising#535

.drone.yml

@@ -29,7 +29,7 @@ steps:
       event: tag
 
   - name: release
-    image: goreleaser/goreleaser:v1.24.0
+    image: goreleaser/goreleaser:v1.18.2
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token

.goreleaser.yml

@@ -29,8 +29,6 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
-      - "-s"
-      - "-w"
 
   - id: kadabra
     binary: kadabra
@@ -52,8 +50,12 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
-      - "-s"
-      - "-w"
+
+archives:
+  - replacements:
+      386: i386
+      amd64: x86_64
+    format: binary
 
 checksum:
   name_template: "checksums.txt"

AUTHORS.md

@@ -7,7 +7,6 @@
 - cassowary
 - codegod100
 - decentral1se
-- fauno
 - frando
 - kawaiipunk
 - knoflook

Dockerfile

@@ -1,29 +1,23 @@
-# Build image
 FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
 RUN apk add --no-cache \
+  ca-certificates \
   gcc \
   git \
   make \
   musl-dev
 
+RUN update-ca-certificates
+
 COPY . /app
 
 WORKDIR /app
 
 RUN CGO_ENABLED=0 make build
 
-# Release image ("slim")
-FROM alpine:3.19.1
-
-RUN apk add --no-cache \
-  ca-certificates \
-  git \
-  openssh
-
-RUN update-ca-certificates
+FROM scratch
 
 COPY --from=build /app/abra /abra
 

Makefile

@@ -53,6 +53,3 @@ test:
 
 loc:
 	@find . -name "*.go" | xargs wc -l
-
-deps:
-	@go get -t -u ./...

cli/app/backup.go

@@ -1,296 +1,414 @@
 package app
 
 import (
+	"archive/tar"
+	"context"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var snapshot string
-var snapshotFlag = &cli.StringFlag{
-	Name:        "snapshot, s",
-	Usage:       "Lists specific snapshot",
-	Destination: &snapshot,
-}
-
-var includePath string
-var includePathFlag = &cli.StringFlag{
-	Name:        "path, p",
-	Usage:       "Include path",
-	Destination: &includePath,
-}
-
-var resticRepo string
-var resticRepoFlag = &cli.StringFlag{
-	Name:        "repo, r",
-	Usage:       "Restic repository",
-	Destination: &resticRepo,
-}
-
-var appBackupListCommand = cli.Command{
-	Name:    "list",
-	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List all backups",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}
-
-var appBackupDownloadCommand = cli.Command{
-	Name:    "download",
-	Aliases: []string{"d"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "Download a backup",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteBackupDir := "/tmp/backup.tar.gz"
-		currentWorkingDir := "."
-		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
-			logrus.Fatal(err)
-		}
-
-		fmt.Println("backup successfully downloaded to current working directory")
-
-		return nil
-	},
-}
-
-var appBackupCreateCommand = cli.Command{
-	Name:    "create",
-	Aliases: []string{"c"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		resticRepoFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "Create a new backup",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if resticRepo != "" {
-			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
-			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}
-
-var appBackupSnapshotsCommand = cli.Command{
-	Name:    "snapshots",
-	Aliases: []string{"s"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List backup snapshots",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
+type backupConfig struct {
+	preHookCmd  string
+	postHookCmd string
+	backupPaths []string
 }
 
 var appBackupCommand = cli.Command{
 	Name:      "backup",
-	Aliases:   []string{"b"},
-	Usage:     "Manage app backups",
-	ArgsUsage: "<domain>",
-	Subcommands: []cli.Command{
-		appBackupListCommand,
-		appBackupSnapshotsCommand,
-		appBackupDownloadCommand,
-		appBackupCreateCommand,
+	Aliases:   []string{"bk"},
+	Usage:     "Run app backup",
+	ArgsUsage: "<domain> [<service>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app backup.
+
+A backup command and pre/post hook commands are defined in the recipe
+configuration. Abra reads this configuration and run the comands in the context
+of the deployed services. Pass <service> if you only want to back up a single
+service. All backups are placed in the ~/.abra/backups directory.
+
+A single backup file is produced for all backup paths specified for a service.
+If we have the following backup configuration:
+
+    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
+
+And we run "abra app backup example.com app", Abra will produce a file that
+looks like:
+
+    ~/.abra/backups/example_com_app_609341138.tar.gz
+
+This file is a compressed archive which contains all backup paths. To see paths, run:
+
+    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
+
+(Make sure to change the name of the backup file)
+
+This single file can be used to restore your app. See "abra app restore" for more.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		backupConfigs := make(map[string]backupConfig)
+		for _, service := range recipe.Config.Services {
+			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
+				if backupsEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					bkConfig := backupConfig{}
+
+					logrus.Debugf("backup config detected for %s", fullServiceName)
+
+					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
+						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
+						bkConfig.backupPaths = strings.Split(paths, ",")
+					}
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						bkConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						bkConfig.postHookCmd = postHookCmd
+					}
+
+					backupConfigs[service.Name] = bkConfig
+				}
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName != "" {
+			backupConfig, ok := backupConfigs[serviceName]
+			if !ok {
+				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
+			}
+
+			logrus.Infof("running backup for the %s service", serviceName)
+
+			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if len(backupConfigs) == 0 {
+				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
+			}
+
+			for serviceName, backupConfig := range backupConfigs {
+				logrus.Infof("running backup for the %s service", serviceName)
+
+				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
 	},
 }
+
+// TimeStamp generates a file name friendly timestamp.
+func TimeStamp() string {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	return strings.Replace(ts, ":", "-", -1)
+}
+
+// runBackup does the actual backup logic.
+func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
+	if len(bkConfig.backupPaths) == 0 {
+		return fmt.Errorf("backup paths are empty for %s?", serviceName)
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if bkConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
+	}
+
+	var tempBackupPaths []string
+	for _, remoteBackupPath := range bkConfig.backupPaths {
+		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
+		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
+
+		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
+
+		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		if err != nil {
+			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+		}
+		defer content.Close()
+
+		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
+		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
+		if err := copyToFile(localBackupPath, preArchive); err != nil {
+			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+		}
+
+		tempBackupPaths = append(tempBackupPaths, localBackupPath)
+	}
+
+	logrus.Infof("compressing and merging archives...")
+
+	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
+		logrus.Debugf("failed to merge archive files: %s", err.Error())
+		if err := cleanupTempArchives(tempBackupPaths); err != nil {
+			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+		}
+		return fmt.Errorf("failed to merge archive files: %s", err.Error())
+	}
+
+	if err := cleanupTempArchives(tempBackupPaths); err != nil {
+		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+	}
+
+	if bkConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
+	}
+
+	return nil
+}
+
+func copyToFile(outfile string, r io.Reader) error {
+	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
+	if err != nil {
+		return err
+	}
+
+	tmpPath := tmpFile.Name()
+
+	_, err = io.Copy(tmpFile, r)
+	tmpFile.Close()
+
+	if err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	if err = os.Rename(tmpPath, outfile); err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	return nil
+}
+
+func cleanupTempArchives(tarPaths []string) error {
+	for _, tarPath := range tarPaths {
+		if err := os.RemoveAll(tarPath); err != nil {
+			return err
+		}
+
+		logrus.Debugf("remove temporary archive file %s", tarPath)
+	}
+
+	return nil
+}
+
+func mergeArchives(tarPaths []string, serviceName string) error {
+	var out io.Writer
+	var cout *pgzip.Writer
+
+	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
+
+	fout, err := os.Create(localBackupPath)
+	if err != nil {
+		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
+	}
+
+	defer fout.Close()
+	out = fout
+
+	cout = pgzip.NewWriter(out)
+	out = cout
+
+	tw := tar.NewWriter(out)
+
+	for _, tarPath := range tarPaths {
+		if err := addTar(tw, tarPath); err != nil {
+			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		return fmt.Errorf("failed to close tar writer %v", err)
+	}
+
+	if cout != nil {
+		if err := cout.Flush(); err != nil {
+			return fmt.Errorf("failed to flush: %s", err)
+		} else if err = cout.Close(); err != nil {
+			return fmt.Errorf("failed to close compressed writer: %s", err)
+		}
+	}
+
+	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
+
+	return nil
+}
+
+func addTar(tw *tar.Writer, pth string) (err error) {
+	var tr *tar.Reader
+	var rc io.ReadCloser
+	var hdr *tar.Header
+
+	if tr, rc, err = openTarFile(pth); err != nil {
+		return
+	}
+
+	for {
+		if hdr, err = tr.Next(); err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			break
+		}
+		if err = tw.WriteHeader(hdr); err != nil {
+			break
+		} else if _, err = io.Copy(tw, tr); err != nil {
+			break
+		}
+	}
+	if err == nil {
+		err = rc.Close()
+	} else {
+		rc.Close()
+	}
+	return
+}
+
+func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
+	var fin *os.File
+	var n int
+	buff := make([]byte, 1024)
+
+	if fin, err = os.Open(pth); err != nil {
+		return
+	}
+
+	if n, err = fin.Read(buff); err != nil {
+		fin.Close()
+		return
+	} else if n == 0 {
+		fin.Close()
+		err = fmt.Errorf("%s is empty", pth)
+		return
+	}
+
+	if _, err = fin.Seek(0, 0); err != nil {
+		fin.Close()
+		return
+	}
+
+	rc = fin
+	tr = tar.NewReader(rc)
+
+	return tr, rc, nil
+}

cli/app/cp.go

@@ -2,24 +2,19 @@ package app
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"io"
 	"os"
-	"path"
-	"path/filepath"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -54,14 +49,46 @@ And if you want to copy that file back to your current working directory locally
 		dst := c.Args().Get(2)
 		if src == "" {
 			logrus.Fatal("missing <src> argument")
-		}
-		if dst == "" {
+		} else if dst == "" {
 			logrus.Fatal("missing <dest> argument")
 		}
 
-		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
-		if err != nil {
-			logrus.Fatal(err)
+		parsedSrc := strings.SplitN(src, ":", 2)
+		parsedDst := strings.SplitN(dst, ":", 2)
+		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
+		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
+			logrus.Fatal(errorMsg)
+		} else if len(parsedSrc) != 2 {
+			if len(parsedDst) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		} else if len(parsedDst) != 2 {
+			if len(parsedSrc) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		}
+
+		var service string
+		var srcPath string
+		var dstPath string
+		isToContainer := false // <container:src> <dst>
+		if len(parsedSrc) == 2 {
+			service = parsedSrc[0]
+			srcPath = parsedSrc[1]
+			dstPath = dst
+			logrus.Debugf("assuming transfer is coming FROM the container")
+		} else if len(parsedDst) == 2 {
+			service = parsedDst[0]
+			dstPath = parsedDst[1]
+			srcPath = src
+			isToContainer = true // <src> <container:dst>
+			logrus.Debugf("assuming transfer is going TO the container")
+		}
+
+		if isToContainer {
+			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", srcPath)
+			}
 		}
 
 		cl, err := client.New(app.Server)
@@ -69,18 +96,7 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Fatal(err)
 		}
 
-		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
-
-		if toContainer {
-			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
-		} else {
-			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
-		}
-		if err != nil {
+		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -88,292 +104,46 @@ And if you want to copy that file back to your current working directory locally
 	},
 }
 
-var errServiceMissing = errors.New("one of <src>/<dest> arguments must take $SERVICE:$PATH form")
+func configureAndCp(
+	c *cli.Context,
+	cl *dockerClient.Client,
+	app config.App,
+	srcPath string,
+	dstPath string,
+	service string,
+	isToContainer bool) error {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
 
-// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH
-func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) {
-	parsedSrc := strings.SplitN(src, ":", 2)
-	parsedDst := strings.SplitN(dst, ":", 2)
-	if len(parsedSrc)+len(parsedDst) != 3 {
-		return "", "", "", false, errServiceMissing
-	}
-	if len(parsedSrc) == 2 {
-		return parsedSrc[1], dst, parsedSrc[0], false, nil
-	}
-	if len(parsedDst) == 2 {
-		return src, parsedDst[1], parsedDst[0], true, nil
-	}
-	return "", "", "", false, errServiceMissing
-}
-
-// CopyToContainer copies a file or directory from the local file system to the container.
-// See the possible copy modes and their documentation.
-func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
-	srcStat, err := os.Stat(srcPath)
+	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
 	if err != nil {
-		return fmt.Errorf("local %s ", err)
+		logrus.Fatal(err)
 	}
 
-	dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath)
-	dstExists := true
-	if err != nil {
-		if errdefs.IsNotFound(err) {
-			dstExists = false
-		} else {
-			return fmt.Errorf("remote path: %s", err)
-		}
-	}
+	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
-	mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists)
-	if err != nil {
-		return err
-	}
-	movePath := ""
-	switch mode {
-	case CopyModeDirToDir:
-		// Add the src directory to the destination path
-		_, srcDir := path.Split(srcPath)
-		dstPath = path.Join(dstPath, srcDir)
-
-		// Make sure the dst directory exits.
-		dcli, err := command.NewDockerCli()
-		if err != nil {
-			return err
-		}
-		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          []string{"mkdir", "-p", dstPath},
-			Detach:       false,
-			Tty:          true,
-		}); err != nil {
-			return fmt.Errorf("create remote directory: %s", err)
-		}
-	case CopyModeFileToFile:
-		// Remove the file component from the path, since docker can only copy
-		// to a directory.
-		dstPath, _ = path.Split(dstPath)
-	case CopyModeFileToFileRename:
-		// Copy the file to the temp directory and move it to its dstPath
-		// afterwards.
-		movePath = dstPath
-		dstPath = "/tmp"
-	}
-
-	toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	if isToContainer {
+		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
 		content, err := archive.TarWithOptions(srcPath, toTarOpts)
 		if err != nil {
-		return err
+			logrus.Fatal(err)
 		}
 
-	logrus.Debugf("copy %s from local to %s on container", srcPath, dstPath)
 		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
-		return err
-	}
-
-	if movePath != "" {
-		_, srcFile := path.Split(srcPath)
-		dcli, err := command.NewDockerCli()
-		if err != nil {
-			return err
-		}
-		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          []string{"mv", path.Join("/tmp", srcFile), movePath},
-			Detach:       false,
-			Tty:          true,
-		}); err != nil {
-			return fmt.Errorf("create remote directory: %s", err)
-		}
-	}
-
-	return nil
-}
-
-// CopyFromContainer copies a file or directory from the given container to the local file system.
-// See the possible copy modes and their documentation.
-func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
-	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
-	if err != nil {
-		if errdefs.IsNotFound(err) {
-			return fmt.Errorf("remote: %s does not exist", srcPath)
-		} else {
-			return fmt.Errorf("remote path: %s", err)
-		}
-	}
-
-	dstStat, err := os.Stat(dstPath)
-	dstExists := true
-	var dstMode os.FileMode
-	if err != nil {
-		if os.IsNotExist(err) {
-			dstExists = false
-		} else {
-			return fmt.Errorf("remote path: %s", err)
+		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
+			logrus.Fatal(err)
 		}
 	} else {
-		dstMode = dstStat.Mode()
-	}
-
-	mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists)
+		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
 		if err != nil {
-		return err
-	}
-
-	moveDstDir := ""
-	moveDstFile := ""
-	switch mode {
-	case CopyModeFileToFile:
-		// Remove the file component from the path, since docker can only copy
-		// to a directory.
-		dstPath, _ = path.Split(dstPath)
-	case CopyModeFileToFileRename:
-		// Copy the file to the temp directory and move it to its dstPath
-		// afterwards.
-		moveDstFile = dstPath
-		dstPath = "/tmp"
-	case CopyModeFilesToDir:
-		// Copy the directory to the temp directory and move it to its
-		// dstPath afterwards.
-		moveDstDir = path.Join(dstPath, "/")
-		dstPath = "/tmp"
-
-		// Make sure the temp directory always gets removed
-		defer os.Remove(path.Join("/tmp"))
-	}
-
-	content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath)
-	if err != nil {
-		return fmt.Errorf("copy: %s", err)
+			logrus.Fatal(err)
 		}
 		defer content.Close()
-	if err := archive.Untar(content, dstPath, &archive.TarOptions{
-		NoOverwriteDirNonDir: true,
-		Compression:          archive.Gzip,
-		NoLchown:             true,
-	}); err != nil {
-		return fmt.Errorf("untar: %s", err)
+		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+			logrus.Fatal(err)
+		}
 	}
 
-	if moveDstFile != "" {
-		_, srcFile := path.Split(strings.TrimSuffix(srcPath, "/"))
-		if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil {
-			return err
-		}
-	}
-	if moveDstDir != "" {
-		_, srcDir := path.Split(strings.TrimSuffix(srcPath, "/"))
-		if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-var (
-	ErrCopyDirToFile  = fmt.Errorf("can't copy dir to file")
-	ErrDstDirNotExist = fmt.Errorf("destination directory does not exist")
-)
-
-type CopyMode int
-
-const (
-	// Copy a src file to a dest file. The src and dest file names are the same.
-	//  <dir_src>/<file> + <dir_dst>/<file> -> <dir_dst>/<file>
-	CopyModeFileToFile = CopyMode(iota)
-	// Copy a src file to a dest file. The src and dest file names are  not the same.
-	//  <dir_src>/<file_src> + <dir_dst>/<file_dst> -> <dir_dst>/<file_dst>
-	CopyModeFileToFileRename
-	// Copy a src file to dest directory. The dest file gets created in the dest
-	// folder with the src filename.
-	//  <dir_src>/<file> + <dir_dst> -> <dir_dst>/<file>
-	CopyModeFileToDir
-	// Copy a src directory to dest directory.
-	//  <dir_src> + <dir_dst> -> <dir_dst>/<dir_src>
-	CopyModeDirToDir
-	// Copy all files in the src directory to the dest directory. This works recursively.
-	//  <dir_src>/ + <dir_dst> -> <dir_dst>/<files_from_dir_src>
-	CopyModeFilesToDir
-)
-
-// copyMode takes a src and dest path and file mode to determine the copy mode.
-// See the possible copy modes and their documentation.
-func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) {
-	_, srcFile := path.Split(srcPath)
-	_, dstFile := path.Split(dstPath)
-	if srcMode.IsDir() {
-		if !dstExists {
-			return -1, ErrDstDirNotExist
-		}
-		if dstMode.IsDir() {
-			if strings.HasSuffix(srcPath, "/") {
-				return CopyModeFilesToDir, nil
-			}
-			return CopyModeDirToDir, nil
-		}
-		return -1, ErrCopyDirToFile
-	}
-
-	if dstMode.IsDir() {
-		return CopyModeFileToDir, nil
-	}
-
-	if srcFile != dstFile {
-		return CopyModeFileToFileRename, nil
-	}
-
-	return CopyModeFileToFile, nil
-}
-
-// moveDir moves all files from a source path to the destination path recursively.
-func moveDir(sourcePath, destPath string) error {
-	return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath))
-		if info.IsDir() {
-			err := os.Mkdir(newPath, info.Mode())
-			if err != nil {
-				if os.IsExist(err) {
-					return nil
-				}
-				return err
-			}
-		}
-		if info.Mode().IsRegular() {
-			return moveFile(p, newPath)
-		}
-		return nil
-	})
-}
-
-// moveFile moves a file from a source path to a destination path.
-func moveFile(sourcePath, destPath string) error {
-	inputFile, err := os.Open(sourcePath)
-	if err != nil {
-		return err
-	}
-	outputFile, err := os.Create(destPath)
-	if err != nil {
-		inputFile.Close()
-		return err
-	}
-	defer outputFile.Close()
-	_, err = io.Copy(outputFile, inputFile)
-	inputFile.Close()
-	if err != nil {
-		return err
-	}
-
-	// Remove file after succesfull copy.
-	err = os.Remove(sourcePath)
-	if err != nil {
-		return err
-	}
 	return nil
 }

cli/app/cp_test.go

@@ -1,113 +0,0 @@
-package app
-
-import (
-	"os"
-	"testing"
-)
-
-func TestParse(t *testing.T) {
-	tests := []struct {
-		src         string
-		dst         string
-		srcPath     string
-		dstPath     string
-		service     string
-		toContainer bool
-		err         error
-	}{
-		{src: "foo", dst: "bar", err: errServiceMissing},
-		{src: "app:foo", dst: "app:bar", err: errServiceMissing},
-		{src: "app:foo", dst: "bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: false},
-		{src: "foo", dst: "app:bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: true},
-	}
-
-	for i, tc := range tests {
-		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(tc.src, tc.dst)
-		if srcPath != tc.srcPath {
-			t.Errorf("[%d] srcPath: want (%s), got(%s)", i, tc.srcPath, srcPath)
-		}
-		if dstPath != tc.dstPath {
-			t.Errorf("[%d] dstPath: want (%s), got(%s)", i, tc.dstPath, dstPath)
-		}
-		if service != tc.service {
-			t.Errorf("[%d] service: want (%s), got(%s)", i, tc.service, service)
-		}
-		if toContainer != tc.toContainer {
-			t.Errorf("[%d] toConainer: want (%t), got(%t)", i, tc.toContainer, toContainer)
-		}
-		if err == nil && tc.err != nil && err.Error() != tc.err.Error() {
-			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
-		}
-	}
-}
-
-func TestCopyMode(t *testing.T) {
-	tests := []struct {
-		srcPath   string
-		dstPath   string
-		srcMode   os.FileMode
-		dstMode   os.FileMode
-		dstExists bool
-		mode      CopyMode
-		err       error
-	}{
-		{
-			srcPath:   "foo.txt",
-			dstPath:   "foo.txt",
-			srcMode:   os.ModePerm,
-			dstMode:   os.ModePerm,
-			dstExists: true,
-			mode:      CopyModeFileToFile,
-		},
-		{
-			srcPath:   "foo.txt",
-			dstPath:   "bar.txt",
-			srcMode:   os.ModePerm,
-			dstExists: true,
-			mode:      CopyModeFileToFileRename,
-		},
-		{
-			srcPath:   "foo",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstMode:   os.ModeDir,
-			dstExists: true,
-			mode:      CopyModeDirToDir,
-		},
-		{
-			srcPath:   "foo/",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstMode:   os.ModeDir,
-			dstExists: true,
-			mode:      CopyModeFilesToDir,
-		},
-		{
-			srcPath:   "foo",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstExists: false,
-			mode:      -1,
-			err:       ErrDstDirNotExist,
-		},
-		{
-			srcPath:   "foo",
-			dstPath:   "foo",
-			srcMode:   os.ModeDir,
-			dstMode:   os.ModePerm,
-			dstExists: true,
-			mode:      -1,
-			err:       ErrCopyDirToFile,
-		},
-	}
-
-	for i, tc := range tests {
-		mode, err := copyMode(tc.srcPath, tc.dstPath, tc.srcMode, tc.dstMode, tc.dstExists)
-		if mode != tc.mode {
-			t.Errorf("[%d] mode: want (%d), got(%d)", i, tc.mode, mode)
-		}
-		if err != tc.err {
-			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
-		}
-	}
-}

cli/app/deploy.go

@@ -97,19 +97,6 @@ recipes.
 			logrus.Fatal(err)
 		}
 
-		// NOTE(d1): check out specific version before dealing with secrets. This
-		// is because we need to deal with GetComposeFiles under the hood and these
-		// files change from version to version which therefore affects which
-		// secrets might be generated
-		version := deployedVersion
-		if specificVersion != "" {
-			version = specificVersion
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		secStats, err := secret.PollSecretsStatus(cl, app)
 		if err != nil {
 			logrus.Fatal(err)
@@ -129,6 +116,15 @@ recipes.
 			}
 		}
 
+		version := deployedVersion
+		if specificVersion != "" {
+			version = specificVersion
+			logrus.Debugf("choosing %s as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if !internal.Chaos && specificVersion == "" {
 			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {

cli/app/errors.go

@@ -13,7 +13,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
@@ -97,7 +97,7 @@ func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error
 		filters := filters.NewArgs()
 		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
 
-		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
+		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
 		if err != nil {
 			return err
 		}

cli/app/logs.go

@@ -2,27 +2,75 @@ package app
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
-	"slices"
 	"sync"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
-	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/api/types/swarm"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+var logOpts = types.ContainerLogsOptions{
+	ShowStderr: true,
+	ShowStdout: true,
+	Since:      "",
+	Until:      "",
+	Timestamps: true,
+	Follow:     true,
+	Tail:       "20",
+	Details:    false,
+}
+
+// stackLogs lists logs for all stack services
+func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
+	filters, err := app.Filters(true, false)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(context.Background(), serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			if internal.StdErrOnly {
+				logOpts.ShowStdout = false
+			}
+
+			logs, err := client.ServiceLogs(context.Background(), s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+
+	wg.Wait()
+
+	os.Exit(0)
+}
+
 var appLogsCommand = cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
@@ -57,70 +105,37 @@ var appLogsCommand = cli.Command{
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
+		logOpts.Since = internal.SinceLogs
+
 		serviceName := c.Args().Get(1)
-		serviceNames := []string{}
-		if serviceName != "" {
-			serviceNames = []string{serviceName}
-		}
-		err = tailLogs(cl, app, serviceNames)
-		if err != nil {
+		if serviceName == "" {
+			logrus.Debugf("tailing logs for all %s services", app.Recipe)
+			stackLogs(c, app, cl)
+		} else {
+			logrus.Debugf("tailing logs for %s", serviceName)
+			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
 				logrus.Fatal(err)
 			}
+		}
 
 		return nil
 	},
 }
 
-// tailLogs prints logs for the given app with optional service names to be
-// filtered on. It also checks if the latest task is not runnning and then
-// prints the past tasks.
-func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) error {
-	f, err := app.Filters(true, false, serviceNames...)
-	if err != nil {
-		return err
-	}
-
-	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: f})
-	if err != nil {
-		return err
-	}
-
-	var wg sync.WaitGroup
-	for _, service := range services {
+func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
 	filters := filters.NewArgs()
-		filters.Add("name", service.Spec.Name)
-		tasks, err := cl.TaskList(context.Background(), types.TaskListOptions{Filters: f})
+	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
+
+	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
 	if err != nil {
-			return err
-		}
-		if len(tasks) > 0 {
-			// Need to sort the tasks by the CreatedAt field in the inverse order.
-			// Otherwise they are in the reversed order and not sorted properly.
-			slices.SortFunc[[]swarm.Task](tasks, func(t1, t2 swarm.Task) int {
-				return int(t2.Meta.CreatedAt.Unix() - t1.Meta.CreatedAt.Unix())
-			})
-			lastTask := tasks[0].Status
-			if lastTask.State != swarm.TaskStateRunning {
-				for _, task := range tasks {
-					logrus.Errorf("[%s] %s State %s: %s", service.Spec.Name, task.Meta.CreatedAt.Format(time.RFC3339), task.Status.State, task.Status.Err)
-				}
-			}
+		logrus.Fatal(err)
 	}
 
-		// Collect the logs in a go routine, so the logs from all services are
-		// collected in parallel.
-		wg.Add(1)
-		go func(serviceID string) {
-			logs, err := cl.ServiceLogs(context.Background(), serviceID, containerTypes.LogsOptions{
-				ShowStderr: true,
-				ShowStdout: !internal.StdErrOnly,
-				Since:      internal.SinceLogs,
-				Until:      "",
-				Timestamps: true,
-				Follow:     true,
-				Tail:       "20",
-				Details:    false,
-			})
+	if internal.StdErrOnly {
+		logOpts.ShowStdout = false
+	}
+
+	logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -130,11 +145,6 @@ func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) er
 	if err != nil && err != io.EOF {
 		logrus.Fatal(err)
 	}
-		}(service.ID)
-	}
-
-	// Wait for all log streams to be closed.
-	wg.Wait()
 
 	return nil
 }

cli/app/new.go

@@ -10,6 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/jsontable"
+	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/AlecAivazis/survey/v2"
@@ -54,16 +55,8 @@ var appNewCommand = cli.Command{
 		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
-	ArgsUsage: "[<recipe>] [<version>]",
-	BashComplete: func(ctx *cli.Context) {
-		args := ctx.Args()
-		switch len(args) {
-		case 0:
-			autocomplete.RecipeNameComplete(ctx)
-		case 1:
-			autocomplete.RecipeVersionComplete(ctx.Args().Get(0))
-		}
-	},
+	ArgsUsage:    "[<recipe>]",
+	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
@@ -76,36 +69,10 @@ var appNewCommand = cli.Command{
 					logrus.Fatal(err)
 				}
 			}
-			if c.Args().Get(1) == "" {
-				var version string
-
-				recipeVersions, err := recipePkg.GetRecipeVersions(recipe.Name, internal.Offline)
-				if err != nil {
-					logrus.Fatal(err)
-				}
-
-				// NOTE(d1): determine whether recipe versions exist or not and check
-				// out the latest version or current HEAD
-				if len(recipeVersions) > 0 {
-					latest := recipeVersions[len(recipeVersions)-1]
-					for tag := range latest {
-						version = tag
-					}
-
-					if err := recipePkg.EnsureVersion(recipe.Name, version); err != nil {
-						logrus.Fatal(err)
-					}
-				} else {
 			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
 				logrus.Fatal(err)
 			}
 		}
-			} else {
-				if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
 
 		if err := ensureServerFlag(); err != nil {
 			logrus.Fatal(err)
@@ -141,7 +108,7 @@ var appNewCommand = cli.Command{
 			}
 
 			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, config.StackName(internal.Domain))
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 			if err != nil {
 				return err
 			}
@@ -201,14 +168,14 @@ var appNewCommand = cli.Command{
 type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) {
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
 	// NOTE(d1): trim to match app.StackName() implementation
-	if len(sanitisedAppName) > config.MAX_SANITISED_APP_NAME_LENGTH {
-		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH])
-		sanitisedAppName = sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH]
+	if len(sanitisedAppName) > 45 {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
+		sanitisedAppName = sanitisedAppName[:45]
 	}
 
-	secrets, err := secret.GenerateSecrets(cl, secretsConfig, internal.NewAppServer)
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
 	if err != nil {
 		return nil, err
 	}
@@ -231,7 +198,7 @@ func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secr
 }
 
 // ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
-func ensureDomainFlag(recipe recipePkg.Recipe, server string) error {
+func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 	if internal.Domain == "" && !internal.NoInput {
 		prompt := &survey.Input{
 			Message: "Specify app domain",
@@ -250,7 +217,7 @@ func ensureDomainFlag(recipe recipePkg.Recipe, server string) error {
 }
 
 // promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(recipeName string, secretsConfig map[string]secret.Secret) error {
+func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
 	if len(secretsConfig) == 0 {
 		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
 		return nil

cli/app/ps.go

@@ -2,8 +2,7 @@ package app
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
+	"strings"
 	"time"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -11,13 +10,11 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/recipe"
-	abraService "coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/buger/goterm"
 	dockerFormatter "github.com/docker/cli/cli/command/formatter"
-	containerTypes "github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -30,7 +27,6 @@ var appPsCommand = cli.Command{
 	ArgsUsage:   "<domain>",
 	Description: "Show a more detailed status output of a specific deployed app",
 	Flags: []cli.Flag{
-		internal.MachineReadableFlag,
 		internal.WatchFlag,
 		internal.DebugFlag,
 	},
@@ -44,7 +40,7 @@ var appPsCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -53,15 +49,6 @@ var appPsCommand = cli.Command{
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		statuses, err := config.GetAppStatuses([]config.App{app}, true)
-		if statusMeta, ok := statuses[app.StackName()]; ok {
-			if _, exists := statusMeta["chaos"]; !exists {
-				if err := recipe.EnsureVersion(app.Recipe, deployedVersion); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-		}
-
 		if !internal.Watch {
 			showPSOutput(c, app, cl)
 			return nil
@@ -79,77 +66,36 @@ var appPsCommand = cli.Command{
 
 // showPSOutput renders ps output.
 func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
-	composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+	filters, err := app.Filters(true, true)
 	if err != nil {
 		logrus.Fatal(err)
-		return
 	}
 
-	deployOpts := stack.Deploy{
-		Composefiles: composeFiles,
-		Namespace:    app.StackName(),
-		Prune:        false,
-		ResolveImage: stack.ResolveImageAlways,
-	}
-	compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+	containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
 	if err != nil {
 		logrus.Fatal(err)
-		return
 	}
 
-	var tablerows [][]string
-	allContainerStats := make(map[string]map[string]string)
-	for _, service := range compose.Services {
-		filters := filters.NewArgs()
-		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
-
-		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-			return
-		}
-
-		var containerStats map[string]string
-
-		if len(containers) == 0 {
-			containerStats = map[string]string{
-				"service name": service.Name,
-				"image":        "unknown",
-				"created":      "unknown",
-				"status":       "unknown",
-				"state":        "unknown",
-				"ports":        "unknown",
-			}
-		} else {
-			container := containers[0]
-			containerStats = map[string]string{
-				"service name": abraService.ContainerToServiceName(container.Names, app.StackName()),
-				"image":        formatter.RemoveSha(container.Image),
-				"created":      formatter.HumanDuration(container.Created),
-				"status":       container.Status,
-				"state":        container.State,
-				"ports":        dockerFormatter.DisplayablePorts(container.Ports),
-			}
-		}
-		allContainerStats[containerStats["service name"]] = containerStats
-
-		var tablerow []string = []string{containerStats["service name"], containerStats["image"], containerStats["created"], containerStats["status"], containerStats["state"], containerStats["ports"]}
-		tablerows = append(tablerows, tablerow)
-	}
-	if internal.MachineReadable {
-		jsonstring, err := json.Marshal(allContainerStats)
-		if err != nil {
-			logrus.Fatal(err)
-		} else {
-			fmt.Println(string(jsonstring))
-		}
-		return
-	} else {
 	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
 	table := formatter.CreateTable(tableCol)
-		for _, row := range tablerows {
-			table.Append(row)
+
+	for _, container := range containers {
+		var containerNames []string
+		for _, containerName := range container.Names {
+			trimmed := strings.TrimPrefix(containerName, "/")
+			containerNames = append(containerNames, trimmed)
 		}
+
+		tableRow := []string{
+			service.ContainerToServiceName(container.Names, app.StackName()),
+			formatter.RemoveSha(container.Image),
+			formatter.HumanDuration(container.Created),
+			container.Status,
+			container.State,
+			dockerFormatter.DisplayablePorts(container.Ports),
+		}
+		table.Append(tableRow)
+	}
+
 	table.Render()
-	}
 }

cli/app/remove.go

@@ -3,7 +3,6 @@ package app
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -12,6 +11,7 @@ import (
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -110,19 +110,26 @@ flag.
 			logrus.Fatal(err)
 		}
 
-		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, fs)
+		volumeListOptions := volume.ListOptions{fs}
+		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
+		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		volumeNames := client.GetVolumeNames(volumeList)
 
-		if len(volumeNames) > 0 {
-			err := client.RemoveVolumes(cl, context.Background(), volumeNames, internal.Force, 5)
-			if err != nil {
-				log.Fatalf("removing volumes failed: %s", err)
+		var vols []string
+		for _, vol := range volumeList {
+			vols = append(vols, vol.Name)
 		}
 
-			logrus.Infof("%d volumes removed successfully", len(volumeNames))
+		if len(vols) > 0 {
+			for _, vol := range vols {
+				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("volume %s removed", vol))
+			}
 		} else {
 			logrus.Info("no volumes to remove")
 		}

cli/app/restore.go

@@ -1,82 +1,223 @@
 package app
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var targetPath string
-var targetPathFlag = &cli.StringFlag{
-	Name:        "target, t",
-	Usage:       "Target path",
-	Destination: &targetPath,
+type restoreConfig struct {
+	preHookCmd  string
+	postHookCmd string
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Restore an app backup",
-	ArgsUsage: "<domain> <service>",
+	Usage:     "Run app restore",
+	ArgsUsage: "<domain> <service> <file>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		targetPathFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app restore.
+
+Pre/post hook commands are defined in the recipe configuration. Abra reads this
+configuration and run the comands in the context of the service before
+restoring the backup.
+
+Unlike "abra app backup", restore must be run on a per-service basis. You can
+not restore all services in one go. Backup files produced by Abra are
+compressed archives which use absolute paths. This allows Abra to restore
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
+
+Example:
+
+    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
+`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
+		}
+
+		backupPath := c.Args().Get(2)
+		if backupPath == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
+		}
+
+		if _, err := os.Stat(backupPath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s doesn't exist?", backupPath)
+			}
+		}
+
+		restoreConfigs := make(map[string]restoreConfig)
+		for _, service := range recipe.Config.Services {
+			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
+				if restoreEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					rsConfig := restoreConfig{}
+
+					logrus.Debugf("restore config detected for %s", fullServiceName)
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						rsConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						rsConfig.postHookCmd = postHookCmd
+					}
+
+					restoreConfigs[service.Name] = rsConfig
+				}
+			}
+		}
+
+		rsConfig, ok := restoreConfigs[serviceName]
+		if !ok {
+			rsConfig = restoreConfig{}
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if targetPath != "" {
-			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
-			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
+		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
+
+// runRestore does the actual restore logic.
+func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if rsConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
+	}
+
+	backupReader, err := os.Open(backupPath)
+	if err != nil {
+		return err
+	}
+
+	content, err := archive.DecompressStream(backupReader)
+	if err != nil {
+		return err
+	}
+
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
+	restorePath := "/"
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
+		return err
+	}
+
+	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
+
+	if rsConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
+	}
+
+	return nil
+}

cli/app/run.go

@@ -91,7 +91,7 @@ var appRunCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 

cli/app/secret.go

@@ -91,7 +91,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -104,7 +104,7 @@ var appSecretGenerateCommand = cli.Command{
 				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
 			s.Version = secretVersion
-			secrets = map[string]secret.Secret{
+			secrets = map[string]secret.SecretValue{
 				secretName: s,
 			}
 		}
@@ -114,7 +114,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secrets, app.Server)
+		secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -274,7 +274,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/services.go

@@ -11,7 +11,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -48,7 +48,7 @@ var appServicesCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
+		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/upgrade.go

@@ -31,7 +31,6 @@ var appUpgradeCommand = cli.Command{
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
 		internal.OfflineFlag,
-		internal.ReleaseNotesFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -194,17 +193,17 @@ recipes.
 		// check out the tag and then they'll appear to be missing. this covers
 		// when we obviously will forget to write release notes before publishing
 		var releaseNotes string
-		if chosenUpgrade != "" {
-			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
-			if err != nil {
-				logrus.Fatal(err)
-			}
 		for _, version := range versions {
 			parsedVersion, err := tagcmp.Parse(version)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-				if parsedVersion.IsGreaterThan(parsedDeployedVersion) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
+			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !(parsedVersion.Equals(parsedDeployedVersion)) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
 				note, err := internal.GetReleaseNotes(app.Recipe, version)
 				if err != nil {
 					return err
@@ -214,7 +213,6 @@ recipes.
 				}
 			}
 		}
-		}
 
 		if !internal.Chaos {
 			if err := recipePkg.EnsureVersion(app.Recipe, chosenUpgrade); err != nil {
@@ -271,12 +269,6 @@ recipes.
 			}
 		}
 
-		if internal.ReleaseNotes {
-			fmt.Println()
-			fmt.Print(releaseNotes)
-			return nil
-		}
-
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}

cli/app/version.go

@@ -10,7 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/distribution/reference"
+	"github.com/docker/distribution/reference"
 	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"

cli/app/volume.go

@@ -2,7 +2,6 @@ package app
 
 import (
 	"context"
-	"log"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -132,12 +131,12 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		}
 
 		if len(volumesToRemove) > 0 {
-			err := client.RemoveVolumes(cl, context.Background(), volumesToRemove, internal.Force, 5)
+			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
 			if err != nil {
-				log.Fatalf("removing volumes failed: %s", err)
+				logrus.Fatal(err)
 			}
 
-			logrus.Infof("%d volumes removed successfully", len(volumesToRemove))
+			logrus.Info("volumes removed successfully")
 		} else {
 			logrus.Info("no volumes removed")
 		}

cli/internal/backup.go

@@ -1,67 +1,35 @@
 package internal
 
 import (
-	"context"
-
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/service"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
+	"strings"
 )
 
-// RetrieveBackupBotContainer gets the deployed backupbot container.
-func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) {
-	ctx := context.Background()
-	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
-	if err != nil {
-		return types.Container{}, err
+// SafeSplit splits up a string into a list of commands safely.
+func SafeSplit(s string) []string {
+	split := strings.Split(s, " ")
+
+	var result []string
+	var inquote string
+	var block string
+	for _, i := range split {
+		if inquote == "" {
+			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
+				inquote = string(i[0])
+				block = strings.TrimPrefix(i, inquote) + " "
+			} else {
+				result = append(result, i)
+			}
+		} else {
+			if !strings.HasSuffix(i, inquote) {
+				block += i + " "
+			} else {
+				block += strings.TrimSuffix(i, inquote)
+				inquote = ""
+				result = append(result, block)
+				block = ""
+			}
+		}
 	}
 
-	logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
-
-	filters := filters.NewArgs()
-	filters.Add("name", chosenService.Spec.Name)
-	targetContainer, err := containerPkg.GetContainer(
-		ctx,
-		cl,
-		filters,
-		NoInput,
-	)
-	if err != nil {
-		return types.Container{}, err
-	}
-
-	return targetContainer, nil
-}
-
-// RunBackupCmdRemote runs a backup related command on a remote backupbot container.
-func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error {
-	execBackupListOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          []string{"/usr/bin/backup", "--", backupCmd},
-		Detach:       false,
-		Env:          execEnv,
-		Tty:          true,
-	}
-
-	logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts)
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil {
-		return err
-	}
-
-	return nil
+	return result
 }

cli/internal/cli.go

@@ -95,16 +95,6 @@ var OfflineFlag = &cli.BoolFlag{
 	Usage:       "Prefer offline & filesystem access when possible",
 }
 
-// ReleaseNotes stores the variable from ReleaseNotesFlag.
-var ReleaseNotes bool
-
-// ReleaseNotesFlag turns on/off printing only release notes when upgrading.
-var ReleaseNotesFlag = &cli.BoolFlag{
-	Name:        "releasenotes, r",
-	Destination: &ReleaseNotes,
-	Usage:       "Only show release notes",
-}
-
 // MachineReadable stores the variable from MachineReadableFlag
 var MachineReadable bool
 

cli/internal/command.go

@@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		Tty:          false,
 	}
 
-	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
 		shell = "/bin/sh"
 	}
@@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		execCreateOpts.Tty = false
 	}
 
-	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		return err
 	}
 

cli/internal/recipe.go

@@ -6,7 +6,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/distribution/reference"
+	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
 )
 

cli/recipe/fetch.go

@@ -3,7 +3,6 @@ package recipe
 import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -18,31 +17,26 @@ var recipeFetchCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
-		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
+
 		if recipeName != "" {
 			internal.ValidateRecipe(c)
-			if err := recipe.Ensure(recipeName); err != nil {
-				logrus.Fatal(err)
-			}
-			return nil
 		}
 
-		catalogue, err := recipe.ReadRecipeCatalogue(internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(recipeName); err != nil {
 			logrus.Fatal(err)
 		}
 
-		catlBar := formatter.CreateProgressbar(len(catalogue), "fetching latest recipes...")
-		for recipeName := range catalogue {
-			if err := recipe.Ensure(recipeName); err != nil {
-				logrus.Error(err)
+		if err := recipe.EnsureUpToDate(recipeName); err != nil {
+			logrus.Fatal(err)
 		}
-			catlBar.Add(1)
+
+		if err := recipe.EnsureLatest(recipeName); err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil

cli/recipe/release.go

@@ -1,9 +1,7 @@
 package recipe
 
 import (
-	"errors"
 	"fmt"
-	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -17,7 +15,7 @@ import (
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/distribution/reference"
+	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -142,7 +140,7 @@ your SSH keys configured on your account.
 
 // getImageVersions retrieves image versions for a recipe
 func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
-	services := make(map[string]string)
+	var services = make(map[string]string)
 
 	missingTag := false
 	for _, service := range recipe.Config.Services {
@@ -209,10 +207,6 @@ func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string
 		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
 	}
 
-	if err := addReleaseNotes(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
 	if err := commitRelease(recipe, tagString); err != nil {
 		logrus.Fatal(err)
 	}
@@ -243,82 +237,6 @@ func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
 	return git.CreateTagOptions{Message: msg}, nil
 }
 
-// addReleaseNotes checks if the release/next release note exists and moves the
-// file to release/<tag>.
-func addReleaseNotes(recipe recipe.Recipe, tag string) error {
-	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	tagReleaseNotePath := path.Join(repoPath, "release", tag)
-	if _, err := os.Stat(tagReleaseNotePath); err == nil {
-		// Release note for current tag already exist exists.
-		return nil
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-
-	nextReleaseNotePath := path.Join(repoPath, "release", "next")
-	if _, err := os.Stat(nextReleaseNotePath); err == nil {
-		// release/next note exists. Move it to release/<tag>
-		if internal.Dry {
-			logrus.Debugf("dry run: move release note from 'next' to %s", tag)
-			return nil
-		}
-		if !internal.NoInput {
-			prompt := &survey.Input{
-				Message: "Use release note in release/next?",
-			}
-			var addReleaseNote bool
-			if err := survey.AskOne(prompt, &addReleaseNote); err != nil {
-				return err
-			}
-			if !addReleaseNote {
-				return nil
-			}
-		}
-		err := os.Rename(nextReleaseNotePath, tagReleaseNotePath)
-		if err != nil {
-			return err
-		}
-		err = gitPkg.Add(repoPath, path.Join("release", "next"), internal.Dry)
-		if err != nil {
-			return err
-		}
-		err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry)
-		if err != nil {
-			return err
-		}
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-
-	// No release note exists for the current release.
-	if internal.NoInput {
-		return nil
-	}
-
-	prompt := &survey.Input{
-		Message: "Release Note (leave empty for no release note)",
-	}
-	var releaseNote string
-	if err := survey.AskOne(prompt, &releaseNote); err != nil {
-		return err
-	}
-
-	if releaseNote == "" {
-		return nil
-	}
-
-	err := os.WriteFile(tagReleaseNotePath, []byte(releaseNote), 0o644)
-	if err != nil {
-		return err
-	}
-	err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func commitRelease(recipe recipe.Recipe, tag string) error {
 	if internal.Dry {
 		logrus.Debugf("dry run: no changes committed")
@@ -486,10 +404,6 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip
 		}
 	}
 
-	if err := addReleaseNotes(recipe, tagString); err != nil {
-		logrus.Fatal(err)
-	}
-
 	if err := commitRelease(recipe, tagString); err != nil {
 		logrus.Fatalf("failed to commit changes: %s", err.Error())
 	}

cli/recipe/upgrade.go

@@ -18,7 +18,7 @@ import (
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/distribution/reference"
+	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )

cli/recipe/version.go

@@ -54,9 +54,8 @@ var recipeVersionCommand = cli.Command{
 			logrus.Fatalf("%s has no catalogue published versions?", recipe.Name)
 		}
 
-		tableCols := []string{"version", "service", "image", "tag"}
-		aggregated_table := formatter.CreateTable(tableCols)
 		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
+			tableCols := []string{"version", "service", "image", "tag"}
 			table := formatter.CreateTable(tableCols)
 			for version, meta := range recipeMeta.Versions[i] {
 				var versions [][]string
@@ -68,10 +67,11 @@ var recipeVersionCommand = cli.Command{
 
 				for _, version := range versions {
 					table.Append(version)
-					aggregated_table.Append(version)
 				}
 
-				if !internal.MachineReadable {
+				if internal.MachineReadable {
+					table.JSONRender()
+				} else {
 					table.SetAutoMergeCellsByColumnIndex([]int{0})
 					table.SetAlignment(tablewriter.ALIGN_LEFT)
 					table.Render()
@@ -79,9 +79,6 @@ var recipeVersionCommand = cli.Command{
 				}
 			}
 		}
-		if internal.MachineReadable {
-			aggregated_table.JSONRender()
-		}
 
 		return nil
 	},

cli/server/add.go

@@ -53,7 +53,7 @@ func cleanUp(domainName string) {
 // Docker manages SSH connection details. These are stored to disk in
 // ~/.docker. Abra can manage this completely for the user, so it's an
 // implementation detail.
-func newContext(c *cli.Context, domainName string) error {
+func newContext(c *cli.Context, domainName, username, port string) error {
 	store := contextPkg.NewDefaultDockerContextStore()
 	contexts, err := store.Store.List()
 	if err != nil {
@@ -67,9 +67,9 @@ func newContext(c *cli.Context, domainName string) error {
 		}
 	}
 
-	logrus.Debugf("creating context with domain %s", domainName)
+	logrus.Debugf("creating context with domain %s, username %s and port %s", domainName, username, port)
 
-	if err := client.CreateContext(domainName); err != nil {
+	if err := client.CreateContext(domainName, username, port); err != nil {
 		return err
 	}
 
@@ -158,7 +158,12 @@ developer machine.
 			logrus.Fatal(err)
 		}
 
-		if err := newContext(c, domainName); err != nil {
+		hostConfig, err := sshPkg.GetHostConfig(domainName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
 			logrus.Fatal(err)
 		}
 

go.mod

@@ -3,131 +3,118 @@ module coopcloud.tech/abra
 go 1.21
 
 require (
-	coopcloud.tech/tagcmp v0.0.0-20230809071031-eb3e7758d4eb
-	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355
+	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/distribution/distribution v2.8.3+incompatible
-	github.com/docker/cli v26.1.4+incompatible
-	github.com/docker/docker v26.1.4+incompatible
+	github.com/docker/cli v24.0.7+incompatible
+	github.com/docker/distribution v2.8.3+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.12.0
-	github.com/google/go-cmp v0.6.0
+	github.com/go-git/go-git/v5 v5.10.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.14.4
+	github.com/schollz/progressbar/v3 v3.14.1
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
-	github.com/BurntSushi/toml v1.4.0 // indirect
-	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/BurntSushi/toml v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.9.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cloudflare/circl v1.3.9 // indirect
-	github.com/containerd/log v0.1.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
-	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
-	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/miekg/pkcs11 v1.0.3 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/moby/docker-image-spec v1.3.1 // indirect
-	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.1.13 // indirect
+	github.com/opencontainers/runc v1.1.0 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.54.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/skeema/knownhosts v1.2.2 // indirect
+	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
-	go.opentelemetry.io/otel v1.27.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
-	go.opentelemetry.io/otel/metric v1.27.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.27.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect
-	go.opentelemetry.io/otel/trace v1.27.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/term v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
-	google.golang.org/grpc v1.64.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	golang.org/x/tools v0.13.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/buger/goterm v1.0.4
-	github.com/containerd/containerd v1.7.18 // indirect
+	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/containers/image v3.0.2+incompatible
 	github.com/containers/storage v1.38.2 // indirect
 	github.com/decentral1se/passgen v1.0.1
-	github.com/docker/docker-credential-helpers v0.8.2 // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/fvbommel/sortorder v1.1.0 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7
-	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.5
+	github.com/klauspost/pgzip v1.2.6
+	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
-	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
-	github.com/spf13/cobra v1.8.1 // indirect
-	github.com/stretchr/testify v1.9.0
+	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/stretchr/testify v1.8.4
 	github.com/theupdateframework/notary v0.7.0 // indirect
-	github.com/urfave/cli v1.22.15
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
-	golang.org/x/sys v0.21.0
+	github.com/urfave/cli v1.22.9
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
+	golang.org/x/sys v0.14.0
 )

pkg/autocomplete/autocomplete.go

@@ -51,20 +51,6 @@ func RecipeNameComplete(c *cli.Context) {
 	}
 }
 
-// RecipeVersionComplete completes versions for the recipe.
-func RecipeVersionComplete(recipeName string) {
-	catl, err := recipe.ReadRecipeCatalogue(false)
-	if err != nil {
-		logrus.Warn(err)
-	}
-
-	for _, v := range catl[recipeName].Versions {
-		for v2 := range v {
-			fmt.Println(v2)
-		}
-	}
-}
-
 // ServerNameComplete completes server names.
 func ServerNameComplete(c *cli.Context) {
 	files, err := config.LoadAppFiles("")

pkg/client/context.go

@@ -14,16 +14,19 @@ import (
 
 type Context = contextStore.Metadata
 
-// CreateContext creates a new Docker context.
-func CreateContext(contextName string) error {
-	host := fmt.Sprintf("ssh://%s", contextName)
-
+func CreateContext(contextName string, user string, port string) error {
+	host := contextName
+	if user != "" {
+		host = fmt.Sprintf("%s@%s", user, host)
+	}
+	if port != "" {
+		host = fmt.Sprintf("%s:%s", host, port)
+	}
+	host = fmt.Sprintf("ssh://%s", host)
 	if err := createContext(contextName, host); err != nil {
 		return err
 	}
-
 	logrus.Debugf("created the %s context", contextName)
-
 	return nil
 }
 

pkg/client/registry.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/containers/image/docker"
 	"github.com/containers/image/types"
-	"github.com/distribution/reference"
+	"github.com/docker/distribution/reference"
 )
 
 // GetRegistryTags retrieves all tags of an image from a container registry.

pkg/client/volumes.go

@@ -2,17 +2,15 @@ package client
 
 import (
 	"context"
-	"fmt"
-	"time"
 
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
 )
 
 func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
-	volumeListOKBody, err := cl.VolumeList(ctx, volume.ListOptions{Filters: fs})
+	volumeListOptions := volume.ListOptions{fs}
+	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
 		return volumeList, err
@@ -31,32 +29,13 @@ func GetVolumeNames(volumes []*volume.Volume) []string {
 	return volumeNames
 }
 
-func RemoveVolumes(cl *client.Client, ctx context.Context, volumeNames []string, force bool, retries int) error {
+func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
 	for _, volName := range volumeNames {
-		err := retryFunc(5, func() error {
-			return cl.VolumeRemove(context.Background(), volName, force)
-		})
+		err := cl.VolumeRemove(ctx, volName, force)
 		if err != nil {
-			return fmt.Errorf("volume %s: %s", volName, err)
+			return err
 		}
 	}
-	return nil
-}
 
-// retryFunc retries the given function for the given retries. After the nth
-// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
-// It returns an error if the function still failed after the last retry.
-func retryFunc(retries int, fn func() error) error {
-	for i := 0; i < retries; i++ {
-		err := fn()
-		if err == nil {
 	return nil
-		}
-		if i+1 < retries {
-			sleep := time.Duration(i+1) * time.Duration(i+1)
-			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
-			time.Sleep(sleep * time.Second)
-		}
-	}
-	return fmt.Errorf("%d retries failed", retries)
 }

pkg/client/volumes_test.go

@@ -1,26 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestRetryFunc(t *testing.T) {
-	err := retryFunc(1, func() error { return nil })
-	if err != nil {
-		t.Errorf("should not return an error: %s", err)
-	}
-
-	i := 0
-	fn := func() error {
-		i++
-		return fmt.Errorf("oh no, something went wrong!")
-	}
-	err = retryFunc(2, fn)
-	if err == nil {
-		t.Error("should return an error")
-	}
-	if i != 2 {
-		t.Errorf("The function should have been called 1 times, got %d", i)
-	}
-}

pkg/compose/compose.go

@@ -11,8 +11,8 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/distribution/reference"
 	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
 )
 

pkg/config/app.go

@@ -50,61 +50,34 @@ type App struct {
 	Path   string
 }
 
-// See documentation of config.StackName
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
 	}
 
-	stackName := StackName(a.Name)
+	stackName := SanitiseAppName(a.Name)
+
+	if len(stackName) > 45 {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:45])
+		stackName = stackName[:45]
+	}
 
 	a.Env["STACK_NAME"] = stackName
 
 	return stackName
 }
 
-// StackName gets whatever the docker safe (uses the right delimiting
-// character, e.g. "_") stack name is for the app. In general, you don't want
-// to use this to show anything to end-users, you want use a.Name instead.
-func StackName(appName string) string {
-	stackName := SanitiseAppName(appName)
-
-	if len(stackName) > MAX_SANITISED_APP_NAME_LENGTH {
-		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:MAX_SANITISED_APP_NAME_LENGTH])
-		stackName = stackName[:MAX_SANITISED_APP_NAME_LENGTH]
-	}
-
-	return stackName
-}
-
-// Filters retrieves app filters for querying the container runtime. By default
-// it filters on all services in the app. It is also possible to pass an
-// otional list of service names, which get filtered instead.
-//
-// Due to upstream issues, filtering works different depending on what you're
+// Filters retrieves exact app filters for querying the container runtime. Due
+// to upstream issues, filtering works different depending on what you're
 // querying. So, for example, secrets don't work with regex! The caller needs
 // to implement their own validation that the right secrets are matched. In
 // order to handle these cases, we provide the `appendServiceNames` /
 // `exactMatch` modifiers.
-func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) {
+func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) {
 	filters := filters.NewArgs()
-	if len(services) > 0 {
-		for _, serviceName := range services {
-			filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch))
-		}
-		return filters, nil
-	}
-
-	// When not appending the service name, just add one filter for the whole
-	// stack.
-	if !appendServiceNames {
-		f := fmt.Sprintf("%s", a.StackName())
-		if exactMatch {
-			f = fmt.Sprintf("^%s", f)
-		}
-		filters.Add("name", f)
-		return filters, nil
-	}
 
 	composeFiles, err := GetComposeFiles(a.Recipe, a.Env)
 	if err != nil {
@@ -118,23 +91,28 @@ func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (f
 	}
 
 	for _, service := range compose.Services {
-		f := ServiceFilter(a.StackName(), service.Name, exactMatch)
-		filters.Add("name", f)
+		var filter string
+
+		if appendServiceNames {
+			if exactMatch {
+				filter = fmt.Sprintf("^%s_%s", a.StackName(), service.Name)
+			} else {
+				filter = fmt.Sprintf("%s_%s", a.StackName(), service.Name)
+			}
+		} else {
+			if exactMatch {
+				filter = fmt.Sprintf("^%s", a.StackName())
+			} else {
+				filter = fmt.Sprintf("%s", a.StackName())
+			}
+		}
+
+		filters.Add("name", filter)
 	}
 
 	return filters, nil
 }
 
-// ServiceFilter creates a filter string for filtering a service in the docker
-// container runtime. When exact match is true, it uses regex to match the
-// string exactly.
-func ServiceFilter(stack, service string, exact bool) string {
-	if exact {
-		return fmt.Sprintf("^%s_%s", stack, service)
-	}
-	return fmt.Sprintf("%s_%s", stack, service)
-}
-
 // ByServer sort a slice of Apps
 type ByServer []App
 
@@ -355,7 +333,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
-	err = ioutil.WriteFile(appEnvPath, envSample, 0o664)
+	err = ioutil.WriteFile(appEnvPath, envSample, 0664)
 	if err != nil {
 		return err
 	}
@@ -617,7 +595,7 @@ func GetLabel(compose *composetypes.Config, stackName string, label string) stri
 
 // GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value
 func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
-	timeout := 50 // Default Timeout
+	var timeout = 50 // Default Timeout
 	var err error = nil
 	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
 		logrus.Debugf("timeout label: %s", timeoutLabel)

pkg/config/app_test.go

@@ -1,15 +1,12 @@
 package config_test
 
 import (
-	"encoding/json"
 	"fmt"
 	"reflect"
 	"testing"
 
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -109,89 +106,3 @@ func TestGetComposeFilesError(t *testing.T) {
 		}
 	}
 }
-
-func TestFilters(t *testing.T) {
-	oldDir := config.RECIPES_DIR
-	config.RECIPES_DIR = "./testdir"
-	defer func() {
-		config.RECIPES_DIR = oldDir
-	}()
-
-	app, err := config.NewApp(config.AppEnv{
-		"DOMAIN": "test.example.com",
-		"RECIPE": "test-recipe",
-	}, "test_example_com", config.AppFile{
-		Path:   "./testdir/filtertest.end",
-		Server: "local",
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	f, err := app.Filters(false, false)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f, map[string]map[string]bool{
-		"name": {
-			"test_example_com": true,
-		},
-	})
-
-	f2, err := app.Filters(false, true)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f2, map[string]map[string]bool{
-		"name": {
-			"^test_example_com": true,
-		},
-	})
-
-	f3, err := app.Filters(true, false)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f3, map[string]map[string]bool{
-		"name": {
-			"test_example_com_bar": true,
-			"test_example_com_foo": true,
-		},
-	})
-
-	f4, err := app.Filters(true, true)
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f4, map[string]map[string]bool{
-		"name": {
-			"^test_example_com_bar": true,
-			"^test_example_com_foo": true,
-		},
-	})
-
-	f5, err := app.Filters(false, false, "foo")
-	if err != nil {
-		t.Error(err)
-	}
-	compareFilter(t, f5, map[string]map[string]bool{
-		"name": {
-			"test_example_com_foo": true,
-		},
-	})
-}
-
-func compareFilter(t *testing.T, f1 filters.Args, f2 map[string]map[string]bool) {
-	t.Helper()
-	j1, err := f1.MarshalJSON()
-	if err != nil {
-		t.Error(err)
-	}
-	j2, err := json.Marshal(f2)
-	if err != nil {
-		t.Error(err)
-	}
-	if diff := cmp.Diff(string(j2), string(j1)); diff != "" {
-		t.Errorf("filters mismatch (-want +got):\n%s", diff)
-	}
-}

pkg/config/env.go

@@ -36,11 +36,6 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
-const MAX_SANITISED_APP_NAME_LENGTH = 45
-const MAX_DOCKER_SECRET_LENGTH = 64
-
-var BackupbotLabel = "coop-cloud.backupbot.enabled"
-
 // envVarModifiers is a list of env var modifier strings. These are added to
 // env vars as comments and modify their processing by Abra, e.g. determining
 // how long secrets should be.

pkg/config/testdir/filtertest.env

@@ -1,2 +0,0 @@
-RECIPE=test-recipe
-DOMAIN=test.example.com

pkg/config/testdir/test-recipe/compose.yml

@@ -1,6 +0,0 @@
-version: "3.8"
-services:
-  foo:
-    image: debian
-  bar:
-    image: debian

pkg/container/container.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
-	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
@@ -18,7 +17,7 @@ import (
 // count of containers does not match 1, then a prompt is presented to let the
 // user choose. A count of 0 is handled gracefully.
 func GetContainer(c context.Context, cl *client.Client, filters filters.Args, noInput bool) (types.Container, error) {
-	containerOpts := containerTypes.ListOptions{Filters: filters}
+	containerOpts := types.ContainerListOptions{Filters: filters}
 	containers, err := cl.ContainerList(c, containerOpts)
 	if err != nil {
 		return types.Container{}, err
@@ -29,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) > 1 {
+	if len(containers) != 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")
@@ -69,15 +68,3 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 
 	return containers[0], nil
 }
-
-// GetContainerFromStackAndService retrieves the container for the given stack and service.
-func GetContainerFromStackAndService(cl *client.Client, stack, service string) (types.Container, error) {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", stack, service))
-
-	container, err := GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return types.Container{}, err
-	}
-	return container, nil
-}

pkg/git/add.go

@@ -1,27 +0,0 @@
-package git
-
-import (
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-)
-
-// Add adds a file to the git index.
-func Add(repoPath, path string, dryRun bool) error {
-	repo, err := git.PlainOpen(repoPath)
-	if err != nil {
-		return err
-	}
-
-	worktree, err := repo.Worktree()
-	if err != nil {
-		return err
-	}
-
-	if dryRun {
-		logrus.Debugf("dry run: adding %s", path)
-	} else {
-		worktree.Add(path)
-	}
-
-	return nil
-}

pkg/lint/recipe.go

@@ -10,7 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
-	"github.com/distribution/reference"
+	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
@@ -115,13 +115,6 @@ var LintRules = map[string][]LintRule{
 			HowToResolve: "upload your recipe to git.coopcloud.tech/coop-cloud/...",
 			Function:     LintHasRecipeRepo,
 		},
-		{
-			Ref:          "R015",
-			Level:        "warn",
-			Description:  "long secret names",
-			HowToResolve: "reduce length of secret names to 12 chars",
-			Function:     LintSecretLengths,
-		},
 	},
 	"error": {
 		{
@@ -408,16 +401,6 @@ func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
 	return true, nil
 }
 
-func LintSecretLengths(recipe recipe.Recipe) (bool, error) {
-	for name := range recipe.Config.Secrets {
-		if len(name) > 12 {
-			return false, fmt.Errorf("secret %s is longer than 12 characters", name)
-		}
-	}
-
-	return true, nil
-}
-
 func LintValidTags(recipe recipe.Recipe) (bool, error) {
 	recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
 

pkg/recipe/recipe.go

@@ -22,8 +22,8 @@ import (
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/abra/pkg/web"
 	"coopcloud.tech/tagcmp"
-	"github.com/distribution/reference"
 	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
@@ -264,20 +264,6 @@ func (r Recipe) SampleEnv() (map[string]string, error) {
 	return sampleEnv, nil
 }
 
-// Ensure makes sure the recipe exists, is up to date and has the latest version checked out.
-func Ensure(recipeName string) error {
-	if err := EnsureExists(recipeName); err != nil {
-		return err
-	}
-	if err := EnsureUpToDate(recipeName); err != nil {
-		return err
-	}
-	if err := EnsureLatest(recipeName); err != nil {
-		return err
-	}
-	return nil
-}
-
 // EnsureExists ensures that a recipe is locally cloned
 func EnsureExists(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
@@ -945,7 +931,6 @@ func GetRecipeVersions(recipeName string, offline bool) (RecipeVersions, error)
 	if err != nil {
 		return versions, err
 	}
-
 	sortRecipeVersions(versions)
 
 	logrus.Debugf("collected %s for %s", versions, recipeName)

pkg/secret/secret.go

@@ -21,24 +21,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// Secret represents a secret.
-type Secret struct {
-	// Version comes from the secret version environment variable.
-	// For example:
-	//  SECRET_FOO=v1
+// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
+// secret definition.
+type SecretValue struct {
 	Version string
-	// Length comes from the length modifier at the secret version environment
-	// variable. For Example:
-	//   SECRET_FOO=v1 # length=12
 	Length  int
-	// RemoteName is the name of the secret on the server. For example:
-	//   name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
-	// With the following:
-	//   STACK_NAME=test_example_com
-	//   SECRET_TEST_PASS_TWO_VERSION=v2
-	// Will have this remote name:
-	//   test_example_com_test_pass_two_v2
-	RemoteName string
 }
 
 // GeneratePasswords generates passwords.
@@ -80,27 +67,24 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) {
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
 	appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
 	if err != nil {
 		return nil, err
 	}
-	// Set the STACK_NAME to be able to generate the remote name correctly.
-	appEnv["STACK_NAME"] = stackName
 
 	opts := stack.Deploy{Composefiles: composeFiles}
-	composeConfig, err := loader.LoadComposefile(opts, appEnv)
+	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
 		return nil, err
 	}
-	// Read the compose files without injecting environment variables.
 	configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
 	if err != nil {
 		return nil, err
 	}
 
 	var enabledSecrets []string
-	for _, service := range composeConfig.Services {
+	for _, service := range config.Services {
 		for _, secret := range service.Secrets {
 			enabledSecrets = append(enabledSecrets, secret.Source)
 		}
@@ -111,8 +95,8 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 		return nil, nil
 	}
 
-	secretValues := map[string]Secret{}
-	for secretId, secretConfig := range composeConfig.Secrets {
+	secretValues := map[string]SecretValue{}
+	for secretId, secretConfig := range config.Secrets {
 		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
 			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
 		}
@@ -124,23 +108,15 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 
 		lastIdx := strings.LastIndex(secretConfig.Name, "_")
 		secretVersion := secretConfig.Name[lastIdx+1:]
-		value := Secret{Version: secretVersion, RemoteName: secretConfig.Name}
-
-		if len(value.RemoteName) > config.MAX_DOCKER_SECRET_LENGTH {
-			return nil, fmt.Errorf("secret %s is > %d chars when combined with %s", secretId, config.MAX_DOCKER_SECRET_LENGTH, stackName)
-		}
+		value := SecretValue{Version: secretVersion}
 
 		// Check if the length modifier is set for this secret.
-		for envName, modifierValues := range appModifiers {
+		for k, v := range appModifiers {
 			// configWithoutEnv contains the raw name as defined in the compose.yaml
-			// The name will look something like this:
-			//   name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
-			// To check if the current modifier is for the current secret we check
-			// if the raw name contains the env name (e.g. SECRET_TEST_PASS_TWO_VERSION).
-			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, envName) {
+			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
 				continue
 			}
-			lengthRaw, ok := modifierValues["length"]
+			lengthRaw, ok := v["length"]
 			if ok {
 				length, err := strconv.Atoi(lengthRaw)
 				if err != nil {
@@ -157,7 +133,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server string) (map[string]string, error) {
+func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
 	secretsGenerated := map[string]string{}
 	var mutex sync.Mutex
 	var wg sync.WaitGroup
@@ -165,10 +141,11 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 	for n, v := range secrets {
 		wg.Add(1)
 
-		go func(secretName string, secret Secret) {
+		go func(secretName string, secret SecretValue) {
 			defer wg.Done()
 
-			logrus.Debugf("attempting to generate and store %s on %s", secret.RemoteName, server)
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
+			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
 
 			if secret.Length > 0 {
 				passwords, err := GeneratePasswords(1, uint(secret.Length))
@@ -177,9 +154,9 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 					return
 				}
 
-				if err := client.StoreSecret(cl, secret.RemoteName, passwords[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secret.RemoteName)
+						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil
 					} else {
 						ch <- err
@@ -197,9 +174,9 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server
 					return
 				}
 
-				if err := client.StoreSecret(cl, secret.RemoteName, passphrases[0], server); err != nil {
+				if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secret.RemoteName)
+						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
 						ch <- nil
 					} else {
 						ch <- err
@@ -248,7 +225,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		return secStats, err
 	}
 
-	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.StackName())
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 	if err != nil {
 		return secStats, err
 	}

pkg/secret/secret_test.go

@@ -1,39 +1,42 @@
 package secret
 
 import (
+	"path"
 	"testing"
 
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestReadSecretsConfig(t *testing.T) {
-	composeFiles := []string{"./testdir/compose.yaml"}
-	secretsFromConfig, err := ReadSecretsConfig("./testdir/.env.sample", composeFiles, "test_example_com")
+	offline := true
+	recipe, err := recipe.Get("matrix-synapse", offline)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Simple secret
-	assert.Equal(t, "test_example_com_test_pass_one_v2", secretsFromConfig["test_pass_one"].RemoteName)
-	assert.Equal(t, "v2", secretsFromConfig["test_pass_one"].Version)
-	assert.Equal(t, 0, secretsFromConfig["test_pass_one"].Length)
-
-	// Has a length modifier
-	assert.Equal(t, "test_example_com_test_pass_two_v1", secretsFromConfig["test_pass_two"].RemoteName)
-	assert.Equal(t, "v1", secretsFromConfig["test_pass_two"].Version)
-	assert.Equal(t, 10, secretsFromConfig["test_pass_two"].Length)
-
-	// Secret name does not include the secret id
-	assert.Equal(t, "test_example_com_pass_three_v2", secretsFromConfig["test_pass_three"].RemoteName)
-	assert.Equal(t, "v2", secretsFromConfig["test_pass_three"].Version)
-	assert.Equal(t, 0, secretsFromConfig["test_pass_three"].Length)
-}
-
-func TestReadSecretsConfigWithLongDomain(t *testing.T) {
-	composeFiles := []string{"./testdir/compose.yaml"}
-	_, err := ReadSecretsConfig("./testdir/.env.sample", composeFiles, "should_break_on_forty_eight_char_stack_nameeeeee")
-	if err == nil {
-		t.Fatal("expected failure, stack name is too long")
+	sampleEnv, err := recipe.SampleEnv()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	config, err := loader.LoadComposefile(opts, sampleEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for secretId := range config.Secrets {
+		assert.Contains(t, secretsFromConfig, secretId)
 	}
-	assert.Contains(t, err.Error(), "is > 64 chars")
 }

pkg/secret/testdir/.env.sample

@@ -1,3 +0,0 @@
-SECRET_TEST_PASS_ONE_VERSION=v2
-SECRET_TEST_PASS_TWO_VERSION=v1 # length=10
-SECRET_TEST_PASS_THREE_VERSION=v2

pkg/secret/testdir/compose.yaml

@@ -1,21 +0,0 @@
----
-version: "3.8"
-
-services:
-  app:
-    image: nginx:1.21.0
-    secrets:
-      - test_pass_one
-      - test_pass_two
-      - test_pass_three
-
-secrets:
-  test_pass_one:
-    external: true
-    name: ${STACK_NAME}_test_pass_one_${SECRET_TEST_PASS_ONE_VERSION}  # should be removed
-  test_pass_two:
-    external: true
-    name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
-  test_pass_three:
-    external: true
-    name: ${STACK_NAME}_pass_three_${SECRET_TEST_PASS_THREE_VERSION} # secretId and name don't match

pkg/service/service.go

@@ -14,70 +14,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// GetService retrieves a service container based on a label. If prompt is true
-// and the retrievd count of service containers does not match 1, then a prompt
-// is presented to let the user choose. An error is returned when no service is
-// found.
-func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
-	services, err := cl.ServiceList(c, types.ServiceListOptions{})
-	if err != nil {
-		return swarm.Service{}, err
-	}
-
-	if len(services) == 0 {
-		return swarm.Service{}, fmt.Errorf("no services deployed?")
-	}
-
-	var matchingServices []swarm.Service
-	for _, service := range services {
-		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
-			matchingServices = append(matchingServices, service)
-		}
-	}
-
-	if len(matchingServices) == 0 {
-		return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label)
-	}
-
-	if len(matchingServices) > 1 {
-		var servicesRaw []string
-		for _, service := range matchingServices {
-			serviceName := service.Spec.Name
-			created := formatter.HumanDuration(service.CreatedAt.Unix())
-			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
-		}
-
-		if !prompt {
-			err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " "))
-			return swarm.Service{}, err
-		}
-
-		logrus.Warnf("ambiguous service list received, prompting for input")
-
-		var response string
-		prompt := &survey.Select{
-			Message: "which service are you looking for?",
-			Options: servicesRaw,
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return swarm.Service{}, err
-		}
-
-		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
-		for _, service := range matchingServices {
-			serviceName := strings.ToLower(service.Spec.Name)
-			if serviceName == chosenService {
-				return service, nil
-			}
-		}
-
-		logrus.Panic("failed to match chosen service")
-	}
-
-	return matchingServices[0], nil
-}
-
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.

pkg/ssh/ssh.go

@@ -2,14 +2,73 @@ package ssh
 
 import (
 	"fmt"
+	"os/exec"
 	"strings"
+
+	"github.com/sirupsen/logrus"
 )
 
+// HostConfig is a SSH host config.
+type HostConfig struct {
+	Host         string
+	IdentityFile string
+	Port         string
+	User         string
+}
+
+// String presents a human friendly output for the HostConfig.
+func (h HostConfig) String() string {
+	return fmt.Sprintf(
+		"{host: %s, username: %s, port: %s, identityfile: %s}",
+		h.Host,
+		h.User,
+		h.Port,
+		h.IdentityFile,
+	)
+}
+
+// GetHostConfig retrieves a ~/.ssh/config config for a host using /usr/bin/ssh
+// directly. We therefore maintain consistent interop with this standard
+// tooling. This is useful because SSH confuses a lot of people and having to
+// learn how two tools (`ssh` and `abra`) handle SSH connection details instead
+// of one (just `ssh`) is Not Cool. Here's to less bug reports on this topic!
+func GetHostConfig(hostname string) (HostConfig, error) {
+	var hostConfig HostConfig
+
+	out, err := exec.Command("ssh", "-G", hostname).Output()
+	if err != nil {
+		return hostConfig, err
+	}
+
+	for _, line := range strings.Split(string(out), "\n") {
+		entries := strings.Split(line, " ")
+		for idx, entry := range entries {
+			if entry == "hostname" {
+				hostConfig.Host = entries[idx+1]
+			}
+			if entry == "user" {
+				hostConfig.User = entries[idx+1]
+			}
+			if entry == "port" {
+				hostConfig.Port = entries[idx+1]
+			}
+			if entry == "identityfile" {
+				if hostConfig.IdentityFile == "" {
+					hostConfig.IdentityFile = entries[idx+1]
+				}
+			}
+		}
+	}
+
+	logrus.Debugf("retrieved ssh config for %s: %s", hostname, hostConfig.String())
+
+	return hostConfig, nil
+}
+
 // Fatal is a error output wrapper which aims to make SSH failures easier to
 // parse through re-wording.
 func Fatal(hostname string, err error) error {
 	out := err.Error()
-
 	if strings.Contains(out, "Host key verification failed.") {
 		return fmt.Errorf("SSH host key verification failed for %s", hostname)
 	} else if strings.Contains(out, "Could not resolve hostname") {
@@ -20,7 +79,7 @@ func Fatal(hostname string, err error) error {
 		return fmt.Errorf("ssh auth: permission denied for %s", hostname)
 	} else if strings.Contains(out, "Network is unreachable") {
 		return fmt.Errorf("unable to connect to %s, network is unreachable?", hostname)
-	}
-
+	} else {
 		return err
+	}
 }

pkg/upstream/commandconn/connection.go

@@ -16,12 +16,12 @@ import (
 // GetConnectionHelper returns Docker-specific connection helper for the given URL.
 // GetConnectionHelper returns nil without error when no helper is registered for the scheme.
 //
-// ssh://<host> URL requires Docker 18.09 or later on the remote host.
+// ssh://<user>@<host> URL requires Docker 18.09 or later on the remote host.
 func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
-	return getConnectionHelper(daemonURL)
+	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=5"})
 }
 
-func getConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
+func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) {
 	url, err := url.Parse(daemonURL)
 	if err != nil {
 		return nil, err
@@ -35,7 +35,7 @@ func getConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error)
 
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return New(ctx, "ssh", ctxConnDetails.Args("docker", "system", "dial-stdio")...)
+				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)
 			},
 			Host: "http://docker.example.com",
 		}, nil
@@ -45,7 +45,6 @@ func getConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error)
 	return nil, err
 }
 
-// NewConnectionHelper creates new connection helper for a remote docker daemon.
 func NewConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
 	helper, err := GetConnectionHelper(daemonURL)
 	if err != nil {
@@ -74,7 +73,6 @@ func getDockerEndpoint(host string) (docker.Endpoint, error) {
 	return ep, nil
 }
 
-// GetDockerEndpointMetadataAndTLS retrieves the docker endpoint and TLS info for a remote host.
 func GetDockerEndpointMetadataAndTLS(host string) (docker.EndpointMeta, *dCliContextStore.EndpointTLSData, error) {
 	ep, err := getDockerEndpoint(host)
 	if err != nil {

pkg/upstream/container/exec.go

@@ -13,10 +13,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// RunExec runs a command on a remote container. io.Writer corresponds to the
-// command output.
-func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string,
-	execConfig *types.ExecConfig) (io.Writer, error) {
+func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error {
 	ctx := context.Background()
 
 	// We need to check the tty _before_ we do the ContainerExecCreate, because
@@ -24,22 +21,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 	// there's no easy way to clean those up). But also in order to make "not
 	// exist" errors take precedence we do a dummy inspect first.
 	if _, err := client.ContainerInspect(ctx, containerID); err != nil {
-		return nil, err
+		return err
 	}
 	if !execConfig.Detach {
 		if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	response, err := client.ContainerExecCreate(ctx, containerID, *execConfig)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	execID := response.ID
 	if execID == "" {
-		return nil, errors.New("exec ID empty")
+		return errors.New("exec ID empty")
 	}
 
 	if execConfig.Detach {
@@ -47,13 +44,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 			Detach: execConfig.Detach,
 			Tty:    execConfig.Tty,
 		}
-		return nil, client.ContainerExecStart(ctx, execID, execStartCheck)
+		return client.ContainerExecStart(ctx, execID, execStartCheck)
 	}
 	return interactiveExec(ctx, dockerCli, client, execConfig, execID)
 }
 
 func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client,
-	execConfig *types.ExecConfig, execID string) (io.Writer, error) {
+	execConfig *types.ExecConfig, execID string) error {
 	// Interactive exec requested.
 	var (
 		out, stderr io.Writer
@@ -79,7 +76,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 	}
 	resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck)
 	if err != nil {
-		return out, err
+		return err
 	}
 	defer resp.Close()
 
@@ -110,10 +107,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 
 	if err := <-errCh; err != nil {
 		logrus.Debugf("Error hijack: %s", err)
-		return out, err
+		return err
 	}
 
-	return out, getExecExitStatus(ctx, client, execID)
+	return getExecExitStatus(ctx, client, execID)
 }
 
 func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {

pkg/upstream/container/tty.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
 	apiclient "github.com/docker/docker/client"
 	"github.com/moby/sys/signal"
@@ -22,7 +22,7 @@ func resizeTtyTo(ctx context.Context, client client.ContainerAPIClient, id strin
 		return nil
 	}
 
-	options := container.ResizeOptions{
+	options := types.ResizeOptions{
 		Height: height,
 		Width:  width,
 	}

pkg/upstream/stack/stack.go

@@ -233,7 +233,7 @@ func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAP
 		network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
 		switch {
 		case dockerClient.IsErrNotFound(err):
-			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed, which you can do by running this on the server: docker network create -d overlay proxy", networkName)
+			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
 		case err != nil:
 			return err
 		case network.Scope != "swarm":

scripts/installer/installer

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.9.0-beta"
+ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.0-rc1-beta"
+RC_VERSION="0.8.1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do
@@ -45,9 +45,7 @@ function install_abra_release {
   fi
 
   ARCH=$(uname -m)
-  if [[ $ARCH =~ "x86_64" ]]; then
-      ARCH="amd64"
-  elif [[ $ARCH =~ "aarch64" ]]; then
+  if [[ $ARCH =~ "aarch64" ]]; then
       ARCH="arm64"
   elif [[ $ARCH =~ "armv5l" ]]; then
       ARCH="armv5"
@@ -57,7 +55,7 @@ function install_abra_release {
       ARCH="armv7"
   fi
   PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')_$ARCH
-  FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM".tar.gz"
+  FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM""
   sed_command_rel='s/.*"assets":\[\{[^]]*"name":"'$FILENAME'"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
   sed_command_checksums='s/.*"assets":\[\{[^\]*"name":"checksums.txt"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
 
@@ -67,22 +65,17 @@ function install_abra_release {
 
   checksums=$(wget -q -O- $checksums_url)
   checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p')
-  abra_download="/tmp/abra-download.tar.gz"
 
   echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..."
-
-  wget -q "$release_url" -O $abra_download 
-  localsum=$(sha256sum $abra_download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
+  wget -q "$release_url" -O "$HOME/.local/bin/.abra-download"
+  localsum=$(sha256sum $HOME/.local/bin/.abra-download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
   echo "checking if checksums match..."
   if [[ "$localsum" != "$checksum" ]]; then
       print_checksum_error
       exit 1
   fi
   echo "$(tput setaf 2)check successful!$(tput sgr0)"
-  cd /tmp/
-  tar xf abra-download.tar.gz
-  mv abra "$HOME/.local/bin/abra"
-  tar tf abra-download.tar.gz | xargs rm -f
+  mv "$HOME/.local/bin/.abra-download" "$HOME/.local/bin/abra"
   chmod +x "$HOME/.local/bin/abra"
 
   x=$(echo $PATH | grep $HOME/.local/bin)

tests/integration/app_check.bats

@@ -70,13 +70,13 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }
@@ -86,7 +86,7 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
+  assert_output --partial 'behind 1'
 
   # NOTE(d1): we can't quite tell if this will fail or not in the future, so,
   # since it isn't an important part of what we're testing here, we don't check
@@ -94,7 +94,7 @@ setup(){
   run $ABRA app check "$TEST_APP_DOMAIN" --offline
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
+  assert_output --partial 'behind 1'
 
   _reset_recipe
 }

tests/integration/app_cmd.bats

@@ -58,7 +58,7 @@ test_cmd_export"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_success
   assert_output --partial 'baz'
 
@@ -70,7 +70,7 @@ test_cmd_export"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_failure
   assert_output --partial 'locally unstaged changes'
 
@@ -83,7 +83,7 @@ test_cmd_export"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
   assert_success
   assert_output --partial 'baz'
 
@@ -96,14 +96,14 @@ test_cmd_export"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_success
   assert_output --partial 'baz'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -113,14 +113,14 @@ test_cmd_export"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app cmd --local --offline "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --offline
   assert_success
   assert_output --partial 'baz'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -132,13 +132,13 @@ test_cmd_export"
 }
 
 @test "error if missing arguments when passing --local" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN"
+  run $ABRA app cmd "$TEST_APP_DOMAIN" --local
   assert_failure
   assert_output --partial 'missing arguments'
 }
 
 @test "cannot use --local and --user at same time" {
-  run $ABRA app cmd --local --user root "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --user root
   assert_failure
   assert_output --partial 'cannot use --local & --user together'
 }
@@ -147,7 +147,7 @@ test_cmd_export"
   run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh"
   assert_success
 
-  run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
   assert_failure
   assert_output --partial "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh does not exist"
 
@@ -155,25 +155,25 @@ test_cmd_export"
 }
 
 @test "error if missing command" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" doesnt_exist
+  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist --local
   assert_failure
   assert_output --partial "doesn't have a doesnt_exist function"
 }
 
 @test "run --local command" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
   assert_success
   assert_output --partial 'baz'
 }
 
 @test "run command with single arg" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_arg -- bing
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_arg --local -- bing
   assert_success
   assert_output --partial 'bing'
 }
 
 @test "run command with several args" {
-  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_args -- bong bang
+  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_args --local -- bong bang
   assert_success
   assert_output --partial 'bong bang'
 }

tests/integration/app_cp.bats

@@ -5,11 +5,9 @@ setup_file(){
   _common_setup
   _add_server
   _new_app
-  _deploy_app
 }
 
 teardown_file(){
-  _undeploy_app
   _rm_app
   _rm_server
 }
@@ -19,6 +17,13 @@ setup(){
   _common_setup
 }
 
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
 @test "validate app argument" {
   run $ABRA app cp
   assert_failure
@@ -49,120 +54,68 @@ setup(){
   assert_output --partial 'arguments must take $SERVICE:$PATH form'
 }
 
-@test "error if local file missing" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" thisfileshouldnotexist.txt app:/somewhere
+@test "detect 'coming FROM' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug
   assert_failure
-  assert_output --partial 'local stat thisfileshouldnotexist.txt: no such file or directory'
+  assert_output --partial 'coming FROM the container'
+}
+
+@test "detect 'going TO' syntax" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug
+  assert_failure
+  assert_output --partial 'going TO the container'
+}
+
+@test "error if local file missing" {
+  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere
+  assert_failure
+  assert_output --partial 'myfile.txt does not exist locally?'
 }
 
 # bats test_tags=slow
 @test "error if service doesn't exist" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+  _deploy_app
 
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ --debug
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/
   assert_failure
   assert_output --partial 'no containers matching'
 
-  _rm "$BATS_TMPDIR/myfile.txt"
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
 }
 
 # bats test_tags=slow
-@test "copy local file to container directory" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+@test "copy to container" {
+  _deploy_app
+
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  assert_success
 
   run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
   assert_success
 
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
   assert_success
-  assert_output --partial "foo"
 
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
+  _undeploy_app
 }
 
 # bats test_tags=slow
-@test "copy local file to container file (and override on remote)" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+@test "copy from container" {
+  _deploy_app
 
-  # create
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
+  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
   assert_success
 
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
-  assert_success
-  assert_output --partial "foo"
-
-  _mkfile "$BATS_TMPDIR/myfile.txt" "bar"
-
-  # override
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
   assert_success
 
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
-  assert_success
-  assert_output --partial "bar"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy local file to container file (and rename)" {
-  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
-
-  # rename
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile2.txt
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile2.txt
-  assert_success
-  assert_output --partial "foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
-}
-
-# bats test_tags=slow
-@test "copy local directory to container directory (and creates missing directory)" {
-  _mkdir "$BATS_TMPDIR/mydir"
-  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir" app:/etc
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/mydir
-  assert_success
-  assert_output --partial "myfile.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/mydir"
-}
-
-# bats test_tags=slow
-@test "copy local files to container directory" {
-  _mkdir "$BATS_TMPDIR/mydir"
-  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
-  _mkfile "$BATS_TMPDIR/mydir/myfile2.txt" "foo"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir/" app:/etc
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile.txt
-  assert_success
-  assert_output --partial "myfile.txt"
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile2.txt
-  assert_success
-  assert_output --partial "myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile*"
-}
-
-# bats test_tags=slow
-@test "copy container file to local directory" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
   assert_success
 
   run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR"
@@ -170,76 +123,8 @@ setup(){
   assert_exists "$BATS_TMPDIR/myfile.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
 
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy container file to local file" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/myfile.txt"
-  assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy container file to local file and rename" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile2.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/myfile2.txt"
-  assert bash -c "cat $BATS_TMPDIR/myfile2.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile2.txt"
-  _rm_remote "/etc/myfile.txt"
-}
-
-# bats test_tags=slow
-@test "copy container directory to local directory" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
-  assert_success
-
-  mkdir "$BATS_TMPDIR/mydir"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc "$BATS_TMPDIR/mydir"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/etc/myfile.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/etc/myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
-}
-
-# bats test_tags=slow
-@test "copy container files to local directory" {
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
-  assert_success
-
-  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
-  assert_success
-
-  mkdir "$BATS_TMPDIR/mydir"
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/ "$BATS_TMPDIR/mydir"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/myfile.txt"
-  assert_success
-  assert_exists "$BATS_TMPDIR/mydir/myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
+  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  assert_success
+
+  _undeploy_app
 }

tests/integration/app_deploy.bats

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 setup_file(){
-  load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/common"
   _common_setup
   _add_server
@@ -17,7 +16,6 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-  _reset_recipe
 }
 
 teardown(){
@@ -84,13 +82,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
   _undeploy_app
@@ -102,7 +100,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
@@ -110,7 +108,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _undeploy_app
   _reset_recipe
@@ -118,9 +116,6 @@ teardown(){
 
 # bats test_tags=slow
 @test "deploy latest commit if no published versions and no --chaos" {
-  # TODO(d1): fix with a new test recipe which has no published versions?
-  skip "known issue, abra-test-recipe has published versions now"
-
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   _remove_tags
@@ -145,7 +140,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
@@ -278,10 +273,6 @@ teardown(){
 }
 
 @test "ensure domain is checked" {
-  if [[ "$TEST_SERVER" == "default" ]]; then
-      skip "domain checks are disabled for local server"
-  fi
-
   appDomain="custom-html.DOESNTEXIST"
 
   run $ABRA app new custom-html \
@@ -363,7 +354,6 @@ teardown(){
   _reset_app
 }
 
-# bats test_tags=slow
 @test "recipe config comments not present in values" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
   assert_success
@@ -372,36 +362,3 @@ teardown(){
   assert_success
   refute_output --partial 'should be removed'
 }
-
-# bats test_tags=slow
-@test "deploy specific version with incompatible HEAD" {
-  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \
-    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-  assert_success
-
-  run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \
-    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-  assert_success
-
-  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
-  assert_success
-  assert_output --partial 'extra_pass'
-
-  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/compose.extra_secret.yml"
-  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/compose.extra_secret.yml"
-
-  _git_commit
-
-  # NOTE(d1): 0.1.1+1.20.2 is a previous version which includes compose.extra_secret.yml
-  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
-  assert_success
-  refute_output --partial 'no such file or directory'
-
-  _undeploy_app
-  _reset_app
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
-
-  _reset_recipe
-}

tests/integration/app_new.bats

@@ -13,22 +13,14 @@ teardown_file(){
 
 setup(){
   load "$PWD/tests/integration/helpers/common"
-  load "$PWD/tests/integration/helpers/git"
   _common_setup
   _fetch_recipe
 }
 
 teardown(){
-  load "$PWD/tests/integration/helpers/common"
   _rm_app
 }
 
-@test "autocomplete" {
-  run $ABRA app new --generate-bash-completion
-  assert_success
-  assert_output --partial "traefik"
-}
-
 @test "create new app" {
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -36,29 +28,10 @@ teardown(){
     --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  _get_head_hash
-  _get_current_hash
-  assert_equal "$headHash" "$currentHash"
-}
-
-@test "create new app with version" {
-  run $ABRA app new "$TEST_RECIPE" 0.1.1+1.20.2 \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
-  assert_success
-  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  assert_equal $(_get_tag_hash 0.1.1+1.20.2) $(_get_current_hash)
 }
 
 @test "does not overwrite existing env files" {
-  run $ABRA app new "$TEST_RECIPE" \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
-  assert_success
+  _new_app
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -101,7 +74,8 @@ teardown(){
     --no-input \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
@@ -110,34 +84,32 @@ teardown(){
 }
 
 @test "ensure recipe up to date if no --offline" {
-  _reset_recipe
-  wantHash=$(_get_n_hash 3)
-
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  assert_equal $(_get_current_hash) "$wantHash"
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  assert_equal $(_get_head_hash) $(_get_current_hash)
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }
 
 @test "ensure recipe not up to date if --offline" {
-  _reset_recipe
-  wantHash=$(_get_n_hash 3)
-
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  assert_equal $(_get_current_hash) "$wantHash"
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app new "$TEST_RECIPE" \
@@ -145,16 +117,17 @@ teardown(){
     --offline \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
+    --secrets
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  assert_equal $(_get_current_hash) "$wantHash"
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial 'behind 3'
 
   _reset_recipe
 }
 
-# bats test_tags=slow
 @test "generate secrets" {
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \

tests/integration/app_remove.bats

@@ -104,7 +104,10 @@ teardown(){
 
   _undeploy_app
 
-  run $ABRA app volume rm "$TEST_APP_DOMAIN" --no-input
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
 
   run $ABRA app volume ls "$TEST_APP_DOMAIN"
@@ -129,6 +132,9 @@ teardown(){
 
   _undeploy_app
 
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
   assert_output --partial 'test-volume'

tests/integration/app_restore.bats

@@ -109,13 +109,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app restore "$TEST_APP_DOMAIN" app
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 }
 
 @test "ensure recipe not up to date if --offline" {
@@ -126,19 +126,19 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
-  run $ABRA app restore "$TEST_APP_DOMAIN" app --offline
+  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST --offline
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "HEAD detached at $latestCommit"
+  refute_output --partial 'behind 3'
 }
 
 @test "error if missing service" {

tests/integration/app_rollback.bats

@@ -50,13 +50,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  refute_output --partial 'behind 3'
 }
 
 @test "ensure recipe not up to date if --offline" {
@@ -67,14 +67,14 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" \
     --no-input --no-converge-checks --offline
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
   assert_success
@@ -131,7 +131,7 @@ teardown(){
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
-    --no-input --chaos
+    --no-input --no-converge-checks --chaos
   assert_success
   assert_output --partial "$latestCommit"
   assert_output --partial 'chaos'

tests/integration/app_secret.bats

@@ -8,7 +8,7 @@ setup_file(){
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 }
@@ -19,6 +19,13 @@ teardown_file(){
   _reset_recipe
 }
 
+teardown(){
+  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
+  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
+    _undeploy_app
+  fi
+}
+
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup

tests/integration/app_version.bats

@@ -59,8 +59,6 @@ teardown(){
 
 # bats test_tags=slow
 @test "error if not in catalogue" {
-  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
-
   _deploy_app
 
   run $ABRA app version "$TEST_APP_DOMAIN"
@@ -94,7 +92,7 @@ teardown(){
   assert_success
 
  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
+  sleep 3
 
   run $ABRA app volume remove "$appDomain" --no-input
   assert_success

tests/integration/app_volume.bats

@@ -78,6 +78,9 @@ teardown(){
 
   _undeploy_app
 
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'
@@ -89,6 +92,9 @@ teardown(){
 
   _undeploy_app
 
+  # NOTE(d1): to let the stack come down before nuking volumes
+  sleep 5
+
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'

tests/integration/helpers/app.bash

@@ -49,7 +49,7 @@ _reset_app(){
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
+    --domain "$TEST_APP_DOMAIN" \
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 }

tests/integration/helpers/common.bash

@@ -5,7 +5,6 @@ _common_setup() {
   bats_load_library bats-assert
   bats_load_library bats-file
 
-  load "$PWD/tests/integration/helpers/file"
   load "$PWD/tests/integration/helpers/app"
   load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/recipe"

tests/integration/helpers/file.bash

@@ -1,24 +0,0 @@
-_mkfile() {
-  run bash -c "echo $2 > $1"
-  assert_success
-}
-
-_mkfile_remote() {
-  run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\""
-  assert_success
-}
-
-_mkdir() {
-  run bash -c "mkdir -p $1"
-  assert_success
-}
-
-_rm() {
-  run rm -rf "$1"
-  assert_success
-}
-
-_rm_remote() {
-  run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1"
-  assert_success
-}

tests/integration/helpers/git.bash

@@ -28,43 +28,3 @@ _reset_tags() {
   assert_success
   refute_output '0'
 }
-
-_set_git_author() {
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.email test@example.com
-  assert_success
-
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.name test
-  assert_success
-}
-
-_git_commit() {
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add .
-  assert_success
-
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "test: helpers/git.bash: _git_commit"
-  assert_success
-}
-
-_get_tag_hash() {
-  tagHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-list -n 1 "$1")
-  assert_success
-  echo "$tagHash"
-}
-
-_get_head_hash() {
-  headHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" HEAD)
-  assert_success
-  echo "$headHash"
-}
-
-_get_current_hash() {
-  currentHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H")
-  assert_success
-  echo "$currentHash"
-}
-
-_get_n_hash() {
-  nHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" "HEAD~$1")
-  assert_success
-  echo "$nHash"
-}

tests/integration/helpers/server.bash

@@ -11,11 +11,7 @@ _add_server() {
 }
 
 _rm_server() {
-  if [[ "$TEST_SERVER" == "default" ]]; then
-      run rm -rf "$ABRA_DIR/servers/default"
-  else
   run $ABRA server remove --no-input "$TEST_SERVER"
-  fi
   assert_success
   assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER"
 }

tests/integration/recipe_fetch.bats

@@ -5,17 +5,7 @@ setup() {
   _common_setup
 }
 
-@test "recipe fetch all" {
-  run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
-  assert_success
-  assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"
-
-  run $ABRA recipe fetch
-  assert_success
-  assert_exists "$ABRA_DIR/recipes/matrix-synapse"
-}
-
-@test "recipe fetch single recipe" {
+@test "recipe fetch" {
   run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"

tests/integration/recipe_lint.bats

@@ -66,13 +66,13 @@ setup() {
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA recipe lint "$TEST_RECIPE"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }
@@ -82,13 +82,13 @@ setup() {
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA recipe lint "$TEST_RECIPE" --offline
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   _reset_recipe
 }

tests/integration/recipe_release.bats

@@ -15,11 +15,6 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-  _set_git_author
-}
-
-teardown() {
-  _reset_recipe
 }
 
 @test "validate recipe argument" {
@@ -56,6 +51,8 @@ teardown() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
   assert_output --partial '0.2.1+1.21.6'
+
+  _reset_recipe
 }
 
 # NOTE(d1): this test can't assert hardcoded versions since we upgrade a minor
@@ -84,6 +81,8 @@ teardown() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
   assert_output --regexp '0\.3\.0\+1\.2.*'
+
+  _reset_recipe "$TEST_RECIPE"
 }
 
 @test "unknown files not committed" {
@@ -101,21 +100,6 @@ teardown() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo
   assert_failure
   assert_output --partial "fatal: pathspec 'foo' did not match any files"
-}
-
-# NOTE: relies on 0.2.x being the last minor version
-@test "release with next release note" {
-  _mkfile "$ABRA_DIR/recipes/$TEST_RECIPE/release/next" "those are some release notes for the next release"
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add release/next
-  assert_success
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "added some release notes"
-  assert_success
-
-  run $ABRA recipe release  "$TEST_RECIPE" --no-input  --minor
-  assert_success
-  assert_output --partial 'no -p/--publish passed, not publishing'
-
-  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/next"
-  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0"
-  assert_file_contains "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0" "those are some release notes for the next release"
+
+  _reset_recipe
 }

tests/integration/recipe_upgrade.bats

@@ -61,14 +61,14 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_output --partial 'behind 3'
 
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input
   assert_success
   assert_output --partial 'can upgrade service: app'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  refute_output --partial 'behind 3'
 
   _reset_recipe
 }

tests/integration/recipe_version.bats

@@ -12,8 +12,6 @@ setup() {
 }
 
 @test "error if not present in catalogue" {
-  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
-
   run $ABRA recipe versions "$TEST_RECIPE"
   assert_failure
   assert_output --partial "is not published on the catalogue"