WIP: backup revolution [ci skip]

See coop-cloud/organising#485

Makefile

@@ -30,11 +30,6 @@ build-kadabra:
 
 build: build-abra build-kadabra
 
-build-docker-abra:
-	docker run -it -v $(PWD):/abra golang:1.21 bash -c 'cd /abra; ./build-docker-inside.sh'
-
-build-docker: build-docker-abra
-
 clean:
 	@rm '$(GOPATH)/bin/abra'
 	@rm '$(GOPATH)/bin/kadabra'

build-docker-inside.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-if [ ! -f .envrc ]; then
-    . .envrc.sample
-else
-    . .envrc
-fi
-git config --global --add safe.directory /abra  # work around funky file permissions
-
-# fixme for some reason we need to do this
-go get coopcloud.tech/abra/pkg/upstream/commandconn
-go get github.com/sirupsen/logrus@v1.9.3
-go get github.com/cloudflare/circl/dh/x25519@v1.3.3
-go get github.com/mattn/go-runewidth@v0.0.14
-go get github.com/mattn/go-colorable@v0.1.12
-go get coopcloud.tech/abra/pkg/config
-#
-
-make build

cli/app/backup.go

@@ -1,414 +1,422 @@
 package app
 
 import (
-	"archive/tar"
 	"context"
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type backupConfig struct {
-	preHookCmd  string
-	postHookCmd string
-	backupPaths []string
+var snapshot string
+var snapshotFlag = &cli.StringFlag{
+	Name:        "snapshot, s",
+	Usage:       "Lists specific snapshot",
+	Destination: &snapshot,
 }
 
-var appBackupCommand = cli.Command{
-	Name:      "backup",
-	Aliases:   []string{"bk"},
-	Usage:     "Run app backup",
-	ArgsUsage: "<domain> [<service>]",
+var includePath string
+var includePathFlag = &cli.StringFlag{
+	Name:        "path, p",
+	Usage:       "Include path",
+	Destination: &includePath,
+}
+
+var resticRepo string
+var resticRepoFlag = &cli.StringFlag{
+	Name:        "repo, r",
+	Usage:       "Restic repository",
+	Destination: &resticRepo,
+}
+
+var appBackupListCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		snapshotFlag,
+		includePathFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	Usage:        "List all backups",
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app backup.
-
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		backupConfigs := make(map[string]backupConfig)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName != "" {
-			backupConfig, ok := backupConfigs[serviceName]
-			if !ok {
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-			logrus.Infof("running backup for the %s service", serviceName)
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
 
-			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if len(backupConfigs) == 0 {
-				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
-			}
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
 
-				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "ls"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
 
-// TimeStamp generates a file name friendly timestamp.
-func TimeStamp() string {
-	ts := time.Now().UTC().Format(time.RFC3339)
-	return strings.Replace(ts, ":", "-", -1)
-}
+var appBackupDownloadCommand = cli.Command{
+	Name:    "download",
+	Aliases: []string{"d"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+		includePathFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Download a backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-// runBackup does the actual backup logic.
-func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-		}
-		defer content.Close()
-
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			logrus.Fatal(err)
 		}
 
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
 		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
-	}
 
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
 
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
 
-		postHookExecOpts := types.ExecConfig{
+		execBackupListOpts := types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
-			Cmd:          splitCmd,
+			Cmd:          []string{"/usr/bin/backup", "--", "download"},
 			Detach:       false,
+			Env:          execEnv,
 			Tty:          true,
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
 		}
 
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
-
-	return nil
-}
-
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
-
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
 		}
 
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
-
-	return nil
+		return nil
+	},
 }
 
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
+var appBackupCreateCommand = cli.Command{
+	Name:    "create",
+	Aliases: []string{"c"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		resticRepoFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Create a new backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
-	}
 
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
-		}
-	}
-
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
-
-	return nil
-}
-
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
-
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
-
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
 			}
-			break
 		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
 		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
+
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if resticRepo != "" {
+			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
+			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
+		}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "create"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
 }
 
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
+var appBackupSnapshotsCommand = cli.Command{
+	Name:    "snapshots",
+	Aliases: []string{"s"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "List backup snapshots",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 
-	rc = fin
-	tr = tar.NewReader(rc)
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
-	return tr, rc, nil
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "snapshots"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+var appBackupCommand = cli.Command{
+	Name:      "backup",
+	Aliases:   []string{"b"},
+	Usage:     "Manage app backups",
+	ArgsUsage: "<domain>",
+	Subcommands: []cli.Command{
+		appBackupListCommand,
+		appBackupSnapshotsCommand,
+		appBackupDownloadCommand,
+		appBackupCreateCommand,
+	},
 }

cli/app/cmd.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -23,7 +22,8 @@ var appCmdCommand = cli.Command{
 	Name:    "command",
 	Aliases: []string{"cmd"},
 	Usage:   "Run app commands",
-	Description: `Run an app specific command.
+	Description: `
+Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
@@ -43,8 +43,8 @@ Example:
 		internal.OfflineFlag,
 		internal.ChaosFlag,
 	},
-	Before:      internal.SubCommandBefore,
-	Subcommands: []cli.Command{appCmdListCommand},
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -186,53 +186,3 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 
 	return hasCmdArgs, parsedCmdArgs
 }
-
-var appCmdListCommand = cli.Command{
-	Name:      "list",
-	Aliases:   []string{"ls"},
-	Usage:     "List all available commands",
-	ArgsUsage: "<domain>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		internal.ChaosFlag,
-	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-		cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		sort.Strings(cmdNames)
-		for _, cmdName := range cmdNames {
-			fmt.Println(cmdName)
-		}
-
-		return nil
-	},
-}

cli/app/restore.go

@@ -2,9 +2,7 @@ package app
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -12,212 +10,112 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/container"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type restoreConfig struct {
-	preHookCmd  string
-	postHookCmd string
+var targetPath string
+var targetPathFlag = &cli.StringFlag{
+	Name:        "target, t",
+	Usage:       "Target path",
+	Destination: &targetPath,
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Run app restore",
-	ArgsUsage: "<domain> <service> <file>",
+	Usage:     "Restore an app backup",
+	ArgsUsage: "<domain> <service>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		targetPathFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app restore.
-
-Pre/post hook commands are defined in the recipe configuration. Abra reads this
-configuration and run the comands in the context of the service before
-restoring the backup.
-
-Unlike "abra app backup", restore must be run on a per-service basis. You can
-not restore all services in one go. Backup files produced by Abra are
-compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic, i.e. the backup will be restored to
-the path it was originally backed up from.
-
-Example:
-
-    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipe.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
-		}
-
-		backupPath := c.Args().Get(2)
-		if backupPath == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
-		}
-
-		if _, err := os.Stat(backupPath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s doesn't exist?", backupPath)
-			}
-		}
-
-		restoreConfigs := make(map[string]restoreConfig)
-		for _, service := range recipe.Config.Services {
-			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
-				if restoreEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					rsConfig := restoreConfig{}
-
-					logrus.Debugf("restore config detected for %s", fullServiceName)
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						rsConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						rsConfig.postHookCmd = postHookCmd
-					}
-
-					restoreConfigs[service.Name] = rsConfig
-				}
-			}
-		}
-
-		rsConfig, ok := restoreConfigs[serviceName]
-		if !ok {
-			rsConfig = restoreConfig{}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
+		chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+		filters := filters.NewArgs()
+		filters.Add("name", chosenService.Spec.Name)
+		targetContainer, err := containerPkg.GetContainer(
+			context.Background(),
+			cl,
+			filters,
+			internal.NoInput,
+		)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if targetPath != "" {
+			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
+			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
+		}
+		execBackupListOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"/usr/bin/backup", "--", "restore"},
+			Detach:       false,
+			Env:          execEnv,
+			Tty:          true,
+		}
+
+		logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
+
+		// FIXME: avoid instantiating a new CLI
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
-
-// runRestore does the actual restore logic.
-func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if rsConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
-	}
-
-	backupReader, err := os.Open(backupPath)
-	if err != nil {
-		return err
-	}
-
-	content, err := archive.DecompressStream(backupReader)
-	if err != nil {
-		return err
-	}
-
-	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
-	// files according to the paths set in the compressed archive
-	restorePath := "/"
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
-		return err
-	}
-
-	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
-
-	if rsConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
-	}
-
-	return nil
-}

cli/catalogue/catalogue.go

@@ -98,6 +98,11 @@ keys configured on your account.
 				continue
 			}
 
+			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
+				catlBar.Add(1)
+				continue
+			}
+
 			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
@@ -168,7 +173,7 @@ keys configured on your account.
 			}
 
 			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil {
+			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
 				logrus.Fatal(err)
 			}
 

cli/recipe/diff.go

@@ -1,40 +0,0 @@
-package recipe
-
-import (
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
-	gitPkg "coopcloud.tech/abra/pkg/git"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var recipeDiffCommand = cli.Command{
-	Name:        "diff",
-	Usage:       "Show unstaged changes in recipe config",
-	Description: "Due to limitations in our underlying Git dependency, this command requires /usr/bin/git.",
-	Aliases:     []string{"d"},
-	ArgsUsage:   "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-
-		if recipeName != "" {
-			internal.ValidateRecipe(c)
-		}
-
-		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-		if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}

cli/recipe/recipe.go

@@ -30,7 +30,5 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 		recipeSyncCommand,
 		recipeUpgradeCommand,
 		recipeVersionCommand,
-		recipeResetCommand,
-		recipeDiffCommand,
 	},
 }

cli/recipe/release.go

@@ -106,18 +106,6 @@ your SSH keys configured on your account.
 			}
 		}
 
-		isClean, err := gitPkg.IsClean(recipe.Dir())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isClean {
-			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
-			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		if len(tags) > 0 {
 			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
 			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
@@ -256,7 +244,7 @@ func commitRelease(recipe recipe.Recipe, tag string) error {
 
 	msg := fmt.Sprintf("chore: publish %s release", tag)
 	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, msg, internal.Dry); err != nil {
+	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
 		return err
 	}
 

cli/recipe/reset.go

@@ -1,56 +0,0 @@
-package recipe
-
-import (
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var recipeResetCommand = cli.Command{
-	Name:        "reset",
-	Usage:       "Remove all unstaged changes from recipe config",
-	Description: "WARNING, this will delete your changes. Be Careful.",
-	Aliases:     []string{"rs"},
-	ArgsUsage:   "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-
-		if recipeName != "" {
-			internal.ValidateRecipe(c)
-		}
-
-		repoPath := path.Join(config.RECIPES_DIR, recipeName)
-		repo, err := git.PlainOpen(repoPath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		ref, err := repo.Head()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		worktree, err := repo.Worktree()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}
-		if err := worktree.Reset(opts); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}

cli/recipe/sync.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -199,17 +198,6 @@ likely to change.
 			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
-		isClean, err := gitPkg.IsClean(recipe.Dir())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if !isClean {
-			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
-			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		return nil
 	},
 }

cli/recipe/upgrade.go

@@ -14,7 +14,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	gitPkg "coopcloud.tech/abra/pkg/git"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -327,7 +326,6 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 
 				fmt.Println(string(jsonstring))
-
 				return nil
 			}
 
@@ -338,18 +336,6 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 		}
-
-		isClean, err := gitPkg.IsClean(recipeDir)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if !isClean {
-			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
-			if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		return nil
 	},
 }

go.mod

@@ -7,16 +7,16 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v24.0.7+incompatible
+	github.com/docker/cli v24.0.6+incompatible
 	github.com/docker/distribution v2.8.3+incompatible
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v24.0.6+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.10.0
+	github.com/go-git/go-git/v5 v5.9.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.14.0
+	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )
@@ -78,11 +78,11 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.13.0 // indirect
@@ -116,5 +116,5 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/sys v0.14.0
+	golang.org/x/sys v0.13.0
 )

go.sum

@@ -339,16 +339,16 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
-github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
+github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
+github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -417,10 +417,10 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
-github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ=
-github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
+github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
+github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -1069,8 +1069,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1168,8 +1168,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1323,8 +1323,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/catalogue/catalogue.go

@@ -12,6 +12,46 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                        true,
+	"abra-apps":                   true,
+	"abra-aur":                    true,
+	"abra-bash":                   true,
+	"abra-capsul":                 true,
+	"abra-gandi":                  true,
+	"abra-hetzner":                true,
+	"abra-test-recipe":            true,
+	"apps":                        true,
+	"aur-abra-git":                true,
+	"auto-mirror":                 true,
+	"auto-recipes-catalogue-json": true,
+	"backup-bot":                  true,
+	"backup-bot-two":              true,
+	"beta.coopcloud.tech":         true,
+	"comrade-renovate-bot":        true,
+	"coopcloud.tech":              true,
+	"coturn":                      true,
+	"docker-cp-deploy":            true,
+	"docker-dind-bats-kcov":       true,
+	"docs.coopcloud.tech":         true,
+	"drone-abra":                  true,
+	"example":                     true,
+	"gardening":                   true,
+	"go-abra":                     true,
+	"organising":                  true,
+	"pyabra":                      true,
+	"radicle-seed-node":           true,
+	"recipes-catalogue-json":      true,
+	"recipes-wishlist":            true,
+	"recipes.coopcloud.tech":      true,
+	"stack-ssh-deploy":            true,
+	"swarm-cronjob":               true,
+	"tagcmp":                      true,
+	"traefik-cert-dumper":         true,
+	"tyop":                        true,
+}
+
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")

pkg/config/env.go

@@ -36,6 +36,8 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+var BackupbotLabel = "coop-cloud.backupbot.enabled"
+
 // envVarModifiers is a list of env var modifier strings. These are added to
 // env vars as comments and modify their processing by Abra, e.g. determining
 // how long secrets should be.
@@ -79,15 +81,15 @@ func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
 		return nil, err
 	}
 
-	//	for idx, envVar := range envVars {
-	//		if strings.Contains(envVar, "#") {
-	//			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
-	//				continue
-	//			}
-	//			vals := strings.Split(envVar, "#")
-	//			envVars[idx] = strings.TrimSpace(vals[0])
-	//		}
-	//	}
+	for idx, envVar := range envVars {
+		if strings.Contains(envVar, "#") {
+			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
+				continue
+			}
+			vals := strings.Split(envVar, "#")
+			envVars[idx] = strings.TrimSpace(vals[0])
+		}
+	}
 
 	logrus.Debugf("read %s from %s", envVars, filePath)
 
@@ -249,39 +251,3 @@ func CheckEnv(app App) ([]EnvVar, error) {
 
 	return envVars, nil
 }
-
-// ReadAbraShCmdNames reads the names of commands.
-func ReadAbraShCmdNames(abraSh string) ([]string, error) {
-	var cmdNames []string
-
-	file, err := os.Open(abraSh)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return cmdNames, nil
-		}
-		return cmdNames, err
-	}
-	defer file.Close()
-
-	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
-	if err != nil {
-		return cmdNames, err
-	}
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-		matches := cmdNameRegex.FindStringSubmatch(line)
-		if len(matches) > 0 {
-			cmdNames = append(cmdNames, matches[1])
-		}
-	}
-
-	if len(cmdNames) > 0 {
-		logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh)
-	} else {
-		logrus.Debugf("read 0 command names from %s", abraSh)
-	}
-
-	return cmdNames, nil
-}

pkg/config/env_test.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path"
 	"reflect"
-	"slices"
 	"strings"
 	"testing"
 
@@ -116,31 +115,6 @@ func TestReadAbraShEnvVars(t *testing.T) {
 	}
 }
 
-func TestReadAbraShCmdNames(t *testing.T) {
-	offline := true
-	r, err := recipe.Get("abra-test-recipe", offline)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
-	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(cmdNames) == 0 {
-		t.Error("at least one command name should be found")
-	}
-
-	expectedCmdNames := []string{"test_cmd", "test_cmd_args"}
-	for _, cmdName := range expectedCmdNames {
-		if !slices.Contains(cmdNames, cmdName) {
-			t.Fatalf("%s should have been found in %s", cmdName, abraShPath)
-		}
-	}
-}
-
 func TestCheckEnv(t *testing.T) {
 	offline := true
 	r, err := recipe.Get("abra-test-recipe", offline)

pkg/container/container.go

@@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) != 1 {
+	if len(containers) > 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")

pkg/git/commit.go

@@ -8,7 +8,7 @@ import (
 )
 
 // Commit runs a git commit
-func Commit(repoPath, commitMessage string, dryRun bool) error {
+func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
 	if commitMessage == "" {
 		return fmt.Errorf("no commit message specified?")
 	}
@@ -33,8 +33,17 @@ func Commit(repoPath, commitMessage string, dryRun bool) error {
 	}
 
 	if !dryRun {
-		// NOTE(d1): `All: true` does not include untracked files
-		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{All: true})
+		err = commitWorktree.AddGlob(glob)
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("staged %s for commit", glob)
+	} else {
+		logrus.Debugf("dry run: did not stage %s for commit", glob)
+	}
+
+	if !dryRun {
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
 		if err != nil {
 			return err
 		}

pkg/git/diff.go

@@ -1,42 +0,0 @@
-package git
-
-import (
-	"fmt"
-	"os/exec"
-
-	"github.com/sirupsen/logrus"
-)
-
-// getGitDiffArgs builds the `git diff` invocation args. It removes the usage
-// of a pager and ensures that colours are specified even when Git might detect
-// otherwise.
-func getGitDiffArgs(repoPath string) []string {
-	return []string{
-		"-C",
-		repoPath,
-		"--no-pager",
-		"-c",
-		"color.diff=always",
-		"diff",
-	}
-}
-
-// DiffUnstaged shows a `git diff`. Due to limitations in the underlying go-git
-// library, this implementation requires the /usr/bin/git binary. It gracefully
-// skips if it cannot find the command on the system.
-func DiffUnstaged(path string) error {
-	if _, err := exec.LookPath("git"); err != nil {
-		logrus.Warnf("unable to locate git command, cannot output diff")
-		return nil
-	}
-
-	gitDiffArgs := getGitDiffArgs(path)
-	diff, err := exec.Command("git", gitDiffArgs...).Output()
-	if err != nil {
-		return nil
-	}
-
-	fmt.Print(string(diff))
-
-	return nil
-}

pkg/recipe/recipe.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -32,7 +31,7 @@ import (
 // RecipeCatalogueURL is the only current recipe catalogue available.
 const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
 
-// ReposMetadataURL is the recipe repository metadata.
+// ReposMetadataURL is the recipe repository metadata
 const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
 
 // tag represents a git tag.
@@ -64,11 +63,6 @@ type RecipeMeta struct {
 	Website       string         `json:"website"`
 }
 
-// TopicMeta represents a list of topics for a repository.
-type TopicMeta struct {
-	Topics []string `json:"topics"`
-}
-
 // LatestVersion returns the latest version of a recipe.
 func (r RecipeMeta) LatestVersion() string {
 	var version string
@@ -828,16 +822,7 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 		}
 
 		for idx, repo := range reposList {
-			var topicMeta TopicMeta
-
-			topicsURL := getReposTopicUrl(repo.Name)
-			if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
-				return reposMeta, err
-			}
-
-			if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
-				reposMeta[repo.Name] = reposList[idx]
-			}
+			reposMeta[repo.Name] = reposList[idx]
 		}
 
 		pageIdx++
@@ -1017,8 +1002,14 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 				retrieveBar.Add(1)
 				return
 			}
+			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
+				ch <- rm.Name
+				retrieveBar.Add(1)
+				return
+			}
 
 			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
+
 			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
 				logrus.Fatal(err)
 			}
@@ -1034,8 +1025,3 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 
 	return nil
 }
-
-// getReposTopicUrl retrieves the repository specific topic listing.
-func getReposTopicUrl(repoName string) string {
-	return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
-}

pkg/service/service.go

@@ -14,6 +14,69 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// GetService retrieves a service container based on a label. If prompt is true
+// and the retrievd count of service containers does not match 1, then a prompt
+// is presented to let the user choose. A count of 0 is handled gracefully.
+func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
+	services, err := cl.ServiceList(c, types.ServiceListOptions{})
+	if err != nil {
+		return swarm.Service{}, err
+	}
+
+	if len(services) == 0 {
+		return swarm.Service{}, fmt.Errorf("no services deployed?")
+	}
+
+	var backupServices []swarm.Service
+	for _, service := range services {
+		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
+			backupServices = append(backupServices, service)
+		}
+	}
+
+	if len(backupServices) == 0 {
+		return swarm.Service{}, fmt.Errorf("no backup services deployed?")
+	}
+
+	if len(backupServices) > 1 {
+		var servicesRaw []string
+		for _, service := range backupServices {
+			serviceName := service.Spec.Name
+			created := formatter.HumanDuration(service.CreatedAt.Unix())
+			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
+		}
+
+		if !prompt {
+			err := fmt.Errorf("expected 1 service but found %v: %s", len(backupServices), strings.Join(servicesRaw, " "))
+			return swarm.Service{}, err
+		}
+
+		logrus.Warnf("ambiguous service list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which service are you looking for?",
+			Options: servicesRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return swarm.Service{}, err
+		}
+
+		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, service := range backupServices {
+			serviceName := strings.ToLower(service.Spec.Name)
+			if serviceName == chosenService {
+				return service, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen service")
+	}
+
+	return backupServices[0], nil
+}
+
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.

tests/integration/recipe_diff.bats

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-setup() {
-  load "$PWD/tests/integration/helpers/common"
-  _common_setup
-}
-
-@test "show unstaged changes" {
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  refute_output --partial 'traefik.enable'
-
-  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
-  assert_success
-
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  assert_output --partial 'traefik.enable'
-
-  _reset_recipe
-}

tests/integration/recipe_release.bats

@@ -84,22 +84,3 @@ setup(){
 
   _reset_recipe "$TEST_RECIPE"
 }
-
-@test "unknown files not committed" {
-  run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch
-  assert_success
-
-  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
-  assert_success
-  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
-
-  run $ABRA recipe release "$TEST_RECIPE" --no-input --patch
-  assert_success
-  assert_output --partial 'no -p/--publish passed, not publishing'
-
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo
-  assert_failure
-  assert_output --partial "fatal: pathspec 'foo' did not match any files"
-
-  _reset_recipe
-}

tests/integration/recipe_reset.bats

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-setup() {
-  load "$PWD/tests/integration/helpers/common"
-  _common_setup
-}
-
-@test "reset unstaged changes" {
-  run $ABRA recipe fetch "$TEST_RECIPE"
-  assert_success
-
-  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
-  assert_success
-
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  assert_output --partial 'traefik.enable'
-
-  run $ABRA recipe reset "$TEST_RECIPE"
-  assert_success
-
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  refute_output --partial 'traefik.enable'
-}