From d11ad61efbb383d0cb26ca0965d7468c1d289b81 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 01:34:53 +0200
Subject: [PATCH 01/76] docs: make chaos flag description more generic [ci
 skip]

---
 cli/internal/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/internal/cli.go b/cli/internal/cli.go
index 07e16ac5..b7555bdc 100644
--- a/cli/internal/cli.go
+++ b/cli/internal/cli.go
@@ -54,7 +54,7 @@ var Chaos bool
 // ChaosFlag turns on/off chaos functionality.
 var ChaosFlag = &cli.BoolFlag{
 	Name:        "chaos, C",
-	Usage:       "Deploy uncommitted recipes changes. Use with care!",
+	Usage:       "Proceed with uncommitted recipes changes. Use with care!",
 	Destination: &Chaos,
 }
 

From 970ae0fc4e21f9ef0a76aac5358c2ea57ea7fb01 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 02:29:39 +0200
Subject: [PATCH 02/76] test: use _test to avoid cyclic imports

---
 pkg/config/app.go      | 10 +++----
 pkg/config/app_test.go | 22 +++++++-------
 pkg/config/env_test.go | 65 +++++++++++++++++++++++-------------------
 3 files changed, 52 insertions(+), 45 deletions(-)

diff --git a/pkg/config/app.go b/pkg/config/app.go
index 95dc7784..a6166e2e 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -149,7 +149,7 @@ func (a ByName) Less(i, j int) bool {
 	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
 }
 
-func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
+func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	env, err := ReadEnv(appFile.Path)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
@@ -157,7 +157,7 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 
 	logrus.Debugf("read env %s from %s", env, appFile.Path)
 
-	app, err := newApp(env, name, appFile)
+	app, err := NewApp(env, name, appFile)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error())
 	}
@@ -165,8 +165,8 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
 	return app, nil
 }
 
-// newApp creates new App object
-func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+// NewApp creates new App object
+func NewApp(env AppEnv, name string, appFile AppFile) (App, error) {
 	domain := env["DOMAIN"]
 
 	recipe, exists := env["RECIPE"]
@@ -232,7 +232,7 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
 		return App{}, fmt.Errorf("cannot find app with name %s", name)
 	}
 
-	app, err := readAppEnvFile(appFile, name)
+	app, err := ReadAppEnvFile(appFile, name)
 	if err != nil {
 		return App{}, err
 	}
diff --git a/pkg/config/app_test.go b/pkg/config/app_test.go
index 113d130b..5403ca59 100644
--- a/pkg/config/app_test.go
+++ b/pkg/config/app_test.go
@@ -1,36 +1,38 @@
-package config
+package config_test
 
 import (
 	"reflect"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/config"
 )
 
 func TestNewApp(t *testing.T) {
-	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestReadAppEnvFile(t *testing.T) {
-	app, err := readAppEnvFile(expectedAppFile, appName)
+	app, err := config.ReadAppEnvFile(ExpectedAppFile, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
 
 func TestGetApp(t *testing.T) {
-	app, err := GetApp(expectedAppFiles, appName)
+	app, err := config.GetApp(ExpectedAppFiles, AppName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(app, expectedApp) {
-		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	if !reflect.DeepEqual(app, ExpectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 87049289..157d6ec9 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -1,60 +1,65 @@
-package config
+package config_test
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"reflect"
 	"strings"
 	"testing"
+
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
 
 // make sure these are in alphabetical order
-var tFolders = []string{"folder1", "folder2"}
-var tFiles = []string{"bar.env", "foo.env"}
+var TFolders = []string{"folder1", "folder2"}
+var TFiles = []string{"bar.env", "foo.env"}
 
-var appName = "ecloud"
-var serverName = "evil.corp"
+var AppName = "ecloud"
+var ServerName = "evil.corp"
 
-var expectedAppEnv = AppEnv{
+var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
 	"RECIPE": "ecloud",
 }
 
-var expectedApp = App{
-	Name:   appName,
-	Recipe: expectedAppEnv["RECIPE"],
-	Domain: expectedAppEnv["DOMAIN"],
-	Env:    expectedAppEnv,
-	Path:   expectedAppFile.Path,
-	Server: expectedAppFile.Server,
+var ExpectedApp = config.App{
+	Name:   AppName,
+	Recipe: ExpectedAppEnv["RECIPE"],
+	Domain: ExpectedAppEnv["DOMAIN"],
+	Env:    ExpectedAppEnv,
+	Path:   ExpectedAppFile.Path,
+	Server: ExpectedAppFile.Server,
 }
 
-var expectedAppFile = AppFile{
-	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
-	Server: serverName,
+var ExpectedAppFile = config.AppFile{
+	Path:   path.Join(ValidAbraConf, "servers", ServerName, AppName+".env"),
+	Server: ServerName,
 }
 
-var expectedAppFiles = map[string]AppFile{
-	appName: expectedAppFile,
+var ExpectedAppFiles = map[string]config.AppFile{
+	AppName: ExpectedAppFile,
 }
 
 // var expectedServerNames = []string{"evil.corp"}
 
 func TestGetAllFoldersInDirectory(t *testing.T) {
-	folders, err := GetAllFoldersInDirectory(testFolder)
+	folders, err := config.GetAllFoldersInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(folders, tFolders) {
-		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	if !reflect.DeepEqual(folders, TFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(TFolders, ","), strings.Join(folders, ","))
 	}
 }
 
 func TestGetAllFilesInDirectory(t *testing.T) {
-	files, err := GetAllFilesInDirectory(testFolder)
+	files, err := config.GetAllFilesInDirectory(TestFolder)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,21 +67,21 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 	for _, file := range files {
 		fileNames = append(fileNames, file.Name())
 	}
-	if !reflect.DeepEqual(fileNames, tFiles) {
-		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	if !reflect.DeepEqual(fileNames, TFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(TFiles, ","), strings.Join(fileNames, ","))
 	}
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := ReadEnv(expectedAppFile.Path)
+	env, err := config.ReadEnv(ExpectedAppFile.Path)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(env, expectedAppEnv) {
+	if !reflect.DeepEqual(env, ExpectedAppEnv) {
 		t.Fatalf(
 			"did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s",
-			expectedAppEnv["DOMAIN"],
-			expectedAppEnv["RECIPE"],
+			ExpectedAppEnv["DOMAIN"],
+			ExpectedAppEnv["RECIPE"],
 			env["DOMAIN"],
 			env["RECIPE"],
 		)

From 379915587c249da3b2cc1473f5c406dadc9d068f Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 02:29:59 +0200
Subject: [PATCH 03/76] fix: don't export from within function

Also, don't explode on command function which has "export" in the name!

See https://git.coopcloud.tech/coop-cloud/organising/issues/498
---
 pkg/config/env.go      | 21 ++++++++++++++++-----
 pkg/config/env_test.go | 36 ++++++++++++++++++++++++++++++++++--
 2 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/pkg/config/env.go b/pkg/config/env.go
index 0874fb57..152f1db0 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/Autonomic-Cooperative/godotenv"
@@ -149,22 +150,32 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 		}
 		return envVars, err
 	}
+	defer file.Close()
+
+	exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`)
+	if err != nil {
+		return envVars, err
+	}
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, "export") {
-			splitVals := strings.Split(line, "export ")
+		txt := scanner.Text()
+		if exportRegex.MatchString(txt) {
+			splitVals := strings.Split(txt, "export ")
 			envVarDef := splitVals[len(splitVals)-1]
 			keyVal := strings.Split(envVarDef, "=")
 			if len(keyVal) != 2 {
-				return envVars, fmt.Errorf("couldn't parse %s", line)
+				return envVars, fmt.Errorf("couldn't parse %s", txt)
 			}
 			envVars[keyVal[0]] = keyVal[1]
 		}
 	}
 
-	logrus.Debugf("read %s from %s", envVars, abraSh)
+	if len(envVars) > 0 {
+		logrus.Debugf("read %s from %s", envVars, abraSh)
+	} else {
+		logrus.Debugf("read 0 env var exports from %s", abraSh)
+	}
 
 	return envVars, nil
 }
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 157d6ec9..fc2c835f 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -46,8 +46,6 @@ var ExpectedAppFiles = map[string]config.AppFile{
 	AppName: ExpectedAppFile,
 }
 
-// var expectedServerNames = []string{"evil.corp"}
-
 func TestGetAllFoldersInDirectory(t *testing.T) {
 	folders, err := config.GetAllFoldersInDirectory(TestFolder)
 	if err != nil {
@@ -87,3 +85,37 @@ func TestReadEnv(t *testing.T) {
 		)
 	}
 }
+
+func TestReadAbraShEnvVars(t *testing.T) {
+	if err := catalogue.EnsureCatalogue(); err != nil {
+		t.Fatal(err)
+	}
+
+	offline := true
+	r, err := recipe.Get("abra-integration-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(abraShEnv) == 0 {
+		t.Error("at least one env var should be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_FOO"]; ok {
+		t.Error("INNER_FOO should not be exported")
+	}
+
+	if _, ok := abraShEnv["INNER_BAZ"]; ok {
+		t.Error("INNER_BAZ should not be exported")
+	}
+
+	if _, ok := abraShEnv["OUTER_FOO"]; !ok {
+		t.Error("OUTER_FOO should be exported")
+	}
+}

From 422c64294937753ce47a737eec62eded76cf80b2 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 03:24:30 +0200
Subject: [PATCH 04/76] fix: ensure ipv4 is checked, not sometimes ipv6

See https://git.coopcloud.tech/coop-cloud/organising/issues/490
---
 pkg/dns/dns.go      |  7 ++++++-
 pkg/dns/dns_test.go | 32 +++++++++++++++++++++++++++++---
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go
index 4c888a3f..46408049 100644
--- a/pkg/dns/dns.go
+++ b/pkg/dns/dns.go
@@ -7,11 +7,16 @@ import (
 
 // EnsureIPv4 ensures that an ipv4 address is set for a domain name
 func EnsureIPv4(domainName string) (string, error) {
-	ipv4, err := net.ResolveIPAddr("ip", domainName)
+	ipv4, err := net.ResolveIPAddr("ip4", domainName)
 	if err != nil {
 		return "", err
 	}
 
+	// NOTE(d1): e.g. when there is only an ipv6 record available
+	if ipv4 == nil {
+		return "", fmt.Errorf("unable to resolve ipv4 address for %s", domainName)
+	}
+
 	return ipv4.String(), nil
 }
 
diff --git a/pkg/dns/dns_test.go b/pkg/dns/dns_test.go
index 49b414fd..99249a71 100644
--- a/pkg/dns/dns_test.go
+++ b/pkg/dns/dns_test.go
@@ -3,6 +3,8 @@ package dns
 import (
 	"fmt"
 	"testing"
+
+	"gotest.tools/v3/assert"
 )
 
 func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
@@ -11,9 +13,10 @@ func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
 		serverName     string
 		shouldValidate bool
 	}{
-		// NOTE(d1): DNS records get checked, so use something we control. if
-		// you're here because of a failing test, try `dig +short <domain>` to
-		// ensure stuff matches first!
+		// NOTE(d1): DNS records get checked, so use something that is maintained
+		// within the federation. if you're here because of a failing test, try
+		// `dig +short <domain>` to ensure stuff matches first! If flakyness
+		// becomes an issue we can look into mocking
 		{"docs.coopcloud.tech", "coopcloud.tech", true},
 		{"docs.coopcloud.tech", "swarm.autonomic.zone", true},
 
@@ -36,3 +39,26 @@ func TestEnsureDomainsResolveSameIPv4(t *testing.T) {
 		}
 	}
 }
+
+func TestEnsureIpv4(t *testing.T) {
+	// NOTE(d1): DNS records get checked, so use something that is maintained
+	// within the federation. if you're here because of a failing test, try `dig
+	// +short <domain>` to ensure stuff matches first! If flakyness becomes an
+	// issue we can look into mocking
+	domainName := "collabora.ostrom.collective.tools"
+	serverName := "ostrom.collective.tools"
+
+	for i := 0; i < 15; i++ {
+		domainIpv4, err := EnsureIPv4(domainName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		serverIpv4, err := EnsureIPv4(serverName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, domainIpv4, serverIpv4)
+	}
+}

From c646263e9ea6c6bdf2b7d4f73305d330c976e0e6 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 12:49:58 +0200
Subject: [PATCH 05/76] fix: validate COMPOSE_FILE

See https://git.coopcloud.tech/coop-cloud/organising/issues/468.
See https://git.coopcloud.tech/coop-cloud/organising/issues/376.
---
 pkg/config/app.go                  | 41 ++++++++++++---
 pkg/config/app_test.go             | 81 ++++++++++++++++++++++++++++++
 tests/integration/app_deploy.bats  | 17 +++++++
 tests/integration/app_secret.bats  | 13 -----
 tests/integration/helpers/app.bash | 13 +++++
 5 files changed, 146 insertions(+), 19 deletions(-)

diff --git a/pkg/config/app.go b/pkg/config/app.go
index a6166e2e..7082db79 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -437,27 +437,56 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
 	return statuses, nil
 }
 
+// ensurePathExists ensures that a path exists.
+func ensurePathExists(path string) error {
+	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
 // GetComposeFiles gets the list of compose files for an app (or recipe if you
 // don't already have an app) which should be merged into a composetypes.Config
 // while respecting the COMPOSE_FILE env var.
 func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
 	var composeFiles []string
 
-	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
-		logrus.Debug("no COMPOSE_FILE detected, loading compose.yml")
+	composeFileEnvVar, ok := appEnv["COMPOSE_FILE"]
+	if !ok {
 		path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path)
 		composeFiles = append(composeFiles, path)
 		return composeFiles, nil
 	}
 
-	composeFileEnvVar := appEnv["COMPOSE_FILE"]
-	envVars := strings.Split(composeFileEnvVar, ":")
-	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
-	for _, file := range strings.Split(composeFileEnvVar, ":") {
+	if !strings.Contains(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
+		logrus.Debugf("COMPOSE_FILE detected, loading %s", path)
+		composeFiles = append(composeFiles, path)
+		return composeFiles, nil
+	}
+
+	numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1
+	envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles)
+	if len(envVars) != numComposeFiles {
+		return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar)
+	}
+
+	for _, file := range envVars {
 		path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file)
+		if err := ensurePathExists(path); err != nil {
+			return composeFiles, err
+		}
 		composeFiles = append(composeFiles, path)
 	}
 
+	logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
 	logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)
 
 	return composeFiles, nil
diff --git a/pkg/config/app_test.go b/pkg/config/app_test.go
index 5403ca59..dbf91c05 100644
--- a/pkg/config/app_test.go
+++ b/pkg/config/app_test.go
@@ -1,12 +1,22 @@
 package config_test
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 
+	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/stretchr/testify/assert"
 )
 
+func setup(t *testing.T) {
+	if err := catalogue.EnsureCatalogue(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestNewApp(t *testing.T) {
 	app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile)
 	if err != nil {
@@ -36,3 +46,74 @@ func TestGetApp(t *testing.T) {
 		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp)
 	}
 }
+
+func TestGetComposeFiles(t *testing.T) {
+	setup(t)
+
+	offline := true
+	r, err := recipe.Get("abra-integration-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		appEnv       map[string]string
+		composeFiles []string
+	}{
+		{
+			map[string]string{},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+		{
+			map[string]string{"COMPOSE_FILE": "compose.yml:compose.extra_secret.yml"},
+			[]string{
+				fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name),
+				fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		composeFiles, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, composeFiles, test.composeFiles)
+	}
+}
+
+func TestGetComposeFilesError(t *testing.T) {
+	setup(t)
+
+	offline := true
+	r, err := recipe.Get("abra-integration-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct{ appEnv map[string]string }{
+		{map[string]string{"COMPOSE_FILE": "compose.yml::compose.foo.yml"}},
+		{map[string]string{"COMPOSE_FILE": "doesnt.exist.yml"}},
+	}
+
+	for _, test := range tests {
+		_, err := config.GetComposeFiles(r.Name, test.appEnv)
+		if err == nil {
+			t.Fatalf("should have failed: %v", test.appEnv)
+		}
+	}
+}
diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index 3f81d225..071c2728 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -311,3 +311,20 @@ teardown(){
 
   _undeploy_app
 }
+
+# bats test_tags=slow
+@test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
+  run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  # NOTE(d1): --chaos used to bypass versions and access compose.extra_env.yml
+  run $ABRA app deploy "$TEST_APP_DOMAIN" \
+    --no-input --no-converge-checks --chaos
+  assert_success
+  assert_output --partial "compose.yml"
+  assert_output --partial "compose.extra_env.yml"
+
+  _undeploy_app
+  _reset_app
+}
diff --git a/tests/integration/app_secret.bats b/tests/integration/app_secret.bats
index 23451ad8..bd847a14 100644
--- a/tests/integration/app_secret.bats
+++ b/tests/integration/app_secret.bats
@@ -26,19 +26,6 @@ teardown(){
   fi
 }
 
-_reset_app(){
-  run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-  assert_success
-  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  run $ABRA app new "$TEST_RECIPE" \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-  assert_success
-  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-}
-
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
diff --git a/tests/integration/helpers/app.bash b/tests/integration/helpers/app.bash
index 17dbbc2f..ee3e30f6 100644
--- a/tests/integration/helpers/app.bash
+++ b/tests/integration/helpers/app.bash
@@ -40,3 +40,16 @@ _rm_app() {
     run $ABRA app remove "$TEST_APP_DOMAIN" --no-input
   fi
 }
+
+_reset_app(){
+  run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+  assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN" \
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+}

From e837835e0071a1d8e290044feece142e6f1d6dc6 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 14:05:02 +0200
Subject: [PATCH 06/76] test: remove duplicate call to EnsureCatalogue

---
 pkg/config/app_test.go    | 11 -----------
 pkg/config/env_test.go    |  5 -----
 pkg/recipe/recipe_test.go |  5 -----
 pkg/secret/secret_test.go |  5 -----
 4 files changed, 26 deletions(-)

diff --git a/pkg/config/app_test.go b/pkg/config/app_test.go
index dbf91c05..5f8cb8e6 100644
--- a/pkg/config/app_test.go
+++ b/pkg/config/app_test.go
@@ -5,18 +5,11 @@ import (
 	"reflect"
 	"testing"
 
-	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/stretchr/testify/assert"
 )
 
-func setup(t *testing.T) {
-	if err := catalogue.EnsureCatalogue(); err != nil {
-		t.Fatal(err)
-	}
-}
-
 func TestNewApp(t *testing.T) {
 	app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile)
 	if err != nil {
@@ -48,8 +41,6 @@ func TestGetApp(t *testing.T) {
 }
 
 func TestGetComposeFiles(t *testing.T) {
-	setup(t)
-
 	offline := true
 	r, err := recipe.Get("abra-integration-test-recipe", offline)
 	if err != nil {
@@ -97,8 +88,6 @@ func TestGetComposeFiles(t *testing.T) {
 }
 
 func TestGetComposeFilesError(t *testing.T) {
-	setup(t)
-
 	offline := true
 	r, err := recipe.Get("abra-integration-test-recipe", offline)
 	if err != nil {
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index fc2c835f..0e25766e 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"testing"
 
-	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 )
@@ -87,10 +86,6 @@ func TestReadEnv(t *testing.T) {
 }
 
 func TestReadAbraShEnvVars(t *testing.T) {
-	if err := catalogue.EnsureCatalogue(); err != nil {
-		t.Fatal(err)
-	}
-
 	offline := true
 	r, err := recipe.Get("abra-integration-test-recipe", offline)
 	if err != nil {
diff --git a/pkg/recipe/recipe_test.go b/pkg/recipe/recipe_test.go
index fb0f29ae..59a57ebc 100644
--- a/pkg/recipe/recipe_test.go
+++ b/pkg/recipe/recipe_test.go
@@ -3,15 +3,10 @@ package recipe
 import (
 	"testing"
 
-	"coopcloud.tech/abra/pkg/catalogue"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestGetVersionLabelLocalDoesNotUseTimeoutLabel(t *testing.T) {
-	if err := catalogue.EnsureCatalogue(); err != nil {
-		t.Fatal(err)
-	}
-
 	offline := true
 	recipe, err := Get("traefik", offline)
 	if err != nil {
diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go
index 82dd200c..443b3bab 100644
--- a/pkg/secret/secret_test.go
+++ b/pkg/secret/secret_test.go
@@ -4,7 +4,6 @@ import (
 	"path"
 	"testing"
 
-	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
@@ -13,10 +12,6 @@ import (
 )
 
 func TestReadSecretsConfig(t *testing.T) {
-	if err := catalogue.EnsureCatalogue(); err != nil {
-		t.Fatal(err)
-	}
-
 	offline := true
 	recipe, err := recipe.Get("matrix-synapse", offline)
 	if err != nil {

From ab6107610cf2ba294158859b3c87b9bef8f1df31 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 14:36:59 +0200
Subject: [PATCH 07/76] test: skip build step, test will do it

---
 .drone.yml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index 26207d49..9dd7664b 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -7,13 +7,6 @@ steps:
     commands:
       - make check
 
-  - name: make build
-    image: golang:1.21
-    commands:
-      - make build
-    depends_on:
-      - make check
-
   - name: make test
     image: golang:1.21
     environment:
@@ -29,7 +22,6 @@ steps:
       - git fetch --tags
     depends_on:
       - make check
-      - make build
       - make test
     when:
       event: tag

From dc5d3a8dd684cda6eed8ec297091a74b4def0820 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 14:37:09 +0200
Subject: [PATCH 08/76] test: build, init & test in one stage

---
 .drone.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.drone.yml b/.drone.yml
index 9dd7664b..655a1561 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -12,6 +12,8 @@ steps:
     environment:
       ABRA_DIR: "/root/.abra"
     commands:
+      - make build-abra
+      - ./abra help # show version, initialise $ABRA_DIR
       - make test
     depends_on:
       - make check

From 8078e91e52e67ddcaf1e6af1f85c1358dc206c25 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 15:08:59 +0200
Subject: [PATCH 09/76] fix: warn if secrets not generated

See https://git.coopcloud.tech/coop-cloud/organising/issues/499
---
 cli/app/deploy.go                 | 12 ++++++
 cli/app/secret.go                 | 40 ++++---------------
 pkg/secret/secret.go              | 66 +++++++++++++++++++++++++++++++
 tests/integration/app_deploy.bats | 16 ++++++++
 4 files changed, 102 insertions(+), 32 deletions(-)

diff --git a/cli/app/deploy.go b/cli/app/deploy.go
index 259fba73..352156cb 100644
--- a/cli/app/deploy.go
+++ b/cli/app/deploy.go
@@ -6,6 +6,7 @@ import (
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/secret"
 
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -91,6 +92,17 @@ recipes.
 			logrus.Fatal(err)
 		}
 
+		secStats, err := secret.PollSecretsStatus(cl, app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, secStat := range secStats {
+			if !secStat.CreatedOnRemote {
+				logrus.Fatalf("unable to deploy, secrets not generated (%s)?", secStat.LocalName)
+			}
+		}
+
 		if isDeployed {
 			if internal.Force || internal.Chaos {
 				logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name)
diff --git a/cli/app/secret.go b/cli/app/secret.go
index 31b86c23..eb00ec1b 100644
--- a/cli/app/secret.go
+++ b/cli/app/secret.go
@@ -383,12 +383,7 @@ var appSecretLsCommand = cli.Command{
 			}
 		}
 
-		composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -396,37 +391,18 @@ var appSecretLsCommand = cli.Command{
 		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
 		table := formatter.CreateTable(tableCol)
 
-		cl, err := client.New(app.Server)
+		secStats, err := secret.PollSecretsStatus(cl, app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		filters, err := app.Filters(false, false)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteSecretNames := make(map[string]bool)
-		for _, cont := range secretList {
-			remoteSecretNames[cont.Spec.Annotations.Name] = true
-		}
-
-		for secretName, secretValue := range secretsConfig {
-			createdRemote := false
-			val, err := secret.ParseSecretValue(secretValue)
-			if err != nil {
-				logrus.Fatal(err)
+		for _, secStat := range secStats {
+			tableRow := []string{
+				secStat.LocalName,
+				secStat.Version,
+				secStat.RemoteName,
+				strconv.FormatBool(secStat.CreatedOnRemote),
 			}
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
-			if _, ok := remoteSecretNames[secretRemoteName]; ok {
-				createdRemote = true
-			}
-			tableRow := []string{secretName, val.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
 			table.Append(tableRow)
 		}
 
diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go
index aea8391a..2e7e151f 100644
--- a/pkg/secret/secret.go
+++ b/pkg/secret/secret.go
@@ -4,6 +4,7 @@
 package secret
 
 import (
+	"context"
 	"fmt"
 	"slices"
 	"strconv"
@@ -11,9 +12,11 @@ import (
 	"sync"
 
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/decentral1se/passgen"
+	"github.com/docker/docker/api/types"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 )
@@ -209,3 +212,66 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 	return secrets, nil
 }
+
+type secretStatus struct {
+	LocalName       string
+	RemoteName      string
+	Version         string
+	CreatedOnRemote bool
+}
+
+type secretStatuses []secretStatus
+
+// PollSecretsStatus checks status of secrets by comparing the local recipe
+// config and deploymend server state.
+func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, error) {
+	var secStats secretStatuses
+
+	composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretsConfig, err := ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+	if err != nil {
+		return secStats, err
+	}
+
+	filters, err := app.Filters(false, false)
+	if err != nil {
+		return secStats, err
+	}
+
+	secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters})
+	if err != nil {
+		return secStats, err
+	}
+
+	remoteSecretNames := make(map[string]bool)
+	for _, cont := range secretList {
+		remoteSecretNames[cont.Spec.Annotations.Name] = true
+	}
+
+	for secretName, secretValue := range secretsConfig {
+		createdRemote := false
+
+		val, err := ParseSecretValue(secretValue)
+		if err != nil {
+			return secStats, err
+		}
+
+		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
+		if _, ok := remoteSecretNames[secretRemoteName]; ok {
+			createdRemote = true
+		}
+
+		secStats = append(secStats, secretStatus{
+			LocalName:       secretName,
+			RemoteName:      secretRemoteName,
+			Version:         val.Version,
+			CreatedOnRemote: createdRemote,
+		})
+	}
+
+	return secStats, nil
+}
diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index 071c2728..a3a5ebdd 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -328,3 +328,19 @@ teardown(){
   _undeploy_app
   _reset_app
 }
+
+@test "error if no secrets generated" {
+  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'unable to deploy, secrets not generated'
+
+  _reset_app
+}

From 47d3b77003b4c720adc0fb1a14a6f0e624ceedea Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 15:09:14 +0200
Subject: [PATCH 10/76] refactor: not generating here, skipping

---
 pkg/secret/secret.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go
index 2e7e151f..e8964739 100644
--- a/pkg/secret/secret.go
+++ b/pkg/secret/secret.go
@@ -96,7 +96,7 @@ func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeNa
 		}
 
 		if !(slices.Contains(enabledSecrets, secretId)) {
-			logrus.Warnf("%s not enabled in recipe config, not generating", secretId)
+			logrus.Warnf("%s not enabled in recipe config, skipping", secretId)
 			continue
 		}
 

From 57692ec3c91b80fc7c6f8e4c13f40a1fd41213c7 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 23:08:39 +0200
Subject: [PATCH 11/76] feat: add --machine to secret ls

See https://git.coopcloud.tech/coop-cloud/organising/issues/481
---
 cli/app/secret.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cli/app/secret.go b/cli/app/secret.go
index eb00ec1b..44385867 100644
--- a/cli/app/secret.go
+++ b/cli/app/secret.go
@@ -356,6 +356,7 @@ var appSecretLsCommand = cli.Command{
 		internal.DebugFlag,
 		internal.OfflineFlag,
 		internal.ChaosFlag,
+		internal.MachineReadableFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	Usage:        "List all secrets",
@@ -407,7 +408,11 @@ var appSecretLsCommand = cli.Command{
 		}
 
 		if table.NumLines() > 0 {
-			table.Render()
+			if internal.MachineReadable {
+				table.JSONRender()
+			} else {
+				table.Render()
+			}
 		} else {
 			logrus.Warnf("no secrets stored for %s", app.Name)
 		}

From 14f2d72ababca7ad316ebe55834a3438bccbd0e6 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 4 Oct 2023 23:08:54 +0200
Subject: [PATCH 12/76] refactor!: lowercase, hyphenate keys

This will potentially break scripts, so time to discuss!
---
 pkg/jsontable/jsontable.go        |  6 ++++--
 tests/integration/app_secret.bats | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/pkg/jsontable/jsontable.go b/pkg/jsontable/jsontable.go
index ef49137c..51dd6020 100644
--- a/pkg/jsontable/jsontable.go
+++ b/pkg/jsontable/jsontable.go
@@ -3,6 +3,7 @@ package jsontable
 import (
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/olekukonko/tablewriter"
 )
@@ -109,6 +110,9 @@ func (t *JSONTable) _JSONRenderInner() {
 		}
 		writeChar(t.out, '{')
 		for keyidx, key := range t.keys {
+			key := strings.ToLower(key)
+			key = strings.ReplaceAll(key, " ", "-")
+
 			value := "nil"
 			if keyidx < len(row) {
 				value = row[keyidx]
@@ -138,10 +142,8 @@ func (t *JSONTable) JSONRender() {
 
 		if t.hasCaption {
 			fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption)
-
 		}
 		fmt.Fprintf(t.out, "\"%s\":", t.dataLabel)
-
 	}
 
 	// write list
diff --git a/tests/integration/app_secret.bats b/tests/integration/app_secret.bats
index bd847a14..41679e32 100644
--- a/tests/integration/app_secret.bats
+++ b/tests/integration/app_secret.bats
@@ -325,6 +325,22 @@ setup(){
   assert_success
 }
 
+@test "ls: show secrets as machine readable" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'false'
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN" --machine
+  assert_success
+  assert_output --partial '"created-on-server":"true"'
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+}
+
 @test "ls: bail if unstaged changes and no --chaos" {
   run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success

From b4fd39828f43e86c2acbe7eb7291dee2fe223911 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Thu, 5 Oct 2023 14:22:11 +0200
Subject: [PATCH 13/76] test: abra-integration-test-recipe -> abra-test-recipe

See https://git.coopcloud.tech/coop-cloud/abra-test-recipe/issues/3
---
 pkg/catalogue/catalogue.go            | 72 +++++++++++++--------------
 pkg/config/app_test.go                |  4 +-
 pkg/config/env_test.go                |  2 +-
 tests/integration/helpers/common.bash |  2 +-
 4 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go
index 1b108670..e65a8aa1 100644
--- a/pkg/catalogue/catalogue.go
+++ b/pkg/catalogue/catalogue.go
@@ -14,42 +14,42 @@ import (
 
 // CatalogueSkipList is all the repos that are not recipes.
 var CatalogueSkipList = map[string]bool{
-	"abra":                         true,
-	"abra-apps":                    true,
-	"abra-aur":                     true,
-	"abra-bash":                    true,
-	"abra-capsul":                  true,
-	"abra-gandi":                   true,
-	"abra-hetzner":                 true,
-	"abra-integration-test-recipe": true,
-	"apps":                         true,
-	"aur-abra-git":                 true,
-	"auto-mirror":                  true,
-	"auto-recipes-catalogue-json":  true,
-	"backup-bot":                   true,
-	"backup-bot-two":               true,
-	"beta.coopcloud.tech":          true,
-	"comrade-renovate-bot":         true,
-	"coopcloud.tech":               true,
-	"coturn":                       true,
-	"docker-cp-deploy":             true,
-	"docker-dind-bats-kcov":        true,
-	"docs.coopcloud.tech":          true,
-	"drone-abra":                   true,
-	"example":                      true,
-	"gardening":                    true,
-	"go-abra":                      true,
-	"organising":                   true,
-	"pyabra":                       true,
-	"radicle-seed-node":            true,
-	"recipes-catalogue-json":       true,
-	"recipes-wishlist":             true,
-	"recipes.coopcloud.tech":       true,
-	"stack-ssh-deploy":             true,
-	"swarm-cronjob":                true,
-	"tagcmp":                       true,
-	"traefik-cert-dumper":          true,
-	"tyop":                         true,
+	"abra":                        true,
+	"abra-apps":                   true,
+	"abra-aur":                    true,
+	"abra-bash":                   true,
+	"abra-capsul":                 true,
+	"abra-gandi":                  true,
+	"abra-hetzner":                true,
+	"abra-test-recipe":            true,
+	"apps":                        true,
+	"aur-abra-git":                true,
+	"auto-mirror":                 true,
+	"auto-recipes-catalogue-json": true,
+	"backup-bot":                  true,
+	"backup-bot-two":              true,
+	"beta.coopcloud.tech":         true,
+	"comrade-renovate-bot":        true,
+	"coopcloud.tech":              true,
+	"coturn":                      true,
+	"docker-cp-deploy":            true,
+	"docker-dind-bats-kcov":       true,
+	"docs.coopcloud.tech":         true,
+	"drone-abra":                  true,
+	"example":                     true,
+	"gardening":                   true,
+	"go-abra":                     true,
+	"organising":                  true,
+	"pyabra":                      true,
+	"radicle-seed-node":           true,
+	"recipes-catalogue-json":      true,
+	"recipes-wishlist":            true,
+	"recipes.coopcloud.tech":      true,
+	"stack-ssh-deploy":            true,
+	"swarm-cronjob":               true,
+	"tagcmp":                      true,
+	"traefik-cert-dumper":         true,
+	"tyop":                        true,
 }
 
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
diff --git a/pkg/config/app_test.go b/pkg/config/app_test.go
index 5f8cb8e6..94398a0c 100644
--- a/pkg/config/app_test.go
+++ b/pkg/config/app_test.go
@@ -42,7 +42,7 @@ func TestGetApp(t *testing.T) {
 
 func TestGetComposeFiles(t *testing.T) {
 	offline := true
-	r, err := recipe.Get("abra-integration-test-recipe", offline)
+	r, err := recipe.Get("abra-test-recipe", offline)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -89,7 +89,7 @@ func TestGetComposeFiles(t *testing.T) {
 
 func TestGetComposeFilesError(t *testing.T) {
 	offline := true
-	r, err := recipe.Get("abra-integration-test-recipe", offline)
+	r, err := recipe.Get("abra-test-recipe", offline)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 0e25766e..481d86d1 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -87,7 +87,7 @@ func TestReadEnv(t *testing.T) {
 
 func TestReadAbraShEnvVars(t *testing.T) {
 	offline := true
-	r, err := recipe.Get("abra-integration-test-recipe", offline)
+	r, err := recipe.Get("abra-test-recipe", offline)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/tests/integration/helpers/common.bash b/tests/integration/helpers/common.bash
index 379569ea..66876a2b 100644
--- a/tests/integration/helpers/common.bash
+++ b/tests/integration/helpers/common.bash
@@ -15,5 +15,5 @@ _common_setup() {
 
   export TEST_APP_NAME="$(basename "${BATS_TEST_FILENAME//./_}")"
   export TEST_APP_DOMAIN="$TEST_APP_NAME.$TEST_SERVER"
-  export TEST_RECIPE="abra-integration-test-recipe"
+  export TEST_RECIPE="abra-test-recipe"
 }

From cbe6676881e4195d9b02f08110105fccb2bca8fb Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Fri, 6 Oct 2023 07:00:49 +0000
Subject: [PATCH 14/76] chore(deps): update module golang.org/x/sys to v0.13.0

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 7f03c92d..99b6a091 100644
--- a/go.mod
+++ b/go.mod
@@ -116,5 +116,5 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/sys v0.12.0
+	golang.org/x/sys v0.13.0
 )

From 6fc4573a712253eb9ee9793baa0168d4ccce5b9d Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Fri, 6 Oct 2023 09:49:03 +0200
Subject: [PATCH 15/76] chore: go mod tidy

---
 go.sum | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/go.sum b/go.sum
index 59d700a8..25dbf734 100644
--- a/go.sum
+++ b/go.sum
@@ -1306,8 +1306,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

From b57edb440accf88d85912b18c58d348408ef49c8 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Fri, 6 Oct 2023 00:39:02 +0200
Subject: [PATCH 16/76] fix: improve app check

See https://git.coopcloud.tech/coop-cloud/organising/issues/446
---
 cli/app/check.go       | 50 +++++++++++++++---------------
 cli/app/deploy.go      | 11 +++++++
 cli/app/upgrade.go     | 11 +++++++
 pkg/config/env.go      | 40 ++++++++++++++++++++++++
 pkg/config/env_test.go | 70 ++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 157 insertions(+), 25 deletions(-)

diff --git a/cli/app/check.go b/cli/app/check.go
index d950fd9f..6b0e8455 100644
--- a/cli/app/check.go
+++ b/cli/app/check.go
@@ -1,13 +1,10 @@
 package app
 
 import (
-	"os"
-	"path"
-	"strings"
-
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
@@ -15,9 +12,21 @@ import (
 )
 
 var appCheckCommand = cli.Command{
-	Name:      "check",
-	Aliases:   []string{"chk"},
-	Usage:     "Check if an app is configured correctly",
+	Name:    "check",
+	Aliases: []string{"chk"},
+	Usage:   "Ensure an app is well configured",
+	Description: `
+This command compares env vars in both the app ".env" and recipe ".env.sample"
+file.
+
+The goal is to ensure that recipe ".env.sample" env vars are defined in your
+app ".env" file. Only env var definitions in the ".env.sample" which are
+uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include
+these env vars, then "check" will complain.
+
+Recipe maintainers may or may not provide defaults for env vars within their
+recipes regardless of commenting or not (e.g. through the use of
+${FOO:<default>} syntax). "check" does not confirm or deny this for you.`,
 	ArgsUsage: "<domain>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
@@ -49,32 +58,23 @@ var appCheckCommand = cli.Command{
 			}
 		}
 
-		envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample")
-		if _, err := os.Stat(envSamplePath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist?", envSamplePath)
-			}
-			logrus.Fatal(err)
-		}
+		tableCol := []string{"recipe env sample", "app env"}
+		table := formatter.CreateTable(tableCol)
 
-		envSample, err := config.ReadEnv(envSamplePath)
+		envVars, err := config.CheckEnv(app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		var missing []string
-		for k := range envSample {
-			if _, ok := app.Env[k]; !ok {
-				missing = append(missing, k)
+		for _, envVar := range envVars {
+			if envVar.Present {
+				table.Append([]string{envVar.Name, "✅"})
+			} else {
+				table.Append([]string{envVar.Name, "❌"})
 			}
 		}
 
-		if len(missing) > 0 {
-			missingEnvVars := strings.Join(missing, ", ")
-			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
-		}
-
-		logrus.Infof("all necessary environment variables defined for %s", app.Name)
+		table.Render()
 
 		return nil
 	},
diff --git a/cli/app/deploy.go b/cli/app/deploy.go
index 352156cb..7e68985e 100644
--- a/cli/app/deploy.go
+++ b/cli/app/deploy.go
@@ -200,6 +200,17 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, version)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil {
 			logrus.Fatal(err)
 		}
diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go
index 845c15fd..64065349 100644
--- a/cli/app/upgrade.go
+++ b/cli/app/upgrade.go
@@ -254,6 +254,17 @@ recipes.
 		config.SetChaosVersionLabel(compose, stackName, chosenUpgrade)
 		config.SetUpdateLabel(compose, stackName, app.Env)
 
+		envVars, err := config.CheckEnv(app)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		for _, envVar := range envVars {
+			if !envVar.Present {
+				logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain)
+			}
+		}
+
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}
diff --git a/pkg/config/env.go b/pkg/config/env.go
index 152f1db0..af9ac822 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -9,6 +9,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/Autonomic-Cooperative/godotenv"
@@ -179,3 +180,42 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
 
 	return envVars, nil
 }
+
+type EnvVar struct {
+	Name    string
+	Present bool
+}
+
+func CheckEnv(app App) ([]EnvVar, error) {
+	var envVars []EnvVar
+
+	envSamplePath := path.Join(RECIPES_DIR, app.Recipe, ".env.sample")
+	if _, err := os.Stat(envSamplePath); err != nil {
+		if os.IsNotExist(err) {
+			return envVars, fmt.Errorf("%s does not exist?", envSamplePath)
+		}
+		return envVars, err
+	}
+
+	envSample, err := ReadEnv(envSamplePath)
+	if err != nil {
+		return envVars, err
+	}
+
+	var keys []string
+	for key := range envSample {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		if _, ok := app.Env[key]; ok {
+			envVars = append(envVars, EnvVar{Name: key, Present: true})
+		} else {
+			envVars = append(envVars, EnvVar{Name: key, Present: false})
+		}
+	}
+
+	return envVars, nil
+}
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 481d86d1..047d3e07 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -114,3 +114,73 @@ func TestReadAbraShEnvVars(t *testing.T) {
 		t.Error("OUTER_FOO should be exported")
 	}
 }
+
+func TestCheckEnv(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if !envVar.Present {
+			t.Fatalf("%s should be present", envVar.Name)
+		}
+	}
+}
+
+func TestCheckEnvError(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	delete(envSample, "DOMAIN")
+
+	app := config.App{
+		Name:   "test-app",
+		Recipe: r.Name,
+		Domain: "example.com",
+		Env:    envSample,
+		Path:   "example.com.env",
+		Server: "example.com",
+	}
+
+	envVars, err := config.CheckEnv(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, envVar := range envVars {
+		if envVar.Name == "DOMAIN" && envVar.Present {
+			t.Fatalf("%s should not be present", envVar.Name)
+		}
+	}
+}

From a43125701c21bec5fcc325626566451a0b3ec584 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sat, 7 Oct 2023 10:32:42 +0200
Subject: [PATCH 17/76] test: optimise default make target for abra hacking [ci
 skip]

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index a1266111..54cb2b51 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,8 @@ DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 
 export GOPRIVATE=coopcloud.tech
 
-all: format check build test
+# NOTE(d1): default `make` optimised for Abra hacking
+all: format check build-abra test
 
 run-abra:
 	@go run -ldflags=$(LDFLAGS) $(ABRA)

From be693e9df09b567f53b2402cbc2a689094369692 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sun, 8 Oct 2023 22:42:34 +0200
Subject: [PATCH 18/76] fix: trim comments that are not modifers

See https://git.coopcloud.tech/coop-cloud/organising/issues/505
---
 pkg/config/env.go               | 33 +++++++++++++++++++++++++++++----
 pkg/config/env_test.go          | 33 +++++++++++++++++++++++++++++++++
 pkg/upstream/convert/service.go |  2 +-
 3 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/pkg/config/env.go b/pkg/config/env.go
index af9ac822..923b406a 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -36,6 +36,11 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+// envVarModifiers is a list of env var modifier strings. These are added to
+// env vars as comments and modify their processing by Abra, e.g. determining
+// how long secrets should be.
+var envVarModifiers = []string{"length"}
+
 // GetServers retrieves all servers.
 func GetServers() ([]string, error) {
 	var servers []string
@@ -50,18 +55,38 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
+// ContainsEnvVarModifier determines if an env var contains a modifier.
+func ContainsEnvVarModifier(envVar string) bool {
+	for _, mod := range envVarModifiers {
+		if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
+			return true
+		}
+	}
+	return false
+}
+
 // ReadEnv loads an app envivornment into a map.
 func ReadEnv(filePath string) (AppEnv, error) {
-	var envFile AppEnv
+	var envVars AppEnv
 
-	envFile, err := godotenv.Read(filePath)
+	envVars, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
-	logrus.Debugf("read %s from %s", envFile, filePath)
+	for idx, envVar := range envVars {
+		if strings.Contains(envVar, "#") {
+			if ContainsEnvVarModifier(envVar) {
+				continue
+			}
+			vals := strings.Split(envVar, "#")
+			envVars[idx] = strings.TrimSpace(vals[0])
+		}
+	}
 
-	return envFile, nil
+	logrus.Debugf("read %s from %s", envVars, filePath)
+
+	return envVars, nil
 }
 
 // ReadServerNames retrieves all server names.
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 047d3e07..d3741f32 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -184,3 +184,36 @@ func TestCheckEnvError(t *testing.T) {
 		}
 	}
 }
+
+func TestContainsEnvVarModifier(t *testing.T) {
+	if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
+		t.Fatal("FOO contains no env var modifier")
+	}
+
+	if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
+		t.Fatal("FOO contains an env var modifier (length)")
+	}
+}
+
+func TestEnvVarCommentsRemoved(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envVar, exists := envSample["WITH_COMMENT"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "should be removed") {
+		t.Fatalf("comment from '%s' should be removed", envVar)
+	}
+}
diff --git a/pkg/upstream/convert/service.go b/pkg/upstream/convert/service.go
index 44cfd569..16cd0cfa 100644
--- a/pkg/upstream/convert/service.go
+++ b/pkg/upstream/convert/service.go
@@ -420,7 +420,7 @@ func convertServiceSecrets(
 			return nil, err
 		}
 
-		// NOTE(d1): strip # length=... modifiers
+		// NOTE(d1): strip all comments
 		if strings.Contains(obj.Name, "#") {
 			vals := strings.Split(obj.Name, "#")
 			obj.Name = strings.TrimSpace(vals[0])

From c249c6ae9c6d8d291815a328900c3760d4730ed1 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 9 Oct 2023 14:37:20 +0200
Subject: [PATCH 19/76] fix: fix: trim comments that are not modifers

See https://git.coopcloud.tech/coop-cloud/organising/issues/505
---
 cli/app/new.go                    |  6 ++++--
 cli/app/secret.go                 |  4 ++--
 pkg/compose/compose.go            |  4 ++--
 pkg/config/app.go                 |  2 +-
 pkg/config/env.go                 | 11 +++++++---
 pkg/config/env_test.go            | 35 +++++++++++++++++++++++++++----
 pkg/lint/recipe.go                |  2 +-
 pkg/recipe/recipe.go              |  6 +++---
 pkg/secret/secret.go              |  9 ++++++--
 pkg/secret/secret_test.go         |  5 +++--
 tests/integration/app_deploy.bats |  9 ++++++++
 11 files changed, 71 insertions(+), 22 deletions(-)

diff --git a/cli/app/new.go b/cli/app/new.go
index 34ee69ea..6ff12e6e 100644
--- a/cli/app/new.go
+++ b/cli/app/new.go
@@ -2,6 +2,7 @@ package app
 
 import (
 	"fmt"
+	"path"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -96,7 +97,7 @@ var appNewCommand = cli.Command{
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
-			sampleEnv, err := recipe.SampleEnv()
+			sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -106,7 +107,8 @@ var appNewCommand = cli.Command{
 				logrus.Fatal(err)
 			}
 
-			secretsConfig, err := secret.ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 			if err != nil {
 				return err
 			}
diff --git a/cli/app/secret.go b/cli/app/secret.go
index 44385867..c0bf460b 100644
--- a/cli/app/secret.go
+++ b/cli/app/secret.go
@@ -87,7 +87,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -276,7 +276,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go
index 3fd98191..86c4dfb4 100644
--- a/pkg/compose/compose.go
+++ b/pkg/compose/compose.go
@@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath)
+		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 		if err != nil {
 			return err
 		}
diff --git a/pkg/config/app.go b/pkg/config/app.go
index 7082db79..c5426786 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -150,7 +150,7 @@ func (a ByName) Less(i, j int) bool {
 }
 
 func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path)
+	env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
diff --git a/pkg/config/env.go b/pkg/config/env.go
index 923b406a..f6fbbd51 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -55,6 +55,11 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
+// ReadEnvOptions modifies the ReadEnv processing of env vars.
+type ReadEnvOptions struct {
+	IncludeModifiers bool
+}
+
 // ContainsEnvVarModifier determines if an env var contains a modifier.
 func ContainsEnvVarModifier(envVar string) bool {
 	for _, mod := range envVarModifiers {
@@ -66,7 +71,7 @@ func ContainsEnvVarModifier(envVar string) bool {
 }
 
 // ReadEnv loads an app envivornment into a map.
-func ReadEnv(filePath string) (AppEnv, error) {
+func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
 	var envVars AppEnv
 
 	envVars, err := godotenv.Read(filePath)
@@ -76,7 +81,7 @@ func ReadEnv(filePath string) (AppEnv, error) {
 
 	for idx, envVar := range envVars {
 		if strings.Contains(envVar, "#") {
-			if ContainsEnvVarModifier(envVar) {
+			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
 				continue
 			}
 			vals := strings.Split(envVar, "#")
@@ -222,7 +227,7 @@ func CheckEnv(app App) ([]EnvVar, error) {
 		return envVars, err
 	}
 
-	envSample, err := ReadEnv(envSamplePath)
+	envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
 	if err != nil {
 		return envVars, err
 	}
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index d3741f32..6c62e108 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -70,7 +70,7 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := config.ReadEnv(ExpectedAppFile.Path)
+	env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -123,7 +123,7 @@ func TestCheckEnv(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -157,7 +157,7 @@ func TestCheckEnvError(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -203,7 +203,7 @@ func TestEnvVarCommentsRemoved(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath)
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -216,4 +216,31 @@ func TestEnvVarCommentsRemoved(t *testing.T) {
 	if strings.Contains(envVar, "should be removed") {
 		t.Fatalf("comment from '%s' should be removed", envVar)
 	}
+
+	envVar, exists = envSample["SECRET_TEST_PASS_TWO_VERSION"]
+	if !exists {
+		t.Fatal("WITH_COMMENT env var should be present in .env.sample")
+	}
+
+	if strings.Contains(envVar, "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should have been removed")
+	}
+}
+
+func TestEnvVarModifiersIncluded(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
+	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
+		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
+	}
 }
diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go
index cae797f3..e9214104 100644
--- a/pkg/lint/recipe.go
+++ b/pkg/lint/recipe.go
@@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}
diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go
index 622d24c7..9ddf7d02 100644
--- a/pkg/recipe/recipe.go
+++ b/pkg/recipe/recipe.go
@@ -221,7 +221,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -249,9 +249,9 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}, nil
 }
 
-func (r Recipe) SampleEnv() (map[string]string, error) {
+func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath)
+	sampleEnv, err := config.ReadEnv(envSamplePath, opts)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}
diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go
index e8964739..aa5262d2 100644
--- a/pkg/secret/secret.go
+++ b/pkg/secret/secret.go
@@ -69,9 +69,14 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeName string) (map[string]string, error) {
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
 	secretConfigs := make(map[string]string)
 
+	appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
+	if err != nil {
+		return secretConfigs, err
+	}
+
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
@@ -232,7 +237,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		return secStats, err
 	}
 
-	secretsConfig, err := ReadSecretsConfig(app.Env, composeFiles, app.Recipe)
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 	if err != nil {
 		return secStats, err
 	}
diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go
index 443b3bab..a4b0fc0c 100644
--- a/pkg/secret/secret_test.go
+++ b/pkg/secret/secret_test.go
@@ -18,13 +18,14 @@ func TestReadSecretsConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sampleEnv, err := recipe.SampleEnv()
+	sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
-	secretsFromConfig, err := ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name)
+	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
+	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index a3a5ebdd..30fb9f74 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -344,3 +344,12 @@ teardown(){
 
   _reset_app
 }
+
+@test "recipe config comments not present in values" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app env
+  assert_success
+  refute_output --partial 'should be removed'
+}

From 7566b4262b4458ac698a73af2e3877af3a692ca5 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 9 Oct 2023 22:07:30 +0200
Subject: [PATCH 20/76] fix: set go version to 1.21

---
 go.mod |  2 +-
 go.sum | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 99b6a091..1cb5f1a3 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module coopcloud.tech/abra
 
-go 1.18
+go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
diff --git a/go.sum b/go.sum
index 25dbf734..038c8dee 100644
--- a/go.sum
+++ b/go.sum
@@ -124,6 +124,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -132,6 +133,7 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -313,6 +315,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
@@ -371,6 +374,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
+github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -408,11 +412,13 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
 github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
 github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -665,6 +671,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -784,6 +791,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -883,6 +891,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -1575,6 +1584,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

From 3e511446aaf61e16d8faaa5f980807dff0d25aae Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 9 Oct 2023 22:53:46 +0200
Subject: [PATCH 21/76] refactor: use `app check` emoji here too

---
 cli/recipe/lint.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cli/recipe/lint.go b/cli/recipe/lint.go
index a01d3ecc..542164f9 100644
--- a/cli/recipe/lint.go
+++ b/cli/recipe/lint.go
@@ -68,7 +68,7 @@ var recipeLintCommand = cli.Command{
 
 				skippedOutput := "-"
 				if skipped {
-					skippedOutput = "yes"
+					skippedOutput = "✅"
 				}
 
 				satisfied := false
@@ -87,9 +87,9 @@ var recipeLintCommand = cli.Command{
 					}
 				}
 
-				satisfiedOutput := "yes"
+				satisfiedOutput := "✅"
 				if !satisfied {
-					satisfiedOutput = "NO"
+					satisfiedOutput = "❌"
 					if skipped {
 						satisfiedOutput = "-"
 					}

From 85b90ef80c1439c9da866ab5d2f4a48445b503c1 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 9 Oct 2023 22:36:39 +0200
Subject: [PATCH 22/76] fix: bail if --chaos and specific version

See https://git.coopcloud.tech/coop-cloud/organising/issues/503.
---
 cli/app/deploy.go                   | 6 +++++-
 cli/app/rollback.go                 | 6 +++++-
 cli/app/upgrade.go                  | 6 +++++-
 tests/integration/app_deploy.bats   | 7 +++++++
 tests/integration/app_rollback.bats | 7 +++++++
 tests/integration/app_upgrade.bats  | 7 +++++++
 6 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/cli/app/deploy.go b/cli/app/deploy.go
index 7e68985e..edf8532f 100644
--- a/cli/app/deploy.go
+++ b/cli/app/deploy.go
@@ -51,6 +51,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
@@ -112,7 +117,6 @@ recipes.
 		}
 
 		version := deployedVersion
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			version = specificVersion
 			logrus.Debugf("choosing %s as version to deploy", version)
diff --git a/cli/app/rollback.go b/cli/app/rollback.go
index c8104877..f39d114b 100644
--- a/cli/app/rollback.go
+++ b/cli/app/rollback.go
@@ -51,6 +51,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
@@ -125,7 +130,6 @@ recipes.
 			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {
diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go
index 64065349..8c10337b 100644
--- a/cli/app/upgrade.go
+++ b/cli/app/upgrade.go
@@ -56,6 +56,11 @@ recipes.
 		app := internal.ValidateApp(c)
 		stackName := app.StackName()
 
+		specificVersion := c.Args().Get(1)
+		if specificVersion != "" && internal.Chaos {
+			logrus.Fatal("cannot use <version> and --chaos together")
+		}
+
 		if !internal.Chaos {
 			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
@@ -126,7 +131,6 @@ recipes.
 			logrus.Warnf("failed to determine deployed version of %s", app.Name)
 		}
 
-		specificVersion := c.Args().Get(1)
 		if specificVersion != "" {
 			parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
 			if err != nil {
diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index 30fb9f74..8329793a 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -312,6 +312,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
   run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
diff --git a/tests/integration/app_rollback.bats b/tests/integration/app_rollback.bats
index d6cbbef1..0439b94f 100644
--- a/tests/integration/app_rollback.bats
+++ b/tests/integration/app_rollback.bats
@@ -194,6 +194,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app rollback "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "rollback to previous version" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
diff --git a/tests/integration/app_upgrade.bats b/tests/integration/app_upgrade.bats
index 376a75e2..0b1dafb8 100644
--- a/tests/integration/app_upgrade.bats
+++ b/tests/integration/app_upgrade.bats
@@ -47,6 +47,13 @@ teardown(){
   _undeploy_app
 }
 
+@test "bail out if specific version and chaos" {
+  run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \
+    --chaos --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial 'cannot use'
+}
+
 # bats test_tags=slow
 @test "no upgrade if lint error" {
   _deploy_app

From 69ce07f81f97129fd1951117b2cca2e17de48853 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 9 Oct 2023 23:40:24 +0200
Subject: [PATCH 23/76] fix: ignore build files for docker [ci skip]

---
 .dockerignore | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index d55c5bc6..e4ddfced 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,8 @@
-Dockerfile
-.dockerignore
-*.swp
 *.swo
+*.swp
+.dockerignore
+Dockerfile
+abra
+dist
+kadabra
+tags

From d90c9b88f1fd86756155d645272c80fb6ef1fe2e Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 10 Oct 2023 00:50:43 +0200
Subject: [PATCH 24/76] fix: include ca-certs to avoid x509 error [ci skip]

---
 Dockerfile | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 3b2db743..21b0f52e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,14 @@ FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
-RUN apk add --no-cache make git gcc musl-dev
+RUN apk add --no-cache \
+  ca-certificates \
+  gcc \
+  git \
+  make \
+  musl-dev
+
+RUN update-ca-certificates
 
 COPY . /app
 

From c9bb7e15c2e4f9901e4043fda1cc5a83fa9e05ee Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 10 Oct 2023 07:27:49 +0200
Subject: [PATCH 25/76] fix: bring back docker build

---
 .drone.yml | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index 655a1561..5861cd4d 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -43,22 +43,22 @@ steps:
     when:
       event: tag
 
-  # - name: publish image
-  #   image: plugins/docker
-  #   settings:
-  #     auto_tag: true
-  #     username: 3wordchant
-  #     password:
-  #       from_secret: git_coopcloud_tech_token_3wc
-  #     repo: git.coopcloud.tech/coop-cloud/abra
-  #     tags: dev
-  #     registry: git.coopcloud.tech
-  #   when:
-  #     event:
-  #       exclude:
-  #         - pull_request
-  #   depends_on:
-  #     - make check
+  - name: publish image
+    image: plugins/docker
+    settings:
+      auto_tag: true
+      username: 3wordchant
+      password:
+        from_secret: git_coopcloud_tech_token_3wc
+      repo: git.coopcloud.tech/coop-cloud/abra
+      tags: dev
+      registry: git.coopcloud.tech
+    when:
+      event:
+        exclude:
+          - pull_request
+    depends_on:
+      - make check
 
 volumes:
   - name: deps

From 704c0e9c74999d87061a06b1900bcb48b63582c0 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 11 Oct 2023 18:34:08 +0200
Subject: [PATCH 26/76] test: adapt failing tests to new changes

---
 tests/integration/app_check.bats  | 10 +++++++---
 tests/integration/app_secret.bats |  7 ++++---
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/tests/integration/app_check.bats b/tests/integration/app_check.bats
index 04d99fe2..2ea628c9 100644
--- a/tests/integration/app_check.bats
+++ b/tests/integration/app_check.bats
@@ -35,7 +35,7 @@ setup(){
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
-  assert_output --partial 'all necessary environment variables defined'
+  refute_output --partial '❌'
 
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 }
@@ -111,12 +111,16 @@ setup(){
 }
 
 @test "error if missing env var" {
+  run $ABRA app check "$TEST_APP_DOMAIN"
+  assert_success
+  refute_output --partial '❌'
+
   run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
   assert_success
 
   run $ABRA app check "$TEST_APP_DOMAIN" --chaos
-  assert_failure
-  assert_output --partial "NEW_VAR"
+  assert_success
+  assert_output --partial '❌'
 
   _checkout_recipe
 }
diff --git a/tests/integration/app_secret.bats b/tests/integration/app_secret.bats
index 41679e32..3d15e959 100644
--- a/tests/integration/app_secret.bats
+++ b/tests/integration/app_secret.bats
@@ -110,10 +110,11 @@ setup(){
   assert_success
   assert_output --partial 'test_pass_one'
 
-  run docker -c "$TEST_SERVER" secret ls
+  run bash -c '$ABRA app secret ls $TEST_APP_DOMAIN --machine | \
+    jq -r ".[] | select(.name==\"test_pass_one\") | .version"'
   assert_success
-  assert_output --regexp ".*_test_pass_one_v2"
-  refute_output --regexp ".*_test_pass_one_v1"
+  assert_output --partial 'v2'
+  refute_output --partial 'v1'
 
   run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
   assert_success

From bc88dac1503d04eaa9c79083d7ff412fcaa736c8 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 11 Oct 2023 19:29:19 +0200
Subject: [PATCH 27/76] test: reset before changing files

---
 tests/integration/app_deploy.bats | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index 8329793a..5747ce6e 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -321,6 +321,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
+  _reset_recipe
+
   run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
     "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
   assert_success

From dcecf3299949a7489dd857fcfb7335c0449bc6be Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Wed, 11 Oct 2023 19:30:59 +0200
Subject: [PATCH 28/76] chore: bump version for installer script [ci skip]

---
 scripts/installer/installer | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/installer/installer b/scripts/installer/installer
index 87a72393..2a73b442 100755
--- a/scripts/installer/installer
+++ b/scripts/installer/installer
@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.8.0-beta"
+ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.0-beta"
+RC_VERSION="0.8.1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do

From f96bf9a8ac19e882c8b119a1423921c563000b00 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sun, 15 Oct 2023 12:17:36 +0200
Subject: [PATCH 29/76] feat: `recipe reset`, `recipe diff`

See https://git.coopcloud.tech/coop-cloud/organising/issues/511
---
 cli/recipe/diff.go                  | 61 +++++++++++++++++++++++++++++
 cli/recipe/recipe.go                |  2 +
 cli/recipe/reset.go                 | 56 ++++++++++++++++++++++++++
 tests/integration/recipe_diff.bats  | 21 ++++++++++
 tests/integration/recipe_reset.bats | 25 ++++++++++++
 5 files changed, 165 insertions(+)
 create mode 100644 cli/recipe/diff.go
 create mode 100644 cli/recipe/reset.go
 create mode 100644 tests/integration/recipe_diff.bats
 create mode 100644 tests/integration/recipe_reset.bats

diff --git a/cli/recipe/diff.go b/cli/recipe/diff.go
new file mode 100644
index 00000000..7067300d
--- /dev/null
+++ b/cli/recipe/diff.go
@@ -0,0 +1,61 @@
+package recipe
+
+import (
+	"fmt"
+	"os/exec"
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+// getGitDiffArgs builds the `git diff` invocation args
+func getGitDiffArgs(repoPath string) []string {
+	return []string{
+		"-C",
+		repoPath,
+		"--no-pager",
+		"-c",
+		"color.diff=always",
+		"diff",
+	}
+}
+
+var recipeDiffCommand = cli.Command{
+	Name:        "diff",
+	Usage:       "Show unstaged changes in recipe config",
+	Description: "Due to limitations in our underlying Git dependency, this command requires /usr/bin/git.",
+	Aliases:     []string{"d"},
+	ArgsUsage:   "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		_, err := exec.LookPath("git")
+		if err != nil {
+			logrus.Fatal("unable to locate 'git' command?")
+		}
+
+		gitDiffArgs := getGitDiffArgs(path.Join(config.RECIPES_DIR, recipeName))
+		diff, err := exec.Command("git", gitDiffArgs...).Output()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Print(string(diff))
+
+		return nil
+	},
+}
diff --git a/cli/recipe/recipe.go b/cli/recipe/recipe.go
index 8333f0f0..ad3b6365 100644
--- a/cli/recipe/recipe.go
+++ b/cli/recipe/recipe.go
@@ -30,5 +30,7 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 		recipeSyncCommand,
 		recipeUpgradeCommand,
 		recipeVersionCommand,
+		recipeResetCommand,
+		recipeDiffCommand,
 	},
 }
diff --git a/cli/recipe/reset.go b/cli/recipe/reset.go
new file mode 100644
index 00000000..67208cdc
--- /dev/null
+++ b/cli/recipe/reset.go
@@ -0,0 +1,56 @@
+package recipe
+
+import (
+	"path"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var recipeResetCommand = cli.Command{
+	Name:        "reset",
+	Usage:       "Remove all unstaged changes from recipe config",
+	Description: "WARNING, this will delete your changes. Be Careful.",
+	Aliases:     []string{"rs"},
+	ArgsUsage:   "<recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.NoInputFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.RecipeNameComplete,
+	Action: func(c *cli.Context) error {
+		recipeName := c.Args().First()
+
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
+
+		repoPath := path.Join(config.RECIPES_DIR, recipeName)
+		repo, err := git.PlainOpen(repoPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		ref, err := repo.Head()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		worktree, err := repo.Worktree()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}
+		if err := worktree.Reset(opts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
diff --git a/tests/integration/recipe_diff.bats b/tests/integration/recipe_diff.bats
new file mode 100644
index 00000000..fd384e35
--- /dev/null
+++ b/tests/integration/recipe_diff.bats
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "show unstaged changes" {
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  refute_output --partial 'traefik.enable'
+
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  assert_output --partial 'traefik.enable'
+
+  _reset_recipe
+}
diff --git a/tests/integration/recipe_reset.bats b/tests/integration/recipe_reset.bats
new file mode 100644
index 00000000..22b46f61
--- /dev/null
+++ b/tests/integration/recipe_reset.bats
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+setup() {
+  load "$PWD/tests/integration/helpers/common"
+  _common_setup
+}
+
+@test "reset unstaged changes" {
+  run $ABRA recipe fetch "$TEST_RECIPE"
+  assert_success
+
+  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
+  assert_success
+
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  assert_output --partial 'traefik.enable'
+
+  run $ABRA recipe reset "$TEST_RECIPE"
+  assert_success
+
+  run $ABRA recipe diff "$TEST_RECIPE"
+  assert_success
+  refute_output --partial 'traefik.enable'
+}

From 7f7f7224c683f018fa7aadcc327a64982f084e3e Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sun, 15 Oct 2023 13:39:04 +0200
Subject: [PATCH 30/76] feat: diff on release flow

Also, don't commit unstaged files.
---
 cli/catalogue/catalogue.go            |  2 +-
 cli/recipe/diff.go                    | 27 ++---------------
 cli/recipe/release.go                 | 14 ++++++++-
 cli/recipe/sync.go                    | 12 ++++++++
 cli/recipe/upgrade.go                 | 14 +++++++++
 pkg/git/commit.go                     | 15 ++--------
 pkg/git/diff.go                       | 42 +++++++++++++++++++++++++++
 tests/integration/recipe_release.bats | 19 ++++++++++++
 8 files changed, 107 insertions(+), 38 deletions(-)
 create mode 100644 pkg/git/diff.go

diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go
index 7c97372f..9a54342c 100644
--- a/cli/catalogue/catalogue.go
+++ b/cli/catalogue/catalogue.go
@@ -173,7 +173,7 @@ keys configured on your account.
 			}
 
 			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
+			if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil {
 				logrus.Fatal(err)
 			}
 
diff --git a/cli/recipe/diff.go b/cli/recipe/diff.go
index 7067300d..f505bbe6 100644
--- a/cli/recipe/diff.go
+++ b/cli/recipe/diff.go
@@ -1,29 +1,16 @@
 package recipe
 
 import (
-	"fmt"
-	"os/exec"
 	"path"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-// getGitDiffArgs builds the `git diff` invocation args
-func getGitDiffArgs(repoPath string) []string {
-	return []string{
-		"-C",
-		repoPath,
-		"--no-pager",
-		"-c",
-		"color.diff=always",
-		"diff",
-	}
-}
-
 var recipeDiffCommand = cli.Command{
 	Name:        "diff",
 	Usage:       "Show unstaged changes in recipe config",
@@ -43,19 +30,11 @@ var recipeDiffCommand = cli.Command{
 			internal.ValidateRecipe(c)
 		}
 
-		_, err := exec.LookPath("git")
-		if err != nil {
-			logrus.Fatal("unable to locate 'git' command?")
-		}
-
-		gitDiffArgs := getGitDiffArgs(path.Join(config.RECIPES_DIR, recipeName))
-		diff, err := exec.Command("git", gitDiffArgs...).Output()
-		if err != nil {
+		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
+		if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
 			logrus.Fatal(err)
 		}
 
-		fmt.Print(string(diff))
-
 		return nil
 	},
 }
diff --git a/cli/recipe/release.go b/cli/recipe/release.go
index 3ce13ac6..74cc83bd 100644
--- a/cli/recipe/release.go
+++ b/cli/recipe/release.go
@@ -106,6 +106,18 @@ your SSH keys configured on your account.
 			}
 		}
 
+		isClean, err := gitPkg.IsClean(recipe.Dir())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		if len(tags) > 0 {
 			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
 			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
@@ -244,7 +256,7 @@ func commitRelease(recipe recipe.Recipe, tag string) error {
 
 	msg := fmt.Sprintf("chore: publish %s release", tag)
 	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
+	if err := gitPkg.Commit(repoPath, msg, internal.Dry); err != nil {
 		return err
 	}
 
diff --git a/cli/recipe/sync.go b/cli/recipe/sync.go
index 1085a193..d5b4ad00 100644
--- a/cli/recipe/sync.go
+++ b/cli/recipe/sync.go
@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -198,6 +199,17 @@ likely to change.
 			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
+		isClean, err := gitPkg.IsClean(recipe.Dir())
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		return nil
 	},
 }
diff --git a/cli/recipe/upgrade.go b/cli/recipe/upgrade.go
index aa258f16..4a4a33e2 100644
--- a/cli/recipe/upgrade.go
+++ b/cli/recipe/upgrade.go
@@ -14,6 +14,7 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
+	gitPkg "coopcloud.tech/abra/pkg/git"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -326,6 +327,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 
 				fmt.Println(string(jsonstring))
+
 				return nil
 			}
 
@@ -336,6 +338,18 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 		}
+
+		isClean, err := gitPkg.IsClean(recipeDir)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !isClean {
+			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
+			if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		return nil
 	},
 }
diff --git a/pkg/git/commit.go b/pkg/git/commit.go
index a3c6433f..4b0e554b 100644
--- a/pkg/git/commit.go
+++ b/pkg/git/commit.go
@@ -8,7 +8,7 @@ import (
 )
 
 // Commit runs a git commit
-func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
+func Commit(repoPath, commitMessage string, dryRun bool) error {
 	if commitMessage == "" {
 		return fmt.Errorf("no commit message specified?")
 	}
@@ -33,17 +33,8 @@ func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
 	}
 
 	if !dryRun {
-		err = commitWorktree.AddGlob(glob)
-		if err != nil {
-			return err
-		}
-		logrus.Debugf("staged %s for commit", glob)
-	} else {
-		logrus.Debugf("dry run: did not stage %s for commit", glob)
-	}
-
-	if !dryRun {
-		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
+		// NOTE(d1): `All: true` does not include untracked files
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{All: true})
 		if err != nil {
 			return err
 		}
diff --git a/pkg/git/diff.go b/pkg/git/diff.go
new file mode 100644
index 00000000..65a658f8
--- /dev/null
+++ b/pkg/git/diff.go
@@ -0,0 +1,42 @@
+package git
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+)
+
+// getGitDiffArgs builds the `git diff` invocation args. It removes the usage
+// of a pager and ensures that colours are specified even when Git might detect
+// otherwise.
+func getGitDiffArgs(repoPath string) []string {
+	return []string{
+		"-C",
+		repoPath,
+		"--no-pager",
+		"-c",
+		"color.diff=always",
+		"diff",
+	}
+}
+
+// DiffUnstaged shows a `git diff`. Due to limitations in the underlying go-git
+// library, this implementation requires the /usr/bin/git binary. It gracefully
+// skips if it cannot find the command on the system.
+func DiffUnstaged(path string) error {
+	if _, err := exec.LookPath("git"); err != nil {
+		logrus.Warnf("unable to locate git command, cannot output diff")
+		return nil
+	}
+
+	gitDiffArgs := getGitDiffArgs(path)
+	diff, err := exec.Command("git", gitDiffArgs...).Output()
+	if err != nil {
+		return nil
+	}
+
+	fmt.Print(string(diff))
+
+	return nil
+}
diff --git a/tests/integration/recipe_release.bats b/tests/integration/recipe_release.bats
index a847206c..2ebe226c 100644
--- a/tests/integration/recipe_release.bats
+++ b/tests/integration/recipe_release.bats
@@ -84,3 +84,22 @@ setup(){
 
   _reset_recipe "$TEST_RECIPE"
 }
+
+@test "unknown files not committed" {
+  run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch
+  assert_success
+
+  run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA recipe release "$TEST_RECIPE" --no-input --patch
+  assert_success
+  assert_output --partial 'no -p/--publish passed, not publishing'
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo
+  assert_failure
+  assert_output --partial "fatal: pathspec 'foo' did not match any files"
+
+  _reset_recipe
+}

From 6a673ef101b072713f3c229b236c4a4793bec1be Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Thu, 5 Oct 2023 09:34:02 +0200
Subject: [PATCH 31/76] refactor: filter by topic when building catalogue

See https://git.coopcloud.tech/coop-cloud/organising/issues/377
---
 cli/catalogue/catalogue.go |  5 -----
 pkg/catalogue/catalogue.go | 40 --------------------------------------
 pkg/recipe/recipe.go       | 30 ++++++++++++++++++++--------
 3 files changed, 22 insertions(+), 53 deletions(-)

diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go
index 9a54342c..b7d09ccc 100644
--- a/cli/catalogue/catalogue.go
+++ b/cli/catalogue/catalogue.go
@@ -98,11 +98,6 @@ keys configured on your account.
 				continue
 			}
 
-			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
 			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go
index e65a8aa1..80e54885 100644
--- a/pkg/catalogue/catalogue.go
+++ b/pkg/catalogue/catalogue.go
@@ -12,46 +12,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                        true,
-	"abra-apps":                   true,
-	"abra-aur":                    true,
-	"abra-bash":                   true,
-	"abra-capsul":                 true,
-	"abra-gandi":                  true,
-	"abra-hetzner":                true,
-	"abra-test-recipe":            true,
-	"apps":                        true,
-	"aur-abra-git":                true,
-	"auto-mirror":                 true,
-	"auto-recipes-catalogue-json": true,
-	"backup-bot":                  true,
-	"backup-bot-two":              true,
-	"beta.coopcloud.tech":         true,
-	"comrade-renovate-bot":        true,
-	"coopcloud.tech":              true,
-	"coturn":                      true,
-	"docker-cp-deploy":            true,
-	"docker-dind-bats-kcov":       true,
-	"docs.coopcloud.tech":         true,
-	"drone-abra":                  true,
-	"example":                     true,
-	"gardening":                   true,
-	"go-abra":                     true,
-	"organising":                  true,
-	"pyabra":                      true,
-	"radicle-seed-node":           true,
-	"recipes-catalogue-json":      true,
-	"recipes-wishlist":            true,
-	"recipes.coopcloud.tech":      true,
-	"stack-ssh-deploy":            true,
-	"swarm-cronjob":               true,
-	"tagcmp":                      true,
-	"traefik-cert-dumper":         true,
-	"tyop":                        true,
-}
-
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go
index 9ddf7d02..6e47e029 100644
--- a/pkg/recipe/recipe.go
+++ b/pkg/recipe/recipe.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -31,7 +32,7 @@ import (
 // RecipeCatalogueURL is the only current recipe catalogue available.
 const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
 
-// ReposMetadataURL is the recipe repository metadata
+// ReposMetadataURL is the recipe repository metadata.
 const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
 
 // tag represents a git tag.
@@ -63,6 +64,11 @@ type RecipeMeta struct {
 	Website       string         `json:"website"`
 }
 
+// TopicMeta represents a list of topics for a repository.
+type TopicMeta struct {
+	Topics []string `json:"topics"`
+}
+
 // LatestVersion returns the latest version of a recipe.
 func (r RecipeMeta) LatestVersion() string {
 	var version string
@@ -822,7 +828,16 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 		}
 
 		for idx, repo := range reposList {
-			reposMeta[repo.Name] = reposList[idx]
+			var topicMeta TopicMeta
+
+			topicsURL := getReposTopicUrl(repo.Name)
+			if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
+				return reposMeta, err
+			}
+
+			if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
+				reposMeta[repo.Name] = reposList[idx]
+			}
 		}
 
 		pageIdx++
@@ -1002,14 +1017,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 				retrieveBar.Add(1)
 				return
 			}
-			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
 
 			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
 			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
 				logrus.Fatal(err)
 			}
@@ -1025,3 +1034,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 
 	return nil
 }
+
+// getReposTopicUrl retrieves the repository specific topic listing.
+func getReposTopicUrl(repoName string) string {
+	return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
+}

From 254df7f2bed61286fbee1da631148792a7a575e8 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 17 Oct 2023 21:16:31 +0200
Subject: [PATCH 32/76] feat: app cmd ls

See https://git.coopcloud.tech/coop-cloud/organising/issues/484
---
 cli/app/cmd.go         | 58 +++++++++++++++++++++++++++++++++++++++---
 pkg/config/env.go      | 36 ++++++++++++++++++++++++++
 pkg/config/env_test.go | 26 +++++++++++++++++++
 3 files changed, 116 insertions(+), 4 deletions(-)

diff --git a/cli/app/cmd.go b/cli/app/cmd.go
index 47c4f54d..9073ce92 100644
--- a/cli/app/cmd.go
+++ b/cli/app/cmd.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"sort"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -22,8 +23,7 @@ var appCmdCommand = cli.Command{
 	Name:    "command",
 	Aliases: []string{"cmd"},
 	Usage:   "Run app commands",
-	Description: `
-Run an app specific command.
+	Description: `Run an app specific command.
 
 These commands are bash functions, defined in the abra.sh of the recipe itself.
 They can be run within the context of a service (e.g. app) or locally on your
@@ -43,8 +43,8 @@ Example:
 		internal.OfflineFlag,
 		internal.ChaosFlag,
 	},
-	BashComplete: autocomplete.AppNameComplete,
-	Before:       internal.SubCommandBefore,
+	Before:      internal.SubCommandBefore,
+	Subcommands: []cli.Command{appCmdListCommand},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -186,3 +186,53 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 
 	return hasCmdArgs, parsedCmdArgs
 }
+
+var appCmdListCommand = cli.Command{
+	Name:      "list",
+	Aliases:   []string{"ls"},
+	Usage:     "List all available commands",
+	ArgsUsage: "<domain>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	BashComplete: autocomplete.AppNameComplete,
+	Before:       internal.SubCommandBefore,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+		cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		sort.Strings(cmdNames)
+		for _, cmdName := range cmdNames {
+			fmt.Println(cmdName)
+		}
+
+		return nil
+	},
+}
diff --git a/pkg/config/env.go b/pkg/config/env.go
index f6fbbd51..2d9b0228 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -249,3 +249,39 @@ func CheckEnv(app App) ([]EnvVar, error) {
 
 	return envVars, nil
 }
+
+// ReadAbraShCmdNames reads the names of commands.
+func ReadAbraShCmdNames(abraSh string) ([]string, error) {
+	var cmdNames []string
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cmdNames, nil
+		}
+		return cmdNames, err
+	}
+	defer file.Close()
+
+	cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`)
+	if err != nil {
+		return cmdNames, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := cmdNameRegex.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			cmdNames = append(cmdNames, matches[1])
+		}
+	}
+
+	if len(cmdNames) > 0 {
+		logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh)
+	} else {
+		logrus.Debugf("read 0 command names from %s", abraSh)
+	}
+
+	return cmdNames, nil
+}
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 6c62e108..cbc5aecd 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 
@@ -115,6 +116,31 @@ func TestReadAbraShEnvVars(t *testing.T) {
 	}
 }
 
+func TestReadAbraShCmdNames(t *testing.T) {
+	offline := true
+	r, err := recipe.Get("abra-test-recipe", offline)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh")
+	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(cmdNames) == 0 {
+		t.Error("at least one command name should be found")
+	}
+
+	expectedCmdNames := []string{"test_cmd", "test_cmd_args"}
+	for _, cmdName := range expectedCmdNames {
+		if !slices.Contains(cmdNames, cmdName) {
+			t.Fatalf("%s should have been found in %s", cmdName, abraShPath)
+		}
+	}
+}
+
 func TestCheckEnv(t *testing.T) {
 	offline := true
 	r, err := recipe.Get("abra-test-recipe", offline)

From ccf021549506f7b2f4112cfed81605ec4daa8a45 Mon Sep 17 00:00:00 2001
From: knoflook <knoflook@disroot.org>
Date: Mon, 23 Oct 2023 19:21:45 +0200
Subject: [PATCH 33/76] hotfix: parse values starting with # correctly

---
 pkg/config/env.go | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/pkg/config/env.go b/pkg/config/env.go
index 2d9b0228..202c29a8 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -79,15 +79,15 @@ func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
 		return nil, err
 	}
 
-	for idx, envVar := range envVars {
-		if strings.Contains(envVar, "#") {
-			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
-				continue
-			}
-			vals := strings.Split(envVar, "#")
-			envVars[idx] = strings.TrimSpace(vals[0])
-		}
-	}
+	//	for idx, envVar := range envVars {
+	//		if strings.Contains(envVar, "#") {
+	//			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
+	//				continue
+	//			}
+	//			vals := strings.Split(envVar, "#")
+	//			envVars[idx] = strings.TrimSpace(vals[0])
+	//		}
+	//	}
 
 	logrus.Debugf("read %s from %s", envVars, filePath)
 

From 51e3df17f1fbee79bbeb6d1654b60209c327e09a Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Thu, 26 Oct 2023 07:00:33 +0000
Subject: [PATCH 34/76] chore(deps): update module github.com/go-git/go-git/v5
 to v5.10.0

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 1cb5f1a3..a8111f1a 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/docker v24.0.6+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.9.0
+	github.com/go-git/go-git/v5 v5.10.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5

From 79d274e0746fd97ef0dd2c445954e2cd2d589eb8 Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Fri, 27 Oct 2023 07:01:16 +0000
Subject: [PATCH 35/76] chore(deps): update module github.com/docker/cli to
 v24.0.7

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index a8111f1a..02ba069a 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v24.0.6+incompatible
+	github.com/docker/cli v24.0.7+incompatible
 	github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/docker v24.0.6+incompatible
 	github.com/docker/go-units v0.5.0

From bd5cdd34437e7f3c2df9c0c1cc1b958c7a5f836c Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Mon, 30 Oct 2023 08:00:53 +0000
Subject: [PATCH 36/76] chore(deps): update module github.com/docker/docker to
 v24.0.7

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 02ba069a..828cb9ad 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/docker/cli v24.0.7+incompatible
 	github.com/docker/distribution v2.8.3+incompatible
-	github.com/docker/docker v24.0.6+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/go-git/go-git/v5 v5.10.0
 	github.com/moby/sys/signal v0.7.0

From 856c9f2f7da91eeb765ba0f2611256fee638bce8 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sat, 4 Nov 2023 09:37:15 +0100
Subject: [PATCH 37/76] chore: go mod tidy

---
 go.mod |  6 +++---
 go.sum | 28 ++++++++++++++--------------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/go.mod b/go.mod
index 828cb9ad..3a1e58a9 100644
--- a/go.mod
+++ b/go.mod
@@ -78,11 +78,11 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.13.0 // indirect
diff --git a/go.sum b/go.sum
index 038c8dee..c3043ae0 100644
--- a/go.sum
+++ b/go.sum
@@ -339,16 +339,16 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=
-github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
+github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
-github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
@@ -417,10 +417,10 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
-github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
+github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ=
+github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -1069,8 +1069,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1168,8 +1168,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1323,8 +1323,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

From 609656b4e1c5c7313ab801a0da9b85b8f35560cb Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Mon, 6 Nov 2023 08:00:33 +0000
Subject: [PATCH 38/76] chore(deps): update module golang.org/x/sys to v0.14.0

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 3a1e58a9..18b586f2 100644
--- a/go.mod
+++ b/go.mod
@@ -116,5 +116,5 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/urfave/cli v1.22.9
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
-	golang.org/x/sys v0.13.0
+	golang.org/x/sys v0.14.0
 )

From 74200318abbec77cb5cd282415a8a49ded0e31c4 Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Tue, 7 Nov 2023 08:01:11 +0000
Subject: [PATCH 39/76] chore(deps): update module
 github.com/schollz/progressbar/v3 to v3.14.0

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 18b586f2..431e4de6 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.13.1
+	github.com/schollz/progressbar/v3 v3.14.0
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )

From f255fa15557bf870ccfb4dd4218ce9ff7798d68f Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Thu, 9 Nov 2023 08:00:33 +0000
Subject: [PATCH 40/76] chore(deps): update module
 github.com/hashicorp/go-retryablehttp to v0.7.5

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 431e4de6..c17fb766 100644
--- a/go.mod
+++ b/go.mod
@@ -104,7 +104,7 @@ require (
 	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.4
+	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/klauspost/pgzip v1.2.6
 	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect

From 9b0dd933b54032a869566dc829bb9d9df0d156c7 Mon Sep 17 00:00:00 2001
From: Comrade Renovate Bot <renovate-bot@autonomic.zone>
Date: Fri, 10 Nov 2023 08:00:52 +0000
Subject: [PATCH 41/76] chore(deps): update module
 github.com/schollz/progressbar/v3 to v3.14.1

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index c17fb766..e5cf4d18 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.14.0
+	github.com/schollz/progressbar/v3 v3.14.1
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )

From 92b073d5b6d32d89a1a3c82f0207bc601a6da269 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 27 Nov 2023 10:28:43 +0100
Subject: [PATCH 42/76] chore: go mod tidy

---
 go.mod |  6 +++---
 go.sum | 24 ++++++++++++------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/go.mod b/go.mod
index e5cf4d18..cda0c18f 100644
--- a/go.mod
+++ b/go.mod
@@ -56,7 +56,7 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
@@ -71,7 +71,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -82,7 +82,7 @@ require (
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.13.0 // indirect
diff --git a/go.sum b/go.sum
index c3043ae0..ac4beb16 100644
--- a/go.sum
+++ b/go.sum
@@ -590,8 +590,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
-github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -705,8 +705,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -885,8 +885,9 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -900,8 +901,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
-github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
+github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI=
+github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
@@ -1310,21 +1311,20 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

From 316b59b465b7cc6e0ab09edfc25760d81fb9a424 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 27 Nov 2023 10:41:46 +0100
Subject: [PATCH 43/76] test: support local-first testing

Cherry-picked from https://git.coopcloud.tech/coop-cloud/abra/pulls/389

Thanks @p4u1.
---
 tests/integration/helpers/common.bash | 6 +++---
 tests/integration/helpers/server.bash | 6 +++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/integration/helpers/common.bash b/tests/integration/helpers/common.bash
index 66876a2b..a97d3e94 100644
--- a/tests/integration/helpers/common.bash
+++ b/tests/integration/helpers/common.bash
@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 _common_setup() {
-  load '/usr/lib/bats/bats-support/load'
-  load '/usr/lib/bats/bats-assert/load'
-  load '/usr/lib/bats/bats-file/load'
+  bats_load_library bats-support
+  bats_load_library bats-assert
+  bats_load_library bats-file
 
   load "$PWD/tests/integration/helpers/app"
   load "$PWD/tests/integration/helpers/git"
diff --git a/tests/integration/helpers/server.bash b/tests/integration/helpers/server.bash
index edeb9115..3c6c74d7 100644
--- a/tests/integration/helpers/server.bash
+++ b/tests/integration/helpers/server.bash
@@ -1,7 +1,11 @@
 #!/usr/bin/env bash
 
 _add_server() {
-  run $ABRA server add "$TEST_SERVER"
+  if [[ "$TEST_SERVER" == "default" ]]; then
+      run $ABRA server add -l
+  else
+      run $ABRA server add "$TEST_SERVER"
+  fi
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER"
 }

From 7b6134f35e25f8457b0c406aed731f080893436e Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Fri, 10 Nov 2023 12:03:10 +0100
Subject: [PATCH 44/76] add bash completion for abra cmd

---
 cli/app/cmd.go                   | 41 +++++++++++++++++++++++++++++---
 pkg/autocomplete/autocomplete.go | 10 ++++++++
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/cli/app/cmd.go b/cli/app/cmd.go
index 9073ce92..002eecfa 100644
--- a/cli/app/cmd.go
+++ b/cli/app/cmd.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/app"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
@@ -45,6 +46,17 @@ Example:
 	},
 	Before:      internal.SubCommandBefore,
 	Subcommands: []cli.Command{appCmdListCommand},
+	BashComplete: func(ctx *cli.Context) {
+		args := ctx.Args()
+		switch len(args) {
+		case 0:
+			autocomplete.AppNameComplete(ctx)
+		case 1:
+			autocomplete.ServiceNameComplete(args.Get(0))
+		case 2:
+			cmdNameComplete(args.Get(0))
+		}
+	},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -187,6 +199,20 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) {
 	return hasCmdArgs, parsedCmdArgs
 }
 
+func cmdNameComplete(appName string) {
+	app, err := app.Get(appName)
+	if err != nil {
+		return
+	}
+	cmdNames, _ := getShCmdNames(app)
+	if err != nil {
+		return
+	}
+	for _, n := range cmdNames {
+		fmt.Println(n)
+	}
+}
+
 var appCmdListCommand = cli.Command{
 	Name:      "list",
 	Aliases:   []string{"ls"},
@@ -222,13 +248,11 @@ var appCmdListCommand = cli.Command{
 			}
 		}
 
-		abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
-		cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+		cmdNames, err := getShCmdNames(app)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		sort.Strings(cmdNames)
 		for _, cmdName := range cmdNames {
 			fmt.Println(cmdName)
 		}
@@ -236,3 +260,14 @@ var appCmdListCommand = cli.Command{
 		return nil
 	},
 }
+
+func getShCmdNames(app config.App) ([]string, error) {
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh")
+	cmdNames, err := config.ReadAbraShCmdNames(abraShPath)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Strings(cmdNames)
+	return cmdNames, nil
+}
diff --git a/pkg/autocomplete/autocomplete.go b/pkg/autocomplete/autocomplete.go
index 807422fb..9d5075d6 100644
--- a/pkg/autocomplete/autocomplete.go
+++ b/pkg/autocomplete/autocomplete.go
@@ -25,6 +25,16 @@ func AppNameComplete(c *cli.Context) {
 	}
 }
 
+func ServiceNameComplete(appName string) {
+	serviceNames, err := config.GetAppServiceNames(appName)
+	if err != nil {
+		return
+	}
+	for _, s := range serviceNames {
+		fmt.Println(s)
+	}
+}
+
 // RecipeNameComplete completes recipe names.
 func RecipeNameComplete(c *cli.Context) {
 	catl, err := recipe.ReadRecipeCatalogue(false)

From b1d3fcbb0b105f2aea33e6caef8131ca6d8bf8fb Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Mon, 13 Nov 2023 10:31:52 +0100
Subject: [PATCH 45/76] add integration test

---
 tests/integration/app_cmd.bats | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tests/integration/app_cmd.bats b/tests/integration/app_cmd.bats
index b7762600..db2808ba 100644
--- a/tests/integration/app_cmd.bats
+++ b/tests/integration/app_cmd.bats
@@ -25,6 +25,24 @@ teardown(){
   fi
 }
 
+# bats test_tags=slow
+@test "autocomplete" {
+  run $ABRA app cmd --generate-bash-completion
+  assert_success
+  assert_output "$TEST_APP_DOMAIN"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" --generate-bash-completion
+  assert_success
+  assert_output "app"
+
+  run $ABRA app cmd "$TEST_APP_DOMAIN" app --generate-bash-completion
+  assert_success
+  assert_output "test_cmd
+test_cmd_arg
+test_cmd_args
+test_cmd_export"
+}
+
 @test "validate app argument" {
   run $ABRA app cmd
   assert_failure

From 31e0ed75b0084dfc05388ee749537b1e0c0b4a39 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 27 Nov 2023 11:15:59 +0100
Subject: [PATCH 46/76] build: target for docker building

Adapted from https://git.coopcloud.tech/coop-cloud/abra/pulls/384.

Thanks @cas.
---
 Makefile |  8 ++++++++
 build.sh | 11 +++++++++++
 2 files changed, 19 insertions(+)
 create mode 100755 build.sh

diff --git a/Makefile b/Makefile
index 54cb2b51..16695637 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,7 @@ ABRA         := ./cmd/abra
 KADABRA      := ./cmd/kadabra
 COMMIT       := $(shell git rev-list -1 HEAD)
 GOPATH       := $(shell go env GOPATH)
+GOVERSION    := 1.21
 LDFLAGS      := "-X 'main.Commit=$(COMMIT)'"
 DIST_LDFLAGS := $(LDFLAGS)" -s -w"
 
@@ -30,6 +31,13 @@ build-kadabra:
 
 build: build-abra build-kadabra
 
+build-docker-abra:
+	@docker run -it \
+		-v $(PWD):/abra golang:$(GOVERSION) \
+		bash -c 'cd /abra; mv scripts/docker/build.sh .; ./build.sh'
+
+build-docker: build-docker-abra
+
 clean:
 	@rm '$(GOPATH)/bin/abra'
 	@rm '$(GOPATH)/bin/kadabra'
diff --git a/build.sh b/build.sh
new file mode 100755
index 00000000..4227c3d3
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ ! -f .envrc ]; then
+    . .envrc.sample
+else
+    . .envrc
+fi
+
+git config --global --add safe.directory /abra  # work around funky file permissions
+
+make build

From 15857e645322fc8b2d2d88679dc0698dd69f2703 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 27 Nov 2023 11:21:27 +0100
Subject: [PATCH 47/76] fix: clean up after cp'ing script [ci skip]

Follows 31e0ed75b0084dfc05388ee749537b1e0c0b4a39.
---
 Makefile                            | 5 ++---
 build.sh => scripts/docker/build.sh | 0
 2 files changed, 2 insertions(+), 3 deletions(-)
 rename build.sh => scripts/docker/build.sh (100%)

diff --git a/Makefile b/Makefile
index 16695637..45795ec0 100644
--- a/Makefile
+++ b/Makefile
@@ -32,9 +32,8 @@ build-kadabra:
 build: build-abra build-kadabra
 
 build-docker-abra:
-	@docker run -it \
-		-v $(PWD):/abra golang:$(GOVERSION) \
-		bash -c 'cd /abra; mv scripts/docker/build.sh .; ./build.sh'
+	@docker run -it -v $(PWD):/abra golang:$(GOVERSION) \
+		bash -c 'cd /abra; cp scripts/docker/build.sh .; ./build.sh; rm build.sh'
 
 build-docker: build-docker-abra
 
diff --git a/build.sh b/scripts/docker/build.sh
similarity index 100%
rename from build.sh
rename to scripts/docker/build.sh

From 049f02f06305597615381d25b7ff54e378627152 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 27 Nov 2023 11:23:03 +0100
Subject: [PATCH 48/76] docs: add p4u1 [ci skip]

---
 AUTHORS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS.md b/AUTHORS.md
index 6698c3cb..9e9c7620 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -11,6 +11,7 @@
 - kawaiipunk
 - knoflook
 - moritz
+- p4u1
 - rix
 - roxxers
 - vera

From 66b40a9189c97925ef63a8866f2c7b5872c834e7 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 27 Nov 2023 11:25:01 +0100
Subject: [PATCH 49/76] fix: just run it in place [ci skip]

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 45795ec0..7657868b 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ build: build-abra build-kadabra
 
 build-docker-abra:
 	@docker run -it -v $(PWD):/abra golang:$(GOVERSION) \
-		bash -c 'cd /abra; cp scripts/docker/build.sh .; ./build.sh; rm build.sh'
+		bash -c 'cd /abra; ./scripts/docker/build.sh'
 
 build-docker: build-docker-abra
 

From 6e54ec7213ed5424d393414af5ab5825be84a66d Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 28 Nov 2023 11:42:36 +0100
Subject: [PATCH 50/76] test: skip failing test for now

See https://git.coopcloud.tech/coop-cloud/organising/issues/535.
---
 pkg/config/env_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index cbc5aecd..86fc4cb7 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -222,6 +222,8 @@ func TestContainsEnvVarModifier(t *testing.T) {
 }
 
 func TestEnvVarCommentsRemoved(t *testing.T) {
+	t.Skip("https://git.coopcloud.tech/coop-cloud/organising/issues/535")
+
 	offline := true
 	r, err := recipe.Get("abra-test-recipe", offline)
 	if err != nil {

From 0d83339d80d60f21eb804faab699d8956955b11e Mon Sep 17 00:00:00 2001
From: Moritz <moritz.m@local-it.org>
Date: Thu, 30 Nov 2023 16:35:24 +0100
Subject: [PATCH 51/76] fix(ssh): increase connection timeout #482

see https://git.coopcloud.tech/coop-cloud/organising/issues/482
---
 pkg/upstream/commandconn/connection.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/upstream/commandconn/connection.go b/pkg/upstream/commandconn/connection.go
index 259b71d1..b2eebdbe 100644
--- a/pkg/upstream/commandconn/connection.go
+++ b/pkg/upstream/commandconn/connection.go
@@ -18,7 +18,7 @@ import (
 //
 // ssh://<user>@<host> URL requires Docker 18.09 or later on the remote host.
 func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
-	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=5"})
+	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=60"})
 }
 
 func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) {

From 3957b7c965bb973d6f254599350d817053332e95 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Wed, 29 Nov 2023 18:35:01 +0100
Subject: [PATCH 52/76] proper env modifiers support

This implements proper modifier support in the env file using this new fork of the godotenv library. The modifier implementation is quite basic for but can be improved later if needed. See this commit for the actual implementation.

Because we are now using proper modifer parsing, it does not affect the parsing of value, so this is possible again:
```
MY_VAR="#foo"
```
Closes coop-cloud/organising#535
---
 cli/app/new.go               |   6 +-
 cli/app/secret.go            |  61 +++++++++----------
 go.mod                       |   2 +-
 go.sum                       |   4 +-
 pkg/compose/compose.go       |   4 +-
 pkg/config/app.go            |   5 +-
 pkg/config/env.go            |  47 ++++++---------
 pkg/config/env_test.go       |  51 ++++++++--------
 pkg/lint/recipe.go           |   2 +-
 pkg/recipe/recipe.go         |   6 +-
 pkg/secret/secret.go         | 111 +++++++++++++++--------------------
 pkg/secret/secret_test.go    |   2 +-
 pkg/upstream/stack/loader.go |  13 +++-
 13 files changed, 146 insertions(+), 168 deletions(-)

diff --git a/cli/app/new.go b/cli/app/new.go
index 6ff12e6e..975ca502 100644
--- a/cli/app/new.go
+++ b/cli/app/new.go
@@ -97,7 +97,7 @@ var appNewCommand = cli.Command{
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
-			sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
+			sampleEnv, err := recipe.SampleEnv()
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -168,7 +168,7 @@ var appNewCommand = cli.Command{
 type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) {
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
 	// NOTE(d1): trim to match app.StackName() implementation
 	if len(sanitisedAppName) > 45 {
 		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
@@ -217,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 }
 
 // promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(recipeName string, secretsConfig map[string]string) error {
+func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
 	if len(secretsConfig) == 0 {
 		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
 		return nil
diff --git a/cli/app/secret.go b/cli/app/secret.go
index c0bf460b..2b2f7fd4 100644
--- a/cli/app/secret.go
+++ b/cli/app/secret.go
@@ -20,19 +20,23 @@ import (
 	"github.com/urfave/cli"
 )
 
-var allSecrets bool
-var allSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &allSecrets,
-	Usage:       "Generate all secrets",
-}
+var (
+	allSecrets     bool
+	allSecretsFlag = &cli.BoolFlag{
+		Name:        "all, a",
+		Destination: &allSecrets,
+		Usage:       "Generate all secrets",
+	}
+)
 
-var rmAllSecrets bool
-var rmAllSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &rmAllSecrets,
-	Usage:       "Remove all secrets",
-}
+var (
+	rmAllSecrets     bool
+	rmAllSecretsFlag = &cli.BoolFlag{
+		Name:        "all, a",
+		Destination: &rmAllSecrets,
+		Usage:       "Remove all secrets",
+	}
+)
 
 var appSecretGenerateCommand = cli.Command{
 	Name:      "generate",
@@ -87,28 +91,22 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		secretsToCreate := make(map[string]string)
-		if allSecrets {
-			secretsToCreate = secretsConfig
-		} else {
+		if !allSecrets {
 			secretName := c.Args().Get(1)
 			secretVersion := c.Args().Get(2)
-			matches := false
-			for name := range secretsConfig {
-				if secretName == name {
-					secretsToCreate[name] = secretVersion
-					matches = true
-				}
-			}
-
-			if !matches {
+			s, ok := secrets[secretName]
+			if !ok {
 				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
+			s.Version = secretVersion
+			secrets = map[string]secret.SecretValue{
+				secretName: s,
+			}
 		}
 
 		cl, err := client.New(app.Server)
@@ -116,7 +114,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
+		secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -276,7 +274,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -311,12 +309,7 @@ Example:
 
 		match := false
 		secretToRm := c.Args().Get(1)
-		for secretName, secretValue := range secretsConfig {
-			val, err := secret.ParseSecretValue(secretValue)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
+		for secretName, val := range secrets {
 			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 			if _, ok := remoteSecretNames[secretRemoteName]; ok {
 				if secretToRm != "" {
diff --git a/go.mod b/go.mod
index cda0c18f..357d57dc 100644
--- a/go.mod
+++ b/go.mod
@@ -4,8 +4,8 @@ go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd
 	github.com/AlecAivazis/survey/v2 v2.3.7
-	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/docker/cli v24.0.7+incompatible
 	github.com/docker/distribution v2.8.3+incompatible
diff --git a/go.sum b/go.sum
index ac4beb16..92d767ac 100644
--- a/go.sum
+++ b/go.sum
@@ -51,12 +51,12 @@ coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd h1:dctCkMhcsgIWMrkB1Br8S0RJF17eG+LKiqcXXVr3mdU=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
 github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
-github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg=
-github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
diff --git a/pkg/compose/compose.go b/pkg/compose/compose.go
index 86c4dfb4..3fd98191 100644
--- a/pkg/compose/compose.go
+++ b/pkg/compose/compose.go
@@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
 			return false, err
 		}
@@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
 			return err
 		}
diff --git a/pkg/config/app.go b/pkg/config/app.go
index c5426786..996c5357 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -25,6 +25,9 @@ import (
 // AppEnv is a map of the values in an apps env config
 type AppEnv = map[string]string
 
+// AppModifiers is a map of modifiers in an apps env config
+type AppModifiers = map[string]map[string]string
+
 // AppName is AppName
 type AppName = string
 
@@ -150,7 +153,7 @@ func (a ByName) Less(i, j int) bool {
 }
 
 func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
+	env, err := ReadEnv(appFile.Path)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}
diff --git a/pkg/config/env.go b/pkg/config/env.go
index 202c29a8..62f6a71d 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -12,7 +12,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/Autonomic-Cooperative/godotenv"
+	"git.coopcloud.tech/coop-cloud/godotenv"
 	"github.com/sirupsen/logrus"
 )
 
@@ -55,45 +55,34 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
-// ReadEnvOptions modifies the ReadEnv processing of env vars.
-type ReadEnvOptions struct {
-	IncludeModifiers bool
-}
-
-// ContainsEnvVarModifier determines if an env var contains a modifier.
-func ContainsEnvVarModifier(envVar string) bool {
-	for _, mod := range envVarModifiers {
-		if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
-			return true
-		}
-	}
-	return false
-}
-
 // ReadEnv loads an app envivornment into a map.
-func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
+func ReadEnv(filePath string) (AppEnv, error) {
 	var envVars AppEnv
 
-	envVars, err := godotenv.Read(filePath)
+	envVars, _, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
-	//	for idx, envVar := range envVars {
-	//		if strings.Contains(envVar, "#") {
-	//			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
-	//				continue
-	//			}
-	//			vals := strings.Split(envVar, "#")
-	//			envVars[idx] = strings.TrimSpace(vals[0])
-	//		}
-	//	}
-
 	logrus.Debugf("read %s from %s", envVars, filePath)
 
 	return envVars, nil
 }
 
+// ReadEnv loads an app envivornment and their modifiers in two different maps.
+func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) {
+	var envVars AppEnv
+
+	envVars, mods, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, mods, err
+	}
+
+	logrus.Debugf("read %s from %s", envVars, filePath)
+
+	return envVars, mods, nil
+}
+
 // ReadServerNames retrieves all server names.
 func ReadServerNames() ([]string, error) {
 	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
@@ -227,7 +216,7 @@ func CheckEnv(app App) ([]EnvVar, error) {
 		return envVars, err
 	}
 
-	envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
+	envSample, err := ReadEnv(envSamplePath)
 	if err != nil {
 		return envVars, err
 	}
diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go
index 86fc4cb7..f0626e6b 100644
--- a/pkg/config/env_test.go
+++ b/pkg/config/env_test.go
@@ -13,15 +13,21 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+var (
+	TestFolder    = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+	ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+)
 
 // make sure these are in alphabetical order
-var TFolders = []string{"folder1", "folder2"}
-var TFiles = []string{"bar.env", "foo.env"}
+var (
+	TFolders = []string{"folder1", "folder2"}
+	TFiles   = []string{"bar.env", "foo.env"}
+)
 
-var AppName = "ecloud"
-var ServerName = "evil.corp"
+var (
+	AppName    = "ecloud"
+	ServerName = "evil.corp"
+)
 
 var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
@@ -71,7 +77,7 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
+	env, err := config.ReadEnv(ExpectedAppFile.Path)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -149,7 +155,7 @@ func TestCheckEnv(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	envSample, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -183,7 +189,7 @@ func TestCheckEnvError(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	envSample, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,19 +217,7 @@ func TestCheckEnvError(t *testing.T) {
 	}
 }
 
-func TestContainsEnvVarModifier(t *testing.T) {
-	if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
-		t.Fatal("FOO contains no env var modifier")
-	}
-
-	if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
-		t.Fatal("FOO contains an env var modifier (length)")
-	}
-}
-
 func TestEnvVarCommentsRemoved(t *testing.T) {
-	t.Skip("https://git.coopcloud.tech/coop-cloud/organising/issues/535")
-
 	offline := true
 	r, err := recipe.Get("abra-test-recipe", offline)
 	if err != nil {
@@ -231,7 +225,7 @@ func TestEnvVarCommentsRemoved(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	envSample, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -263,12 +257,19 @@ func TestEnvVarModifiersIncluded(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
+	envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
-		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "v1") {
+		t.Errorf("value should be 'v1', got: '%s'", envSample["SECRET_TEST_PASS_TWO_VERSION"])
+	}
+	if modifiers == nil || modifiers["SECRET_TEST_PASS_TWO_VERSION"] == nil {
+		t.Errorf("no modifiers included")
+	} else {
+		if modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"] != "10" {
+			t.Errorf("length modifier should be '10', got: '%s'", modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"])
+		}
 	}
 }
diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go
index e9214104..cae797f3 100644
--- a/pkg/lint/recipe.go
+++ b/pkg/lint/recipe.go
@@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}
diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go
index 6e47e029..dbd4a520 100644
--- a/pkg/recipe/recipe.go
+++ b/pkg/recipe/recipe.go
@@ -227,7 +227,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -255,9 +255,9 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}, nil
 }
 
-func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
+func (r Recipe) SampleEnv() (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath, opts)
+	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}
diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go
index aa5262d2..947808e7 100644
--- a/pkg/secret/secret.go
+++ b/pkg/secret/secret.go
@@ -21,9 +21,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
+// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
 // secret definition.
-type secretValue struct {
+type SecretValue struct {
 	Version string
 	Length  int
 }
@@ -35,7 +35,6 @@ func GeneratePasswords(count, length uint) ([]string, error) {
 		length,
 		passgen.AlphabetDefault,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -54,7 +53,6 @@ func GeneratePassphrases(count uint) ([]string, error) {
 		passgen.PassphraseCasingDefault,
 		passgen.WordListDefault,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -69,18 +67,20 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
-	secretConfigs := make(map[string]string)
-
-	appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
+	appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
 	if err != nil {
-		return secretConfigs, err
+		return nil, err
 	}
 
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
-		return secretConfigs, err
+		return nil, err
+	}
+	configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
+	if err != nil {
+		return nil, err
 	}
 
 	var enabledSecrets []string
@@ -92,12 +92,13 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 
 	if len(enabledSecrets) == 0 {
 		logrus.Debugf("not generating app secrets, none enabled in recipe config")
-		return secretConfigs, nil
+		return nil, nil
 	}
 
+	secretValues := map[string]SecretValue{}
 	for secretId, secretConfig := range config.Secrets {
 		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
-			return secretConfigs, fmt.Errorf("missing version for secret? (%s)", secretId)
+			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
 		}
 
 		if !(slices.Contains(enabledSecrets, secretId)) {
@@ -107,60 +108,47 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 
 		lastIdx := strings.LastIndex(secretConfig.Name, "_")
 		secretVersion := secretConfig.Name[lastIdx+1:]
-		secretConfigs[secretId] = secretVersion
+		value := SecretValue{Version: secretVersion}
+
+		// Check if the length modifier is set for this secret.
+		for k, v := range appModifiers {
+			// configWithoutEnv contains the raw name as defined in the compose.yaml
+			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
+				continue
+			}
+			lengthRaw, ok := v["length"]
+			if ok {
+				length, err := strconv.Atoi(lengthRaw)
+				if err != nil {
+					return nil, err
+				}
+				value.Length = length
+			}
+			break
+		}
+		secretValues[secretId] = value
 	}
 
-	return secretConfigs, nil
-}
-
-func ParseSecretValue(secret string) (secretValue, error) {
-	values := strings.Split(secret, "#")
-	if len(values) == 0 {
-		return secretValue{}, fmt.Errorf("unable to parse %s", secret)
-	}
-
-	if len(values) == 1 {
-		return secretValue{Version: values[0], Length: 0}, nil
-	}
-
-	split := strings.Split(values[1], "=")
-	parsed := split[len(split)-1]
-	stripped := strings.ReplaceAll(parsed, " ", "")
-	length, err := strconv.Atoi(stripped)
-	if err != nil {
-		return secretValue{}, err
-	}
-	version := strings.ReplaceAll(values[0], " ", "")
-
-	logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
-
-	return secretValue{Version: version, Length: length}, nil
+	return secretValues, nil
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, appName, server string) (map[string]string, error) {
-	secrets := make(map[string]string)
-
+func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
+	secretsGenerated := map[string]string{}
 	var mutex sync.Mutex
 	var wg sync.WaitGroup
-	ch := make(chan error, len(secretsFromConfig))
-	for n, v := range secretsFromConfig {
+	ch := make(chan error, len(secrets))
+	for n, v := range secrets {
 		wg.Add(1)
 
-		go func(secretName, secretValue string) {
+		go func(secretName string, secret SecretValue) {
 			defer wg.Done()
 
-			parsedSecretValue, err := ParseSecretValue(secretValue)
-			if err != nil {
-				ch <- err
-				return
-			}
-
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version)
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
 			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
 
-			if parsedSecretValue.Length > 0 {
-				passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length))
+			if secret.Length > 0 {
+				passwords, err := GeneratePasswords(1, uint(secret.Length))
 				if err != nil {
 					ch <- err
 					return
@@ -178,7 +166,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secrets[secretName] = passwords[0]
+				secretsGenerated[secretName] = passwords[0]
 			} else {
 				passphrases, err := GeneratePassphrases(1)
 				if err != nil {
@@ -198,7 +186,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secrets[secretName] = passphrases[0]
+				secretsGenerated[secretName] = passphrases[0]
 			}
 			ch <- nil
 		}(n, v)
@@ -206,16 +194,16 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 	wg.Wait()
 
-	for range secretsFromConfig {
+	for range secrets {
 		err := <-ch
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	logrus.Debugf("generated and stored %s on %s", secrets, server)
+	logrus.Debugf("generated and stored %v on %s", secrets, server)
 
-	return secrets, nil
+	return secretsGenerated, nil
 }
 
 type secretStatus struct {
@@ -257,14 +245,9 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		remoteSecretNames[cont.Spec.Annotations.Name] = true
 	}
 
-	for secretName, secretValue := range secretsConfig {
+	for secretName, val := range secretsConfig {
 		createdRemote := false
 
-		val, err := ParseSecretValue(secretValue)
-		if err != nil {
-			return secStats, err
-		}
-
 		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 		if _, ok := remoteSecretNames[secretRemoteName]; ok {
 			createdRemote = true
diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go
index a4b0fc0c..bb462a8a 100644
--- a/pkg/secret/secret_test.go
+++ b/pkg/secret/secret_test.go
@@ -18,7 +18,7 @@ func TestReadSecretsConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
+	sampleEnv, err := recipe.SampleEnv()
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/pkg/upstream/stack/loader.go b/pkg/upstream/stack/loader.go
index 51b678b0..9790903f 100644
--- a/pkg/upstream/stack/loader.go
+++ b/pkg/upstream/stack/loader.go
@@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) {
 	opts.SkipValidation = false
 }
 
+// SkipInterpolation skip interpolating environment variables.
+func SkipInterpolation(opts *loader.Options) {
+	opts.SkipInterpolation = true
+}
+
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
-func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
+func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
 	if err != nil {
 		return nil, err
 	}
 
+	if options == nil {
+		options = []func(*loader.Options){DontSkipValidation}
+	}
+
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails, DontSkipValidation)
+	config, err := loader.Load(configDetails, options...)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
 			return nil, fmt.Errorf("compose file contains unsupported options: %s",

From 9affda8a70270632ecea60ef592e7f3287bd0374 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sat, 2 Dec 2023 12:59:42 +0100
Subject: [PATCH 53/76] chore: update godotenv fork commit pointer

Follows https://git.coopcloud.tech/coop-cloud/abra/pulls/391
---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 357d57dc..5287ec3d 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
-	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd
+	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/docker/cli v24.0.7+incompatible
diff --git a/go.sum b/go.sum
index 92d767ac..c3c08ba0 100644
--- a/go.sum
+++ b/go.sum
@@ -53,6 +53,8 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd h1:dctCkMhcsgIWMrkB1Br8S0RJF17eG+LKiqcXXVr3mdU=
 git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355 h1:tCv2B4qoN6RMheKDnCzIafOkWS5BB1h7hwhmo+9bVeE=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=

From cb49cf06d1e14385ed2225c2389c9774d6ed43c2 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sat, 2 Dec 2023 13:02:24 +0100
Subject: [PATCH 54/76] chore: drop old godotenv pointers [ci skip]

Follows 9affda8a70270632ecea60ef592e7f3287bd0374
---
 go.sum | 2 --
 1 file changed, 2 deletions(-)

diff --git a/go.sum b/go.sum
index c3c08ba0..d767d61e 100644
--- a/go.sum
+++ b/go.sum
@@ -51,8 +51,6 @@ coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd h1:dctCkMhcsgIWMrkB1Br8S0RJF17eG+LKiqcXXVr3mdU=
-git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
 git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355 h1:tCv2B4qoN6RMheKDnCzIafOkWS5BB1h7hwhmo+9bVeE=
 git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=

From 2de6934322a6430e2634162668b4ce95489212e9 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Tue, 14 Nov 2023 15:29:53 +0100
Subject: [PATCH 55/76] feat: abra app cp enhancements

---
 cli/app/cp.go                 | 372 +++++++++++++++++++++++++++-------
 cli/app/cp_test.go            | 113 +++++++++++
 go.sum                        |   1 +
 pkg/container/container.go    |  12 ++
 tests/integration/app_cp.bats | 224 ++++++++++++++++----
 5 files changed, 609 insertions(+), 113 deletions(-)
 create mode 100644 cli/app/cp_test.go

diff --git a/cli/app/cp.go b/cli/app/cp.go
index 937c7975..bfc2c789 100644
--- a/cli/app/cp.go
+++ b/cli/app/cp.go
@@ -2,19 +2,24 @@ package app
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"os"
+	"path"
+	"path/filepath"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/container"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -49,46 +54,14 @@ And if you want to copy that file back to your current working directory locally
 		dst := c.Args().Get(2)
 		if src == "" {
 			logrus.Fatal("missing <src> argument")
-		} else if dst == "" {
+		}
+		if dst == "" {
 			logrus.Fatal("missing <dest> argument")
 		}
 
-		parsedSrc := strings.SplitN(src, ":", 2)
-		parsedDst := strings.SplitN(dst, ":", 2)
-		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
-		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
-			logrus.Fatal(errorMsg)
-		} else if len(parsedSrc) != 2 {
-			if len(parsedDst) != 2 {
-				logrus.Fatal(errorMsg)
-			}
-		} else if len(parsedDst) != 2 {
-			if len(parsedSrc) != 2 {
-				logrus.Fatal(errorMsg)
-			}
-		}
-
-		var service string
-		var srcPath string
-		var dstPath string
-		isToContainer := false // <container:src> <dst>
-		if len(parsedSrc) == 2 {
-			service = parsedSrc[0]
-			srcPath = parsedSrc[1]
-			dstPath = dst
-			logrus.Debugf("assuming transfer is coming FROM the container")
-		} else if len(parsedDst) == 2 {
-			service = parsedDst[0]
-			dstPath = parsedDst[1]
-			srcPath = src
-			isToContainer = true // <src> <container:dst>
-			logrus.Debugf("assuming transfer is going TO the container")
-		}
-
-		if isToContainer {
-			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist locally?", srcPath)
-			}
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
+		if err != nil {
+			logrus.Fatal(err)
 		}
 
 		cl, err := client.New(app.Server)
@@ -96,7 +69,18 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Fatal(err)
 		}
 
-		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
+		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+
+		if toContainer {
+			err = copyToContainer(cl, container.ID, srcPath, dstPath)
+		} else {
+			err = copyFromContainer(cl, container.ID, srcPath, dstPath)
+		}
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -104,46 +88,292 @@ And if you want to copy that file back to your current working directory locally
 	},
 }
 
-func configureAndCp(
-	c *cli.Context,
-	cl *dockerClient.Client,
-	app config.App,
-	srcPath string,
-	dstPath string,
-	service string,
-	isToContainer bool) error {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
+var errServiceMissing = errors.New("one of <src>/<dest> arguments must take $SERVICE:$PATH form")
 
-	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
+// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH
+func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) {
+	parsedSrc := strings.SplitN(src, ":", 2)
+	parsedDst := strings.SplitN(dst, ":", 2)
+	if len(parsedSrc)+len(parsedDst) != 3 {
+		return "", "", "", false, errServiceMissing
+	}
+	if len(parsedSrc) == 2 {
+		return parsedSrc[1], dst, parsedSrc[0], false, nil
+	}
+	if len(parsedDst) == 2 {
+		return src, parsedDst[1], parsedDst[0], true, nil
+	}
+	return "", "", "", false, errServiceMissing
+}
+
+// copyToContainer copies a file or directory from the local file system to the container.
+// See the possible copy modes and their documentation.
+func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := os.Stat(srcPath)
 	if err != nil {
-		logrus.Fatal(err)
+		return fmt.Errorf("local %s ", err)
 	}
 
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+	dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath)
+	dstExists := true
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			dstExists = false
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	}
 
-	if isToContainer {
-		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		content, err := archive.TarWithOptions(srcPath, toTarOpts)
-		if err != nil {
-			logrus.Fatal(err)
-		}
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists)
+	if err != nil {
+		return err
+	}
+	movePath := ""
+	switch mode {
+	case CopyModeDirToDir:
+		// Add the src directory to the destination path
+		_, srcDir := path.Split(srcPath)
+		dstPath = path.Join(dstPath, srcDir)
 
-		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	} else {
-		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
+		// Make sure the dst directory exits.
+		dcli, err := command.NewDockerCli()
 		if err != nil {
-			logrus.Fatal(err)
+			return err
 		}
-		defer content.Close()
-		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-			logrus.Fatal(err)
+		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mkdir", "-p", dstPath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return fmt.Errorf("create remote directory: %s", err)
+		}
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		movePath = dstPath
+		dstPath = "/tmp"
+	}
+
+	toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(srcPath, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("copy %s from local to %s on container", srcPath, dstPath)
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
+		return err
+	}
+
+	if movePath != "" {
+		_, srcFile := path.Split(srcPath)
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			return err
+		}
+		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mv", path.Join("/tmp", srcFile), movePath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return fmt.Errorf("create remote directory: %s", err)
 		}
 	}
 
 	return nil
 }
+
+// copyFromContainer copies a file or directory from the given container to the local file system.
+// See the possible copy modes and their documentation.
+func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return fmt.Errorf("remote: %s does not exist", srcPath)
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	}
+
+	dstStat, err := os.Stat(dstPath)
+	dstExists := true
+	var dstMode os.FileMode
+	if err != nil {
+		if os.IsNotExist(err) {
+			dstExists = false
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	} else {
+		dstMode = dstStat.Mode()
+	}
+
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists)
+	if err != nil {
+		return err
+	}
+
+	moveDstDir := ""
+	moveDstFile := ""
+	switch mode {
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		moveDstFile = dstPath
+		dstPath = "/tmp"
+	case CopyModeFilesToDir:
+		// Copy the directory to the temp directory and move it to its
+		// dstPath afterwards.
+		moveDstDir = path.Join(dstPath, "/")
+		dstPath = "/tmp"
+
+		// Make sure the temp directory always gets removed
+		defer os.Remove(path.Join("/tmp"))
+	}
+
+	content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath)
+	if err != nil {
+		return fmt.Errorf("copy: %s", err)
+	}
+	defer content.Close()
+	if err := archive.Untar(content, dstPath, &archive.TarOptions{
+		NoOverwriteDirNonDir: true,
+		Compression:          archive.Gzip,
+		NoLchown:             true,
+	}); err != nil {
+		return fmt.Errorf("untar: %s", err)
+	}
+
+	if moveDstFile != "" {
+		_, srcFile := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil {
+			return err
+		}
+	}
+	if moveDstDir != "" {
+		_, srcDir := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var (
+	ErrCopyDirToFile  = fmt.Errorf("can't copy dir to file")
+	ErrDstDirNotExist = fmt.Errorf("destination directory does not exist")
+)
+
+type CopyMode int
+
+const (
+	// Copy a src file to a dest file. The src and dest file names are the same.
+	//  <dir_src>/<file> + <dir_dst>/<file> -> <dir_dst>/<file>
+	CopyModeFileToFile = CopyMode(iota)
+	// Copy a src file to a dest file. The src and dest file names are  not the same.
+	//  <dir_src>/<file_src> + <dir_dst>/<file_dst> -> <dir_dst>/<file_dst>
+	CopyModeFileToFileRename
+	// Copy a src file to dest directory. The dest file gets created in the dest
+	// folder with the src filename.
+	//  <dir_src>/<file> + <dir_dst> -> <dir_dst>/<file>
+	CopyModeFileToDir
+	// Copy a src directory to dest directory.
+	//  <dir_src> + <dir_dst> -> <dir_dst>/<dir_src>
+	CopyModeDirToDir
+	// Copy all files in the src directory to the dest directory. This works recursively.
+	//  <dir_src>/ + <dir_dst> -> <dir_dst>/<files_from_dir_src>
+	CopyModeFilesToDir
+)
+
+// copyMode takes a src and dest path and file mode to determine the copy mode.
+// See the possible copy modes and their documentation.
+func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) {
+	_, srcFile := path.Split(srcPath)
+	_, dstFile := path.Split(dstPath)
+	if srcMode.IsDir() {
+		if !dstExists {
+			return -1, ErrDstDirNotExist
+		}
+		if dstMode.IsDir() {
+			if strings.HasSuffix(srcPath, "/") {
+				return CopyModeFilesToDir, nil
+			}
+			return CopyModeDirToDir, nil
+		}
+		return -1, ErrCopyDirToFile
+	}
+
+	if dstMode.IsDir() {
+		return CopyModeFileToDir, nil
+	}
+
+	if srcFile != dstFile {
+		return CopyModeFileToFileRename, nil
+	}
+
+	return CopyModeFileToFile, nil
+}
+
+// moveDir moves all files from a source path to the destination path recursively.
+func moveDir(sourcePath, destPath string) error {
+	return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath))
+		if info.IsDir() {
+			err := os.Mkdir(newPath, info.Mode())
+			if err != nil {
+				if os.IsExist(err) {
+					return nil
+				}
+				return err
+			}
+		}
+		if info.Mode().IsRegular() {
+			return moveFile(p, newPath)
+		}
+		return nil
+	})
+}
+
+// moveFile moves a file from a source path to a destination path.
+func moveFile(sourcePath, destPath string) error {
+	inputFile, err := os.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	outputFile, err := os.Create(destPath)
+	if err != nil {
+		inputFile.Close()
+		return err
+	}
+	defer outputFile.Close()
+	_, err = io.Copy(outputFile, inputFile)
+	inputFile.Close()
+	if err != nil {
+		return err
+	}
+
+	// Remove file after succesfull copy.
+	err = os.Remove(sourcePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/cli/app/cp_test.go b/cli/app/cp_test.go
new file mode 100644
index 00000000..e1fef30f
--- /dev/null
+++ b/cli/app/cp_test.go
@@ -0,0 +1,113 @@
+package app
+
+import (
+	"os"
+	"testing"
+)
+
+func TestParse(t *testing.T) {
+	tests := []struct {
+		src         string
+		dst         string
+		srcPath     string
+		dstPath     string
+		service     string
+		toContainer bool
+		err         error
+	}{
+		{src: "foo", dst: "bar", err: errServiceMissing},
+		{src: "app:foo", dst: "app:bar", err: errServiceMissing},
+		{src: "app:foo", dst: "bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: false},
+		{src: "foo", dst: "app:bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: true},
+	}
+
+	for i, tc := range tests {
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(tc.src, tc.dst)
+		if srcPath != tc.srcPath {
+			t.Errorf("[%d] srcPath: want (%s), got(%s)", i, tc.srcPath, srcPath)
+		}
+		if dstPath != tc.dstPath {
+			t.Errorf("[%d] dstPath: want (%s), got(%s)", i, tc.dstPath, dstPath)
+		}
+		if service != tc.service {
+			t.Errorf("[%d] service: want (%s), got(%s)", i, tc.service, service)
+		}
+		if toContainer != tc.toContainer {
+			t.Errorf("[%d] toConainer: want (%t), got(%t)", i, tc.toContainer, toContainer)
+		}
+		if err == nil && tc.err != nil && err.Error() != tc.err.Error() {
+			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
+		}
+	}
+}
+
+func TestCopyMode(t *testing.T) {
+	tests := []struct {
+		srcPath   string
+		dstPath   string
+		srcMode   os.FileMode
+		dstMode   os.FileMode
+		dstExists bool
+		mode      CopyMode
+		err       error
+	}{
+		{
+			srcPath:   "foo.txt",
+			dstPath:   "foo.txt",
+			srcMode:   os.ModePerm,
+			dstMode:   os.ModePerm,
+			dstExists: true,
+			mode:      CopyModeFileToFile,
+		},
+		{
+			srcPath:   "foo.txt",
+			dstPath:   "bar.txt",
+			srcMode:   os.ModePerm,
+			dstExists: true,
+			mode:      CopyModeFileToFileRename,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModeDir,
+			dstExists: true,
+			mode:      CopyModeDirToDir,
+		},
+		{
+			srcPath:   "foo/",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModeDir,
+			dstExists: true,
+			mode:      CopyModeFilesToDir,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstExists: false,
+			mode:      -1,
+			err:       ErrDstDirNotExist,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModePerm,
+			dstExists: true,
+			mode:      -1,
+			err:       ErrCopyDirToFile,
+		},
+	}
+
+	for i, tc := range tests {
+		mode, err := copyMode(tc.srcPath, tc.dstPath, tc.srcMode, tc.dstMode, tc.dstExists)
+		if mode != tc.mode {
+			t.Errorf("[%d] mode: want (%d), got(%d)", i, tc.mode, mode)
+		}
+		if err != tc.err {
+			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
+		}
+	}
+}
diff --git a/go.sum b/go.sum
index d767d61e..a891fa08 100644
--- a/go.sum
+++ b/go.sum
@@ -1315,6 +1315,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
diff --git a/pkg/container/container.go b/pkg/container/container.go
index 3bdf46f2..09d5703b 100644
--- a/pkg/container/container.go
+++ b/pkg/container/container.go
@@ -68,3 +68,15 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 
 	return containers[0], nil
 }
+
+// GetContainerFromStackAndService retrieves the container for the given stack and service.
+func GetContainerFromStackAndService(cl *client.Client, stack, service string) (types.Container, error) {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", stack, service))
+
+	container, err := GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return types.Container{}, err
+	}
+	return container, nil
+}
diff --git a/tests/integration/app_cp.bats b/tests/integration/app_cp.bats
index 2687e9d4..4c1f52a0 100644
--- a/tests/integration/app_cp.bats
+++ b/tests/integration/app_cp.bats
@@ -5,9 +5,11 @@ setup_file(){
   _common_setup
   _add_server
   _new_app
+  _deploy_app
 }
 
 teardown_file(){
+  _undeploy_app
   _rm_app
   _rm_server
 }
@@ -17,11 +19,29 @@ setup(){
   _common_setup
 }
 
-teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+_mkfile() {
+  run bash -c "echo $2 > $1"
+  assert_success
+}
+
+_mkfile_remote() {
+  run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\""
+  assert_success
+}
+
+_mkdir() {
+  run bash -c "mkdir -p $1"
+  assert_success
+}
+
+_rm() {
+  run rm -rf "$1"
+  assert_success
+}
+
+_rm_remote() {
+  run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1"
+  assert_success
 }
 
 @test "validate app argument" {
@@ -54,68 +74,120 @@ teardown(){
   assert_output --partial 'arguments must take $SERVICE:$PATH form'
 }
 
-@test "detect 'coming FROM' syntax" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug
-  assert_failure
-  assert_output --partial 'coming FROM the container'
-}
-
-@test "detect 'going TO' syntax" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug
-  assert_failure
-  assert_output --partial 'going TO the container'
-}
-
 @test "error if local file missing" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere
+  run $ABRA app cp "$TEST_APP_DOMAIN" thisfileshouldnotexist.txt app:/somewhere
   assert_failure
-  assert_output --partial 'myfile.txt does not exist locally?'
+  assert_output --partial 'local stat thisfileshouldnotexist.txt: no such file or directory'
 }
 
 # bats test_tags=slow
 @test "error if service doesn't exist" {
-  _deploy_app
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
 
-  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
-  assert_success
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ --debug
   assert_failure
   assert_output --partial 'no containers matching'
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
-  assert_success
-
-  _undeploy_app
+  _rm "$BATS_TMPDIR/myfile.txt"
 }
 
 # bats test_tags=slow
-@test "copy to container" {
-  _deploy_app
-
-  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
-  assert_success
+@test "copy local file to container directory" {
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
 
   run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
   assert_success
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
   assert_success
+  assert_output --partial "foo"
 
-  _undeploy_app
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
-@test "copy from container" {
-  _deploy_app
+@test "copy local file to container file (and override on remote)" {
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
 
-  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  # create
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
   assert_success
 
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  assert_success
+  assert_output --partial "foo"
+
+  _mkfile "$BATS_TMPDIR/myfile.txt" "bar"
+
+  # override
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
   assert_success
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  assert_success
+  assert_output --partial "bar"
+
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy local file to container file (and rename)" {
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+
+  # rename
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile2.txt
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile2.txt
+  assert_success
+  assert_output --partial "foo"
+
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile2.txt"
+}
+
+# bats test_tags=slow
+@test "copy local directory to container directory (and creates missing directory)" {
+  _mkdir "$BATS_TMPDIR/mydir"
+  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir" app:/etc
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/mydir
+  assert_success
+  assert_output --partial "myfile.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/mydir"
+}
+
+# bats test_tags=slow
+@test "copy local files to container directory" {
+  _mkdir "$BATS_TMPDIR/mydir"
+  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
+  _mkfile "$BATS_TMPDIR/mydir/myfile2.txt" "foo"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir/" app:/etc
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile.txt
+  assert_success
+  assert_output --partial "myfile.txt"
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile2.txt
+  assert_success
+  assert_output --partial "myfile2.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/myfile*"
+}
+
+# bats test_tags=slow
+@test "copy container file to local directory" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
   assert_success
 
   run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR"
@@ -123,8 +195,76 @@ teardown(){
   assert_exists "$BATS_TMPDIR/myfile.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy container file to local file" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
   assert_success
 
-  _undeploy_app
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/myfile.txt"
+  assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
+
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy container file to local file and rename" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile2.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/myfile2.txt"
+  assert bash -c "cat $BATS_TMPDIR/myfile2.txt | grep -q foo"
+
+  _rm "$BATS_TMPDIR/myfile2.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy container directory to local directory" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
+  assert_success
+
+  mkdir "$BATS_TMPDIR/mydir"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc "$BATS_TMPDIR/mydir"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/etc/myfile.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/etc/myfile2.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/myfile.txt"
+  _rm_remote "/etc/myfile2.txt"
+}
+
+# bats test_tags=slow
+@test "copy container files to local directory" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
+  assert_success
+
+  mkdir "$BATS_TMPDIR/mydir"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/ "$BATS_TMPDIR/mydir"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/myfile.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/myfile2.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/myfile.txt"
+  _rm_remote "/etc/myfile2.txt"
 }

From 02b726db02896d79870f20d49996ff0db8e1ca8c Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Mon, 4 Dec 2023 09:29:56 +0100
Subject: [PATCH 56/76] add comments to better explain how the length modifier
 gets added to the secret

---
 pkg/secret/secret.go | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go
index 947808e7..34c1103c 100644
--- a/pkg/secret/secret.go
+++ b/pkg/secret/secret.go
@@ -78,6 +78,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 	if err != nil {
 		return nil, err
 	}
+	// Read the compose files without injecting environment variables.
 	configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
 	if err != nil {
 		return nil, err
@@ -111,12 +112,16 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 		value := SecretValue{Version: secretVersion}
 
 		// Check if the length modifier is set for this secret.
-		for k, v := range appModifiers {
+		for envName, modifierValues := range appModifiers {
 			// configWithoutEnv contains the raw name as defined in the compose.yaml
-			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
+			// The name will look something like this:
+			//   name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
+			// To check if the current modifier is for the current secret we check
+			// if the raw name contains the env name (e.g. SECRET_TEST_PASS_TWO_VERSION).
+			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, envName) {
 				continue
 			}
-			lengthRaw, ok := v["length"]
+			lengthRaw, ok := modifierValues["length"]
 			if ok {
 				length, err := strconv.Atoi(lengthRaw)
 				if err != nil {

From 99e3ed416fb154db565760935dbe5717a3376454 Mon Sep 17 00:00:00 2001
From: test <test@example.com>
Date: Mon, 4 Dec 2023 14:37:41 +0100
Subject: [PATCH 57/76] fix: secret name generation when secretId is not part
 of the secret name

---
 cli/app/new.go                  | 14 +++-------
 cli/app/secret.go               |  8 +++---
 pkg/config/app.go               | 19 +++++++++-----
 pkg/secret/secret.go            | 46 +++++++++++++++++++++------------
 pkg/secret/secret_test.go       | 40 ++++++++++------------------
 pkg/secret/testdir/.env.sample  |  3 +++
 pkg/secret/testdir/compose.yaml | 21 +++++++++++++++
 7 files changed, 89 insertions(+), 62 deletions(-)
 create mode 100644 pkg/secret/testdir/.env.sample
 create mode 100644 pkg/secret/testdir/compose.yaml

diff --git a/cli/app/new.go b/cli/app/new.go
index 975ca502..22f51fd2 100644
--- a/cli/app/new.go
+++ b/cli/app/new.go
@@ -108,7 +108,7 @@ var appNewCommand = cli.Command{
 			}
 
 			envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
+			secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, config.StackName(internal.Domain))
 			if err != nil {
 				return err
 			}
@@ -168,14 +168,8 @@ var appNewCommand = cli.Command{
 type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
-	// NOTE(d1): trim to match app.StackName() implementation
-	if len(sanitisedAppName) > 45 {
-		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
-		sanitisedAppName = sanitisedAppName[:45]
-	}
-
-	secrets, err := secret.GenerateSecrets(cl, secretsConfig, sanitisedAppName, internal.NewAppServer)
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) {
+	secrets, err := secret.GenerateSecrets(cl, secretsConfig, internal.NewAppServer)
 	if err != nil {
 		return nil, err
 	}
@@ -217,7 +211,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 }
 
 // promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
+func promptForSecrets(recipeName string, secretsConfig map[string]secret.Secret) error {
 	if len(secretsConfig) == 0 {
 		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
 		return nil
diff --git a/cli/app/secret.go b/cli/app/secret.go
index 2b2f7fd4..3b491055 100644
--- a/cli/app/secret.go
+++ b/cli/app/secret.go
@@ -91,7 +91,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -104,7 +104,7 @@ var appSecretGenerateCommand = cli.Command{
 				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
 			s.Version = secretVersion
-			secrets = map[string]secret.SecretValue{
+			secrets = map[string]secret.Secret{
 				secretName: s,
 			}
 		}
@@ -114,7 +114,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
+		secretVals, err := secret.GenerateSecrets(cl, secrets, app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -274,7 +274,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName())
 		if err != nil {
 			logrus.Fatal(err)
 		}
diff --git a/pkg/config/app.go b/pkg/config/app.go
index 996c5357..b3efed2a 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -50,23 +50,30 @@ type App struct {
 	Path   string
 }
 
-// StackName gets whatever the docker safe (uses the right delimiting
-// character, e.g. "_") stack name is for the app. In general, you don't want
-// to use this to show anything to end-users, you want use a.Name instead.
+// See documentation of config.StackName
 func (a App) StackName() string {
 	if _, exists := a.Env["STACK_NAME"]; exists {
 		return a.Env["STACK_NAME"]
 	}
 
-	stackName := SanitiseAppName(a.Name)
+	stackName := StackName(a.Name)
+
+	a.Env["STACK_NAME"] = stackName
+
+	return stackName
+}
+
+// StackName gets whatever the docker safe (uses the right delimiting
+// character, e.g. "_") stack name is for the app. In general, you don't want
+// to use this to show anything to end-users, you want use a.Name instead.
+func StackName(appName string) string {
+	stackName := SanitiseAppName(appName)
 
 	if len(stackName) > 45 {
 		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:45])
 		stackName = stackName[:45]
 	}
 
-	a.Env["STACK_NAME"] = stackName
-
 	return stackName
 }
 
diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go
index 34c1103c..282a0fab 100644
--- a/pkg/secret/secret.go
+++ b/pkg/secret/secret.go
@@ -21,11 +21,24 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
-// secret definition.
-type SecretValue struct {
+// Secret represents a secret.
+type Secret struct {
+	// Version comes from the secret version environment variable.
+	// For example:
+	//  SECRET_FOO=v1
 	Version string
-	Length  int
+	// Length comes from the length modifier at the secret version environment
+	// variable. For Example:
+	//   SECRET_FOO=v1 # length=12
+	Length int
+	// RemoteName is the name of the secret on the server. For example:
+	//   name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
+	// With the following:
+	//   STACK_NAME=test_example_com
+	//   SECRET_TEST_PASS_TWO_VERSION=v2
+	// Will have this remote name:
+	//   test_example_com_test_pass_two_v2
+	RemoteName string
 }
 
 // GeneratePasswords generates passwords.
@@ -67,11 +80,13 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) {
 	appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
 	if err != nil {
 		return nil, err
 	}
+	// Set the STACK_NAME to be able to generate the remote name correctly.
+	appEnv["STACK_NAME"] = stackName
 
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
@@ -96,7 +111,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 		return nil, nil
 	}
 
-	secretValues := map[string]SecretValue{}
+	secretValues := map[string]Secret{}
 	for secretId, secretConfig := range config.Secrets {
 		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
 			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
@@ -109,7 +124,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 
 		lastIdx := strings.LastIndex(secretConfig.Name, "_")
 		secretVersion := secretConfig.Name[lastIdx+1:]
-		value := SecretValue{Version: secretVersion}
+		value := Secret{Version: secretVersion, RemoteName: secretConfig.Name}
 
 		// Check if the length modifier is set for this secret.
 		for envName, modifierValues := range appModifiers {
@@ -138,7 +153,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
+func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server string) (map[string]string, error) {
 	secretsGenerated := map[string]string{}
 	var mutex sync.Mutex
 	var wg sync.WaitGroup
@@ -146,11 +161,10 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, ap
 	for n, v := range secrets {
 		wg.Add(1)
 
-		go func(secretName string, secret SecretValue) {
+		go func(secretName string, secret Secret) {
 			defer wg.Done()
 
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
-			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
+			logrus.Debugf("attempting to generate and store %s on %s", secret.RemoteName, server)
 
 			if secret.Length > 0 {
 				passwords, err := GeneratePasswords(1, uint(secret.Length))
@@ -159,9 +173,9 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, ap
 					return
 				}
 
-				if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil {
+				if err := client.StoreSecret(cl, secret.RemoteName, passwords[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
+						logrus.Warnf("%s already exists, moving on...", secret.RemoteName)
 						ch <- nil
 					} else {
 						ch <- err
@@ -179,9 +193,9 @@ func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, ap
 					return
 				}
 
-				if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil {
+				if err := client.StoreSecret(cl, secret.RemoteName, passphrases[0], server); err != nil {
 					if strings.Contains(err.Error(), "AlreadyExists") {
-						logrus.Warnf("%s already exists, moving on...", secretRemoteName)
+						logrus.Warnf("%s already exists, moving on...", secret.RemoteName)
 						ch <- nil
 					} else {
 						ch <- err
@@ -230,7 +244,7 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		return secStats, err
 	}
 
-	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+	secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.StackName())
 	if err != nil {
 		return secStats, err
 	}
diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go
index bb462a8a..fc10c098 100644
--- a/pkg/secret/secret_test.go
+++ b/pkg/secret/secret_test.go
@@ -1,42 +1,30 @@
 package secret
 
 import (
-	"path"
 	"testing"
 
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/stack"
-	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestReadSecretsConfig(t *testing.T) {
-	offline := true
-	recipe, err := recipe.Get("matrix-synapse", offline)
+	composeFiles := []string{"./testdir/compose.yaml"}
+	secretsFromConfig, err := ReadSecretsConfig("./testdir/.env.sample", composeFiles, "test_example_com")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	sampleEnv, err := recipe.SampleEnv()
-	if err != nil {
-		t.Fatal(err)
-	}
+	// Simple secret
+	assert.Equal(t, "test_example_com_test_pass_one_v2", secretsFromConfig["test_pass_one"].RemoteName)
+	assert.Equal(t, "v2", secretsFromConfig["test_pass_one"].Version)
+	assert.Equal(t, 0, secretsFromConfig["test_pass_one"].Length)
 
-	composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")}
-	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	secretsFromConfig, err := ReadSecretsConfig(envSamplePath, composeFiles, recipe.Name)
-	if err != nil {
-		t.Fatal(err)
-	}
+	// Has a length modifier
+	assert.Equal(t, "test_example_com_test_pass_two_v1", secretsFromConfig["test_pass_two"].RemoteName)
+	assert.Equal(t, "v1", secretsFromConfig["test_pass_two"].Version)
+	assert.Equal(t, 10, secretsFromConfig["test_pass_two"].Length)
 
-	opts := stack.Deploy{Composefiles: composeFiles}
-	config, err := loader.LoadComposefile(opts, sampleEnv)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for secretId := range config.Secrets {
-		assert.Contains(t, secretsFromConfig, secretId)
-	}
+	// Secret name does not include the secret id
+	assert.Equal(t, "test_example_com_pass_three_v2", secretsFromConfig["test_pass_three"].RemoteName)
+	assert.Equal(t, "v2", secretsFromConfig["test_pass_three"].Version)
+	assert.Equal(t, 0, secretsFromConfig["test_pass_three"].Length)
 }
diff --git a/pkg/secret/testdir/.env.sample b/pkg/secret/testdir/.env.sample
new file mode 100644
index 00000000..bbcc2904
--- /dev/null
+++ b/pkg/secret/testdir/.env.sample
@@ -0,0 +1,3 @@
+SECRET_TEST_PASS_ONE_VERSION=v2
+SECRET_TEST_PASS_TWO_VERSION=v1 # length=10
+SECRET_TEST_PASS_THREE_VERSION=v2
diff --git a/pkg/secret/testdir/compose.yaml b/pkg/secret/testdir/compose.yaml
new file mode 100644
index 00000000..86cc2575
--- /dev/null
+++ b/pkg/secret/testdir/compose.yaml
@@ -0,0 +1,21 @@
+---
+version: "3.8"
+
+services:
+  app:
+    image: nginx:1.21.0
+    secrets:
+      - test_pass_one
+      - test_pass_two
+      - test_pass_three
+
+secrets:
+  test_pass_one:
+    external: true
+    name: ${STACK_NAME}_test_pass_one_${SECRET_TEST_PASS_ONE_VERSION}  # should be removed
+  test_pass_two:
+    external: true
+    name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION}
+  test_pass_three:
+    external: true
+    name: ${STACK_NAME}_pass_three_${SECRET_TEST_PASS_THREE_VERSION} # secretId and name don't match

From cdd7516e5499abe9fde7e3fe1260b6ca3483d255 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 4 Dec 2023 22:56:58 +0100
Subject: [PATCH 58/76] chore: go mod tidy [ci skip]

---
 go.sum | 1 -
 1 file changed, 1 deletion(-)

diff --git a/go.sum b/go.sum
index a891fa08..d767d61e 100644
--- a/go.sum
+++ b/go.sum
@@ -1315,7 +1315,6 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

From af8cd1f67a7b1d52d7b4b5ac88595795e747708a Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Tue, 12 Dec 2023 14:46:20 +0000
Subject: [PATCH 59/76] feat: abra release now asks for a release note (!393)

This implements https://git.coopcloud.tech/coop-cloud/organising/issues/540 by checking if a`release/next` file exists and if so moves it to `release/<tag>`. When no release notes exists it prompts for them.

Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/393
Reviewed-by: moritz <moritz.m@local-it.org>
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 cli/recipe/release.go                 | 88 ++++++++++++++++++++++++++-
 pkg/git/add.go                        | 27 ++++++++
 tests/integration/app_cp.bats         | 25 --------
 tests/integration/helpers/common.bash |  1 +
 tests/integration/helpers/file.bash   | 24 ++++++++
 tests/integration/helpers/git.bash    |  7 +++
 tests/integration/recipe_release.bats | 28 +++++++--
 7 files changed, 168 insertions(+), 32 deletions(-)
 create mode 100644 pkg/git/add.go
 create mode 100644 tests/integration/helpers/file.bash

diff --git a/cli/recipe/release.go b/cli/recipe/release.go
index 74cc83bd..d2da8230 100644
--- a/cli/recipe/release.go
+++ b/cli/recipe/release.go
@@ -1,7 +1,9 @@
 package recipe
 
 import (
+	"errors"
 	"fmt"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -140,7 +142,7 @@ your SSH keys configured on your account.
 
 // getImageVersions retrieves image versions for a recipe
 func getImageVersions(recipe recipe.Recipe) (map[string]string, error) {
-	var services = make(map[string]string)
+	services := make(map[string]string)
 
 	missingTag := false
 	for _, service := range recipe.Config.Services {
@@ -207,6 +209,10 @@ func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string
 		tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion)
 	}
 
+	if err := addReleaseNotes(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
 	if err := commitRelease(recipe, tagString); err != nil {
 		logrus.Fatal(err)
 	}
@@ -237,6 +243,82 @@ func getTagCreateOptions(tag string) (git.CreateTagOptions, error) {
 	return git.CreateTagOptions{Message: msg}, nil
 }
 
+// addReleaseNotes checks if the release/next release note exists and moves the
+// file to release/<tag>.
+func addReleaseNotes(recipe recipe.Recipe, tag string) error {
+	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
+	tagReleaseNotePath := path.Join(repoPath, "release", tag)
+	if _, err := os.Stat(tagReleaseNotePath); err == nil {
+		// Release note for current tag already exist exists.
+		return nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	nextReleaseNotePath := path.Join(repoPath, "release", "next")
+	if _, err := os.Stat(nextReleaseNotePath); err == nil {
+		// release/next note exists. Move it to release/<tag>
+		if internal.Dry {
+			logrus.Debugf("dry run: move release note from 'next' to %s", tag)
+			return nil
+		}
+		if !internal.NoInput {
+			prompt := &survey.Input{
+				Message: "Use release note in release/next?",
+			}
+			var addReleaseNote bool
+			if err := survey.AskOne(prompt, &addReleaseNote); err != nil {
+				return err
+			}
+			if !addReleaseNote {
+				return nil
+			}
+		}
+		err := os.Rename(nextReleaseNotePath, tagReleaseNotePath)
+		if err != nil {
+			return err
+		}
+		err = gitPkg.Add(repoPath, path.Join("release", "next"), internal.Dry)
+		if err != nil {
+			return err
+		}
+		err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry)
+		if err != nil {
+			return err
+		}
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	// No release note exists for the current release.
+	if internal.NoInput {
+		return nil
+	}
+
+	prompt := &survey.Input{
+		Message: "Release Note (leave empty for no release note)",
+	}
+	var releaseNote string
+	if err := survey.AskOne(prompt, &releaseNote); err != nil {
+		return err
+	}
+
+	if releaseNote == "" {
+		return nil
+	}
+
+	err := os.WriteFile(tagReleaseNotePath, []byte(releaseNote), 0o644)
+	if err != nil {
+		return err
+	}
+	err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func commitRelease(recipe recipe.Recipe, tag string) error {
 	if internal.Dry {
 		logrus.Debugf("dry run: no changes committed")
@@ -404,6 +486,10 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip
 		}
 	}
 
+	if err := addReleaseNotes(recipe, tagString); err != nil {
+		logrus.Fatal(err)
+	}
+
 	if err := commitRelease(recipe, tagString); err != nil {
 		logrus.Fatalf("failed to commit changes: %s", err.Error())
 	}
diff --git a/pkg/git/add.go b/pkg/git/add.go
new file mode 100644
index 00000000..36668568
--- /dev/null
+++ b/pkg/git/add.go
@@ -0,0 +1,27 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// Add adds a file to the git index.
+func Add(repoPath, path string, dryRun bool) error {
+	repo, err := git.PlainOpen(repoPath)
+	if err != nil {
+		return err
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	if dryRun {
+		logrus.Debugf("dry run: adding %s", path)
+	} else {
+		worktree.Add(path)
+	}
+
+	return nil
+}
diff --git a/tests/integration/app_cp.bats b/tests/integration/app_cp.bats
index 4c1f52a0..524ff866 100644
--- a/tests/integration/app_cp.bats
+++ b/tests/integration/app_cp.bats
@@ -19,31 +19,6 @@ setup(){
   _common_setup
 }
 
-_mkfile() {
-  run bash -c "echo $2 > $1"
-  assert_success
-}
-
-_mkfile_remote() {
-  run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\""
-  assert_success
-}
-
-_mkdir() {
-  run bash -c "mkdir -p $1"
-  assert_success
-}
-
-_rm() {
-  run rm -rf "$1"
-  assert_success
-}
-
-_rm_remote() {
-  run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1"
-  assert_success
-}
-
 @test "validate app argument" {
   run $ABRA app cp
   assert_failure
diff --git a/tests/integration/helpers/common.bash b/tests/integration/helpers/common.bash
index a97d3e94..9f434436 100644
--- a/tests/integration/helpers/common.bash
+++ b/tests/integration/helpers/common.bash
@@ -5,6 +5,7 @@ _common_setup() {
   bats_load_library bats-assert
   bats_load_library bats-file
 
+  load "$PWD/tests/integration/helpers/file"
   load "$PWD/tests/integration/helpers/app"
   load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/recipe"
diff --git a/tests/integration/helpers/file.bash b/tests/integration/helpers/file.bash
new file mode 100644
index 00000000..5cb91869
--- /dev/null
+++ b/tests/integration/helpers/file.bash
@@ -0,0 +1,24 @@
+_mkfile() {
+  run bash -c "echo $2 > $1"
+  assert_success
+}
+
+_mkfile_remote() {
+  run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\""
+  assert_success
+}
+
+_mkdir() {
+  run bash -c "mkdir -p $1"
+  assert_success
+}
+
+_rm() {
+  run rm -rf "$1"
+  assert_success
+}
+
+_rm_remote() {
+  run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1"
+  assert_success
+}
diff --git a/tests/integration/helpers/git.bash b/tests/integration/helpers/git.bash
index f2654944..e11b76de 100644
--- a/tests/integration/helpers/git.bash
+++ b/tests/integration/helpers/git.bash
@@ -28,3 +28,10 @@ _reset_tags() {
   assert_success
   refute_output '0'
 }
+
+_set_git_author() {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.email test@example.com
+  assert_success
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.name test
+  assert_success
+}
diff --git a/tests/integration/recipe_release.bats b/tests/integration/recipe_release.bats
index 2ebe226c..4c10f691 100644
--- a/tests/integration/recipe_release.bats
+++ b/tests/integration/recipe_release.bats
@@ -15,6 +15,11 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
+  _set_git_author
+}
+
+teardown() {
+  _reset_recipe
 }
 
 @test "validate recipe argument" {
@@ -51,8 +56,6 @@ setup(){
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
   assert_output --partial '0.2.1+1.21.6'
-
-  _reset_recipe
 }
 
 # NOTE(d1): this test can't assert hardcoded versions since we upgrade a minor
@@ -81,8 +84,6 @@ setup(){
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
   assert_output --regexp '0\.3\.0\+1\.2.*'
-
-  _reset_recipe "$TEST_RECIPE"
 }
 
 @test "unknown files not committed" {
@@ -100,6 +101,21 @@ setup(){
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo
   assert_failure
   assert_output --partial "fatal: pathspec 'foo' did not match any files"
-
-  _reset_recipe
+}
+
+# NOTE: relies on 0.2.x being the last minor version
+@test "release with next release note" {
+  _mkfile "$ABRA_DIR/recipes/$TEST_RECIPE/release/next" "those are some release notes for the next release"
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add release/next
+  assert_success
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "added some release notes"
+  assert_success
+
+  run $ABRA recipe release  "$TEST_RECIPE" --no-input  --minor
+  assert_success
+  assert_output --partial 'no -p/--publish passed, not publishing'
+
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/next"
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0"
+  assert_file_contains "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0" "those are some release notes for the next release"
 }

From d4727db8f9c64189def520317d1c8c269d0e02c7 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Thu, 14 Dec 2023 13:15:24 +0000
Subject: [PATCH 60/76] feat: abra app logs shows task errors (!395)

The log command now checks for the ready state in the task list. If it is not ready. It shows the task logs. This might look like this:
```
ERRO[0000] Service abra-test-recipe_default_app: State rejected: No such image: ngaaaax:1.21.0
ERRO[0000] Service abra-test-recipe_default_app: State preparing:
ERRO[0000] Service abra-test-recipe_default_app: State rejected: No such image: ngaaaax:1.21.0
ERRO[0000] Service abra-test-recipe_default_app: State rejected: No such image: ngaaaax:1.21.0
ERRO[0000] Service abra-test-recipe_default_app: State rejected: No such image: ngaaaax:1.21.0
```

Closes https://git.coopcloud.tech/coop-cloud/organising/issues/518

Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/395
Reviewed-by: decentral1se <decentral1se@noreply.git.coopcloud.tech>
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 cli/app/logs.go   | 145 +++++++++++++++++++++-------------------------
 pkg/config/app.go |  57 +++++++++++-------
 2 files changed, 102 insertions(+), 100 deletions(-)

diff --git a/cli/app/logs.go b/cli/app/logs.go
index b1566f3f..b06634a2 100644
--- a/cli/app/logs.go
+++ b/cli/app/logs.go
@@ -2,75 +2,26 @@ package app
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"os"
+	"slices"
 	"sync"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/service"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var logOpts = types.ContainerLogsOptions{
-	ShowStderr: true,
-	ShowStdout: true,
-	Since:      "",
-	Until:      "",
-	Timestamps: true,
-	Follow:     true,
-	Tail:       "20",
-	Details:    false,
-}
-
-// stackLogs lists logs for all stack services
-func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) {
-	filters, err := app.Filters(true, false)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	serviceOpts := types.ServiceListOptions{Filters: filters}
-	services, err := client.ServiceList(context.Background(), serviceOpts)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	for _, service := range services {
-		wg.Add(1)
-		go func(s string) {
-			if internal.StdErrOnly {
-				logOpts.ShowStdout = false
-			}
-
-			logs, err := client.ServiceLogs(context.Background(), s, logOpts)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			defer logs.Close()
-
-			_, err = io.Copy(os.Stdout, logs)
-			if err != nil && err != io.EOF {
-				logrus.Fatal(err)
-			}
-		}(service.ID)
-	}
-
-	wg.Wait()
-
-	os.Exit(0)
-}
-
 var appLogsCommand = cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
@@ -105,46 +56,84 @@ var appLogsCommand = cli.Command{
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
-		logOpts.Since = internal.SinceLogs
-
 		serviceName := c.Args().Get(1)
-		if serviceName == "" {
-			logrus.Debugf("tailing logs for all %s services", app.Recipe)
-			stackLogs(c, app, cl)
-		} else {
-			logrus.Debugf("tailing logs for %s", serviceName)
-			if err := tailServiceLogs(c, cl, app, serviceName); err != nil {
-				logrus.Fatal(err)
-			}
+		serviceNames := []string{}
+		if serviceName != "" {
+			serviceNames = []string{serviceName}
+		}
+		err = tailLogs(cl, app, serviceNames)
+		if err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
 
-func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
-
-	chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput)
+// tailLogs prints logs for the given app with optional service names to be
+// filtered on. It also checks if the latest task is not runnning and then
+// prints the past tasks.
+func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) error {
+	f, err := app.Filters(true, false, serviceNames...)
 	if err != nil {
-		logrus.Fatal(err)
+		return err
 	}
 
-	if internal.StdErrOnly {
-		logOpts.ShowStdout = false
-	}
-
-	logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts)
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: f})
 	if err != nil {
-		logrus.Fatal(err)
+		return err
 	}
-	defer logs.Close()
 
-	_, err = io.Copy(os.Stdout, logs)
-	if err != nil && err != io.EOF {
-		logrus.Fatal(err)
+	var wg sync.WaitGroup
+	for _, service := range services {
+		filters := filters.NewArgs()
+		filters.Add("name", service.Spec.Name)
+		tasks, err := cl.TaskList(context.Background(), types.TaskListOptions{Filters: f})
+		if err != nil {
+			return err
+		}
+		if len(tasks) > 0 {
+			// Need to sort the tasks by the CreatedAt field in the inverse order.
+			// Otherwise they are in the reversed order and not sorted properly.
+			slices.SortFunc[[]swarm.Task](tasks, func(t1, t2 swarm.Task) int {
+				return int(t2.Meta.CreatedAt.Unix() - t1.Meta.CreatedAt.Unix())
+			})
+			lastTask := tasks[0].Status
+			if lastTask.State != swarm.TaskStateRunning {
+				for _, task := range tasks {
+					logrus.Errorf("[%s] %s State %s: %s", service.Spec.Name, task.Meta.CreatedAt.Format(time.RFC3339), task.Status.State, task.Status.Err)
+				}
+			}
+		}
+
+		// Collect the logs in a go routine, so the logs from all services are
+		// collected in parallel.
+		wg.Add(1)
+		go func(serviceID string) {
+			logs, err := cl.ServiceLogs(context.Background(), serviceID, types.ContainerLogsOptions{
+				ShowStderr: true,
+				ShowStdout: !internal.StdErrOnly,
+				Since:      internal.SinceLogs,
+				Until:      "",
+				Timestamps: true,
+				Follow:     true,
+				Tail:       "20",
+				Details:    false,
+			})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
 	}
 
+	// Wait for all log streams to be closed.
+	wg.Wait()
+
 	return nil
 }
diff --git a/pkg/config/app.go b/pkg/config/app.go
index b3efed2a..a0d3e869 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -77,14 +77,32 @@ func StackName(appName string) string {
 	return stackName
 }
 
-// Filters retrieves exact app filters for querying the container runtime. Due
-// to upstream issues, filtering works different depending on what you're
+// Filters retrieves app filters for querying the container runtime. By default
+// it filters on all services in the app. It is also possible to pass an
+// otional list of service names, which get filtered instead.
+//
+// Due to upstream issues, filtering works different depending on what you're
 // querying. So, for example, secrets don't work with regex! The caller needs
 // to implement their own validation that the right secrets are matched. In
 // order to handle these cases, we provide the `appendServiceNames` /
 // `exactMatch` modifiers.
-func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) {
+func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) {
 	filters := filters.NewArgs()
+	if len(services) > 0 {
+		for _, serviceName := range services {
+			filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch))
+		}
+		return filters, nil
+	}
+
+	if appendServiceNames {
+		f := fmt.Sprintf("%s", a.StackName())
+		if exactMatch {
+			f = fmt.Sprintf("^%s", f)
+		}
+		filters.Add("name", f)
+		return filters, nil
+	}
 
 	composeFiles, err := GetComposeFiles(a.Recipe, a.Env)
 	if err != nil {
@@ -98,28 +116,23 @@ func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error)
 	}
 
 	for _, service := range compose.Services {
-		var filter string
-
-		if appendServiceNames {
-			if exactMatch {
-				filter = fmt.Sprintf("^%s_%s", a.StackName(), service.Name)
-			} else {
-				filter = fmt.Sprintf("%s_%s", a.StackName(), service.Name)
-			}
-		} else {
-			if exactMatch {
-				filter = fmt.Sprintf("^%s", a.StackName())
-			} else {
-				filter = fmt.Sprintf("%s", a.StackName())
-			}
-		}
-
-		filters.Add("name", filter)
+		f := ServiceFilter(a.StackName(), service.Name, exactMatch)
+		filters.Add("name", f)
 	}
 
 	return filters, nil
 }
 
+// ServiceFilter creates a filter string for filtering a service in the docker
+// container runtime. When exact match is true, it uses regex to match the
+// string exactly.
+func ServiceFilter(stack, service string, exact bool) string {
+	if exact {
+		return fmt.Sprintf("^%s_%s", stack, service)
+	}
+	return fmt.Sprintf("%s_%s", stack, service)
+}
+
 // ByServer sort a slice of Apps
 type ByServer []App
 
@@ -340,7 +353,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error {
 		return fmt.Errorf("%s already exists?", appEnvPath)
 	}
 
-	err = ioutil.WriteFile(appEnvPath, envSample, 0664)
+	err = ioutil.WriteFile(appEnvPath, envSample, 0o664)
 	if err != nil {
 		return err
 	}
@@ -602,7 +615,7 @@ func GetLabel(compose *composetypes.Config, stackName string, label string) stri
 
 // GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value
 func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) {
-	var timeout = 50 // Default Timeout
+	timeout := 50 // Default Timeout
 	var err error = nil
 	if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" {
 		logrus.Debugf("timeout label: %s", timeoutLabel)

From ca91abbed9d0297fc49d9dad8012749217e269d5 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Fri, 22 Dec 2023 12:08:12 +0000
Subject: [PATCH 61/76] fix: correct append service name logic in Filters
 function (!396)

This fixes a regression introduced by #395

Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/396
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 go.mod                                     |  2 +-
 pkg/config/app.go                          |  4 +-
 pkg/config/app_test.go                     | 89 ++++++++++++++++++++++
 pkg/config/testdir/filtertest.env          |  2 +
 pkg/config/testdir/test-recipe/compose.yml |  6 ++
 5 files changed, 101 insertions(+), 2 deletions(-)
 create mode 100644 pkg/config/testdir/filtertest.env
 create mode 100644 pkg/config/testdir/test-recipe/compose.yml

diff --git a/go.mod b/go.mod
index 5287ec3d..950841f8 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/go-git/go-git/v5 v5.10.0
+	github.com/google/go-cmp v0.5.9
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
@@ -47,7 +48,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
diff --git a/pkg/config/app.go b/pkg/config/app.go
index a0d3e869..38b9ddc6 100644
--- a/pkg/config/app.go
+++ b/pkg/config/app.go
@@ -95,7 +95,9 @@ func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (f
 		return filters, nil
 	}
 
-	if appendServiceNames {
+	// When not appending the service name, just add one filter for the whole
+	// stack.
+	if !appendServiceNames {
 		f := fmt.Sprintf("%s", a.StackName())
 		if exactMatch {
 			f = fmt.Sprintf("^%s", f)
diff --git a/pkg/config/app_test.go b/pkg/config/app_test.go
index 94398a0c..0823f4e2 100644
--- a/pkg/config/app_test.go
+++ b/pkg/config/app_test.go
@@ -1,12 +1,15 @@
 package config_test
 
 import (
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"testing"
 
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -106,3 +109,89 @@ func TestGetComposeFilesError(t *testing.T) {
 		}
 	}
 }
+
+func TestFilters(t *testing.T) {
+	oldDir := config.RECIPES_DIR
+	config.RECIPES_DIR = "./testdir"
+	defer func() {
+		config.RECIPES_DIR = oldDir
+	}()
+
+	app, err := config.NewApp(config.AppEnv{
+		"DOMAIN": "test.example.com",
+		"RECIPE": "test-recipe",
+	}, "test_example_com", config.AppFile{
+		Path:   "./testdir/filtertest.end",
+		Server: "local",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := app.Filters(false, false)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f, map[string]map[string]bool{
+		"name": {
+			"test_example_com": true,
+		},
+	})
+
+	f2, err := app.Filters(false, true)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f2, map[string]map[string]bool{
+		"name": {
+			"^test_example_com": true,
+		},
+	})
+
+	f3, err := app.Filters(true, false)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f3, map[string]map[string]bool{
+		"name": {
+			"test_example_com_bar": true,
+			"test_example_com_foo": true,
+		},
+	})
+
+	f4, err := app.Filters(true, true)
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f4, map[string]map[string]bool{
+		"name": {
+			"^test_example_com_bar": true,
+			"^test_example_com_foo": true,
+		},
+	})
+
+	f5, err := app.Filters(false, false, "foo")
+	if err != nil {
+		t.Error(err)
+	}
+	compareFilter(t, f5, map[string]map[string]bool{
+		"name": {
+			"test_example_com_foo": true,
+		},
+	})
+}
+
+func compareFilter(t *testing.T, f1 filters.Args, f2 map[string]map[string]bool) {
+	t.Helper()
+	j1, err := f1.MarshalJSON()
+	if err != nil {
+		t.Error(err)
+	}
+	j2, err := json.Marshal(f2)
+	if err != nil {
+		t.Error(err)
+	}
+	if diff := cmp.Diff(string(j2), string(j1)); diff != "" {
+		t.Errorf("filters mismatch (-want +got):\n%s", diff)
+	}
+}
diff --git a/pkg/config/testdir/filtertest.env b/pkg/config/testdir/filtertest.env
new file mode 100644
index 00000000..9250f3b4
--- /dev/null
+++ b/pkg/config/testdir/filtertest.env
@@ -0,0 +1,2 @@
+RECIPE=test-recipe
+DOMAIN=test.example.com
diff --git a/pkg/config/testdir/test-recipe/compose.yml b/pkg/config/testdir/test-recipe/compose.yml
new file mode 100644
index 00000000..8232eca0
--- /dev/null
+++ b/pkg/config/testdir/test-recipe/compose.yml
@@ -0,0 +1,6 @@
+version: "3.8"
+services:
+  foo:
+    image: debian
+  bar:
+    image: debian

From c5687dfbd7413f87ad202a4a275da495edae4730 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sun, 1 Oct 2023 08:02:30 +0200
Subject: [PATCH 62/76] feat: backup revolution

See https://git.coopcloud.tech/coop-cloud/organising/issues/485
---
 cli/app/backup.go              | 564 +++++++++++++--------------------
 cli/app/cp.go                  |  16 +-
 cli/app/restore.go             | 197 ++----------
 cli/app/run.go                 |   2 +-
 cli/internal/backup.go         |  86 +++--
 cli/internal/command.go        |   4 +-
 pkg/config/env.go              |   2 +
 pkg/container/container.go     |   2 +-
 pkg/service/service.go         |  64 ++++
 pkg/upstream/container/exec.go |  23 +-
 10 files changed, 401 insertions(+), 559 deletions(-)

diff --git a/cli/app/backup.go b/cli/app/backup.go
index f9a239b1..f6c8d2ae 100644
--- a/cli/app/backup.go
+++ b/cli/app/backup.go
@@ -1,414 +1,296 @@
 package app
 
 import (
-	"archive/tar"
-	"context"
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/klauspost/pgzip"
+	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type backupConfig struct {
-	preHookCmd  string
-	postHookCmd string
-	backupPaths []string
+var snapshot string
+var snapshotFlag = &cli.StringFlag{
+	Name:        "snapshot, s",
+	Usage:       "Lists specific snapshot",
+	Destination: &snapshot,
 }
 
-var appBackupCommand = cli.Command{
-	Name:      "backup",
-	Aliases:   []string{"bk"},
-	Usage:     "Run app backup",
-	ArgsUsage: "<domain> [<service>]",
+var includePath string
+var includePathFlag = &cli.StringFlag{
+	Name:        "path, p",
+	Usage:       "Include path",
+	Destination: &includePath,
+}
+
+var resticRepo string
+var resticRepoFlag = &cli.StringFlag{
+	Name:        "repo, r",
+	Usage:       "Restic repository",
+	Destination: &resticRepo,
+}
+
+var appBackupListCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		snapshotFlag,
+		includePathFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	Usage:        "List all backups",
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app backup.
-
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		backupConfigs := make(map[string]backupConfig)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName != "" {
-			backupConfig, ok := backupConfigs[serviceName]
-			if !ok {
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-			logrus.Infof("running backup for the %s service", serviceName)
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
 
-			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if len(backupConfigs) == 0 {
-				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
-			}
-
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
-
-				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
+		if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
 
-// TimeStamp generates a file name friendly timestamp.
-func TimeStamp() string {
-	ts := time.Now().UTC().Format(time.RFC3339)
-	return strings.Replace(ts, ":", "-", -1)
-}
+var appBackupDownloadCommand = cli.Command{
+	Name:    "download",
+	Aliases: []string{"d"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+		includePathFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Download a backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-// runBackup does the actual backup logic.
-func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			logrus.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
+		}
+
+		remoteBackupDir := "/tmp/backup.tar.gz"
+		currentWorkingDir := "."
+		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Println("backup successfully downloaded to current working directory")
+
+		return nil
+	},
+}
+
+var appBackupCreateCommand = cli.Command{
+	Name:    "create",
+	Aliases: []string{"c"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		resticRepoFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Create a new backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
 			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-		}
-		defer content.Close()
 
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
 			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-		}
 
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
-	}
-
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
-
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
-
-	return nil
-}
-
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
-
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
-		}
-
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
-
-	return nil
-}
-
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
-
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
-		}
-	}
-
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
-		}
-	}
-
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
-
-	return nil
-}
-
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
-
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
-
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
 			}
-			break
 		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
 		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if resticRepo != "" {
+			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
+			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
 }
 
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
+var appBackupSnapshotsCommand = cli.Command{
+	Name:    "snapshots",
+	Aliases: []string{"s"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "List backup snapshots",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 
-	rc = fin
-	tr = tar.NewReader(rc)
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
-	return tr, rc, nil
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+var appBackupCommand = cli.Command{
+	Name:      "backup",
+	Aliases:   []string{"b"},
+	Usage:     "Manage app backups",
+	ArgsUsage: "<domain>",
+	Subcommands: []cli.Command{
+		appBackupListCommand,
+		appBackupSnapshotsCommand,
+		appBackupDownloadCommand,
+		appBackupCreateCommand,
+	},
 }
diff --git a/cli/app/cp.go b/cli/app/cp.go
index bfc2c789..f27b42ce 100644
--- a/cli/app/cp.go
+++ b/cli/app/cp.go
@@ -76,9 +76,9 @@ And if you want to copy that file back to your current working directory locally
 		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 		if toContainer {
-			err = copyToContainer(cl, container.ID, srcPath, dstPath)
+			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
 		} else {
-			err = copyFromContainer(cl, container.ID, srcPath, dstPath)
+			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
 		}
 		if err != nil {
 			logrus.Fatal(err)
@@ -106,9 +106,9 @@ func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service st
 	return "", "", "", false, errServiceMissing
 }
 
-// copyToContainer copies a file or directory from the local file system to the container.
+// CopyToContainer copies a file or directory from the local file system to the container.
 // See the possible copy modes and their documentation.
-func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := os.Stat(srcPath)
 	if err != nil {
 		return fmt.Errorf("local %s ", err)
@@ -140,7 +140,7 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -179,7 +179,7 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -194,9 +194,9 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 	return nil
 }
 
-// copyFromContainer copies a file or directory from the given container to the local file system.
+// CopyFromContainer copies a file or directory from the given container to the local file system.
 // See the possible copy modes and their documentation.
-func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
 	if err != nil {
 		if errdefs.IsNotFound(err) {
diff --git a/cli/app/restore.go b/cli/app/restore.go
index 1bf9c840..c80347f5 100644
--- a/cli/app/restore.go
+++ b/cli/app/restore.go
@@ -1,223 +1,82 @@
 package app
 
 import (
-	"context"
-	"errors"
 	"fmt"
-	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type restoreConfig struct {
-	preHookCmd  string
-	postHookCmd string
+var targetPath string
+var targetPathFlag = &cli.StringFlag{
+	Name:        "target, t",
+	Usage:       "Target path",
+	Destination: &targetPath,
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Run app restore",
-	ArgsUsage: "<domain> <service> <file>",
+	Usage:     "Restore an app backup",
+	ArgsUsage: "<domain> <service>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		targetPathFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app restore.
-
-Pre/post hook commands are defined in the recipe configuration. Abra reads this
-configuration and run the comands in the context of the service before
-restoring the backup.
-
-Unlike "abra app backup", restore must be run on a per-service basis. You can
-not restore all services in one go. Backup files produced by Abra are
-compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic, i.e. the backup will be restored to
-the path it was originally backed up from.
-
-Example:
-
-    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipe.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
-		}
-
-		backupPath := c.Args().Get(2)
-		if backupPath == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
-		}
-
-		if _, err := os.Stat(backupPath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s doesn't exist?", backupPath)
-			}
-		}
-
-		restoreConfigs := make(map[string]restoreConfig)
-		for _, service := range recipe.Config.Services {
-			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
-				if restoreEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					rsConfig := restoreConfig{}
-
-					logrus.Debugf("restore config detected for %s", fullServiceName)
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						rsConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						rsConfig.postHookCmd = postHookCmd
-					}
-
-					restoreConfigs[service.Name] = rsConfig
-				}
-			}
-		}
-
-		rsConfig, ok := restoreConfigs[serviceName]
-		if !ok {
-			rsConfig = restoreConfig{}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if targetPath != "" {
+			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
+			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
-
-// runRestore does the actual restore logic.
-func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if rsConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
-	}
-
-	backupReader, err := os.Open(backupPath)
-	if err != nil {
-		return err
-	}
-
-	content, err := archive.DecompressStream(backupReader)
-	if err != nil {
-		return err
-	}
-
-	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
-	// files according to the paths set in the compressed archive
-	restorePath := "/"
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
-		return err
-	}
-
-	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
-
-	if rsConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
-	}
-
-	return nil
-}
diff --git a/cli/app/run.go b/cli/app/run.go
index 4ae68c1b..b5e0a9ce 100644
--- a/cli/app/run.go
+++ b/cli/app/run.go
@@ -91,7 +91,7 @@ var appRunCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
diff --git a/cli/internal/backup.go b/cli/internal/backup.go
index 79951810..530735c9 100644
--- a/cli/internal/backup.go
+++ b/cli/internal/backup.go
@@ -1,35 +1,67 @@
 package internal
 
 import (
-	"strings"
+	"context"
+
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
 )
 
-// SafeSplit splits up a string into a list of commands safely.
-func SafeSplit(s string) []string {
-	split := strings.Split(s, " ")
-
-	var result []string
-	var inquote string
-	var block string
-	for _, i := range split {
-		if inquote == "" {
-			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
-				inquote = string(i[0])
-				block = strings.TrimPrefix(i, inquote) + " "
-			} else {
-				result = append(result, i)
-			}
-		} else {
-			if !strings.HasSuffix(i, inquote) {
-				block += i + " "
-			} else {
-				block += strings.TrimSuffix(i, inquote)
-				inquote = ""
-				result = append(result, block)
-				block = ""
-			}
-		}
+// RetrieveBackupBotContainer gets the deployed backupbot container.
+func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) {
+	ctx := context.Background()
+	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
+	if err != nil {
+		return types.Container{}, err
 	}
 
-	return result
+	logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+	filters := filters.NewArgs()
+	filters.Add("name", chosenService.Spec.Name)
+	targetContainer, err := containerPkg.GetContainer(
+		ctx,
+		cl,
+		filters,
+		NoInput,
+	)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	return targetContainer, nil
+}
+
+// RunBackupCmdRemote runs a backup related command on a remote backupbot container.
+func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error {
+	execBackupListOpts := types.ExecConfig{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          []string{"/usr/bin/backup", "--", backupCmd},
+		Detach:       false,
+		Env:          execEnv,
+		Tty:          true,
+	}
+
+	logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts)
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil {
+		return err
+	}
+
+	return nil
 }
diff --git a/cli/internal/command.go b/cli/internal/command.go
index 6f02ae1c..13c007be 100644
--- a/cli/internal/command.go
+++ b/cli/internal/command.go
@@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		Tty:          false,
 	}
 
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
 		shell = "/bin/sh"
 	}
@@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		execCreateOpts.Tty = false
 	}
 
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		return err
 	}
 
diff --git a/pkg/config/env.go b/pkg/config/env.go
index 62f6a71d..adedc6f4 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -36,6 +36,8 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+var BackupbotLabel = "coop-cloud.backupbot.enabled"
+
 // envVarModifiers is a list of env var modifier strings. These are added to
 // env vars as comments and modify their processing by Abra, e.g. determining
 // how long secrets should be.
diff --git a/pkg/container/container.go b/pkg/container/container.go
index 09d5703b..1354b0dd 100644
--- a/pkg/container/container.go
+++ b/pkg/container/container.go
@@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) != 1 {
+	if len(containers) > 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")
diff --git a/pkg/service/service.go b/pkg/service/service.go
index 3d92d821..48cdce75 100644
--- a/pkg/service/service.go
+++ b/pkg/service/service.go
@@ -14,6 +14,70 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// GetService retrieves a service container based on a label. If prompt is true
+// and the retrievd count of service containers does not match 1, then a prompt
+// is presented to let the user choose. An error is returned when no service is
+// found.
+func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
+	services, err := cl.ServiceList(c, types.ServiceListOptions{})
+	if err != nil {
+		return swarm.Service{}, err
+	}
+
+	if len(services) == 0 {
+		return swarm.Service{}, fmt.Errorf("no services deployed?")
+	}
+
+	var matchingServices []swarm.Service
+	for _, service := range services {
+		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
+			matchingServices = append(matchingServices, service)
+		}
+	}
+
+	if len(matchingServices) == 0 {
+		return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label)
+	}
+
+	if len(matchingServices) > 1 {
+		var servicesRaw []string
+		for _, service := range matchingServices {
+			serviceName := service.Spec.Name
+			created := formatter.HumanDuration(service.CreatedAt.Unix())
+			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
+		}
+
+		if !prompt {
+			err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " "))
+			return swarm.Service{}, err
+		}
+
+		logrus.Warnf("ambiguous service list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which service are you looking for?",
+			Options: servicesRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return swarm.Service{}, err
+		}
+
+		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, service := range matchingServices {
+			serviceName := strings.ToLower(service.Spec.Name)
+			if serviceName == chosenService {
+				return service, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen service")
+	}
+
+	return matchingServices[0], nil
+}
+
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.
diff --git a/pkg/upstream/container/exec.go b/pkg/upstream/container/exec.go
index e811481a..82a2c570 100644
--- a/pkg/upstream/container/exec.go
+++ b/pkg/upstream/container/exec.go
@@ -13,7 +13,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error {
+// RunExec runs a command on a remote container. io.Writer corresponds to the
+// command output.
+func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string,
+	execConfig *types.ExecConfig) (io.Writer, error) {
 	ctx := context.Background()
 
 	// We need to check the tty _before_ we do the ContainerExecCreate, because
@@ -21,22 +24,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 	// there's no easy way to clean those up). But also in order to make "not
 	// exist" errors take precedence we do a dummy inspect first.
 	if _, err := client.ContainerInspect(ctx, containerID); err != nil {
-		return err
+		return nil, err
 	}
 	if !execConfig.Detach {
 		if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	response, err := client.ContainerExecCreate(ctx, containerID, *execConfig)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	execID := response.ID
 	if execID == "" {
-		return errors.New("exec ID empty")
+		return nil, errors.New("exec ID empty")
 	}
 
 	if execConfig.Detach {
@@ -44,13 +47,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 			Detach: execConfig.Detach,
 			Tty:    execConfig.Tty,
 		}
-		return client.ContainerExecStart(ctx, execID, execStartCheck)
+		return nil, client.ContainerExecStart(ctx, execID, execStartCheck)
 	}
 	return interactiveExec(ctx, dockerCli, client, execConfig, execID)
 }
 
 func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client,
-	execConfig *types.ExecConfig, execID string) error {
+	execConfig *types.ExecConfig, execID string) (io.Writer, error) {
 	// Interactive exec requested.
 	var (
 		out, stderr io.Writer
@@ -76,7 +79,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 	}
 	resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck)
 	if err != nil {
-		return err
+		return out, err
 	}
 	defer resp.Close()
 
@@ -107,10 +110,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 
 	if err := <-errCh; err != nil {
 		logrus.Debugf("Error hijack: %s", err)
-		return err
+		return out, err
 	}
 
-	return getExecExitStatus(ctx, client, execID)
+	return out, getExecExitStatus(ctx, client, execID)
 }
 
 func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {

From 0a3624c15bc46607d2b2b7da33c91e26df4768cf Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Fri, 19 Jan 2024 15:08:41 +0000
Subject: [PATCH 63/76] feat: add version input to abra app new (!400)

Closes https://git.coopcloud.tech/coop-cloud/organising/issues/519

Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/400
Reviewed-by: decentral1se <decentral1se@noreply.git.coopcloud.tech>
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 cli/app/new.go                   | 24 ++++++++++++---
 pkg/autocomplete/autocomplete.go | 14 +++++++++
 tests/integration/app_new.bats   | 53 +++++++++++++++++++++++++-------
 3 files changed, 75 insertions(+), 16 deletions(-)

diff --git a/cli/app/new.go b/cli/app/new.go
index 22f51fd2..1a430dd3 100644
--- a/cli/app/new.go
+++ b/cli/app/new.go
@@ -54,9 +54,17 @@ var appNewCommand = cli.Command{
 		internal.OfflineFlag,
 		internal.ChaosFlag,
 	},
-	Before:       internal.SubCommandBefore,
-	ArgsUsage:    "[<recipe>]",
-	BashComplete: autocomplete.RecipeNameComplete,
+	Before:    internal.SubCommandBefore,
+	ArgsUsage: "[<recipe>] [<version>]",
+	BashComplete: func(ctx *cli.Context) {
+		args := ctx.Args()
+		switch len(args) {
+		case 0:
+			autocomplete.RecipeNameComplete(ctx)
+		case 1:
+			autocomplete.RecipeVersionComplete(ctx.Args().Get(0))
+		}
+	},
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
@@ -69,8 +77,14 @@ var appNewCommand = cli.Command{
 					logrus.Fatal(err)
 				}
 			}
-			if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
-				logrus.Fatal(err)
+			if c.Args().Get(1) == "" {
+				if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+					logrus.Fatal(err)
+				}
+			} else {
+				if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil {
+					logrus.Fatal(err)
+				}
 			}
 		}
 
diff --git a/pkg/autocomplete/autocomplete.go b/pkg/autocomplete/autocomplete.go
index 9d5075d6..7b28f134 100644
--- a/pkg/autocomplete/autocomplete.go
+++ b/pkg/autocomplete/autocomplete.go
@@ -51,6 +51,20 @@ func RecipeNameComplete(c *cli.Context) {
 	}
 }
 
+// RecipeVersionComplete completes versions for the recipe.
+func RecipeVersionComplete(recipeName string) {
+	catl, err := recipe.ReadRecipeCatalogue(false)
+	if err != nil {
+		logrus.Warn(err)
+	}
+
+	for _, v := range catl[recipeName].Versions {
+		for v2 := range v {
+			fmt.Println(v2)
+		}
+	}
+}
+
 // ServerNameComplete completes server names.
 func ServerNameComplete(c *cli.Context) {
 	files, err := config.LoadAppFiles("")
diff --git a/tests/integration/app_new.bats b/tests/integration/app_new.bats
index ac8c94fd..885e02cd 100644
--- a/tests/integration/app_new.bats
+++ b/tests/integration/app_new.bats
@@ -18,9 +18,24 @@ setup(){
 }
 
 teardown(){
+  load "$PWD/tests/integration/helpers/common"
   _rm_app
 }
 
+@test "autocomplete" {
+  run $ABRA app new --generate-bash-completion
+  assert_success
+  assert_output --partial "traefik"
+  assert_output --partial "abra-test-recipe"
+
+  # Note: this test needs to be updated when a new version of the test recipe is published.
+  run $ABRA app new abra-test-recipe --generate-bash-completion
+  assert_success
+  assert_output "0.1.0+1.20.0
+0.1.1+1.20.2
+0.2.0+1.21.0"
+}
+
 @test "create new app" {
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -28,10 +43,29 @@ teardown(){
     --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial "Your branch is up to date with 'origin/main'."
+}
+
+@test "create new app with version" {
+  run $ABRA app new "$TEST_RECIPE" 0.1.1+1.20.2 \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" log -1
+  assert_output --partial "453db7121c0a56a7a8f15378f18fe3bf21ccfdef"
 }
 
 @test "does not overwrite existing env files" {
-  _new_app
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN"
+  assert_success
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -74,8 +108,7 @@ teardown(){
     --no-input \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-    --secrets
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
@@ -88,18 +121,17 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-    --secrets
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --partial "Your branch is up to date with 'origin/main'."
 
   _reset_recipe
 }
@@ -109,7 +141,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app new "$TEST_RECIPE" \
@@ -117,13 +149,12 @@ teardown(){
     --offline \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-    --secrets
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
 
   _reset_recipe
 }

From 4920dfedb38d312e966d8cfab2fdaa6e19c57b71 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Fri, 19 Jan 2024 15:09:00 +0000
Subject: [PATCH 64/76] fix: retry docker volume remove (!399)

Closes https://git.coopcloud.tech/coop-cloud/organising/issues/509

Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/399
Reviewed-by: decentral1se <decentral1se@noreply.git.coopcloud.tech>
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 cli/app/remove.go                 | 26 ++++++++++++++++++++++++--
 cli/app/remove_test.go            | 26 ++++++++++++++++++++++++++
 tests/integration/app_remove.bats |  8 +-------
 3 files changed, 51 insertions(+), 9 deletions(-)
 create mode 100644 cli/app/remove_test.go

diff --git a/cli/app/remove.go b/cli/app/remove.go
index ea4efedf..82499fd1 100644
--- a/cli/app/remove.go
+++ b/cli/app/remove.go
@@ -3,7 +3,9 @@ package app
 import (
 	"context"
 	"fmt"
+	"log"
 	"os"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -124,9 +126,11 @@ flag.
 
 		if len(vols) > 0 {
 			for _, vol := range vols {
-				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				err = retryFunc(5, func() error {
+					return cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				})
 				if err != nil {
-					logrus.Fatal(err)
+					log.Fatalf("removing volumes failed: %s", err)
 				}
 				logrus.Info(fmt.Sprintf("volume %s removed", vol))
 			}
@@ -143,3 +147,21 @@ flag.
 		return nil
 	},
 }
+
+// retryFunc retries the given function for the given retries. After the nth
+// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
+// It returns an error if the function still failed after the last retry.
+func retryFunc(retries int, fn func() error) error {
+	for i := 0; i < retries; i++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		if i+1 < retries {
+			sleep := time.Duration(i+1) * time.Duration(i+1)
+			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
+			time.Sleep(sleep * time.Second)
+		}
+	}
+	return fmt.Errorf("%d retries failed", retries)
+}
diff --git a/cli/app/remove_test.go b/cli/app/remove_test.go
new file mode 100644
index 00000000..c3c9f8a0
--- /dev/null
+++ b/cli/app/remove_test.go
@@ -0,0 +1,26 @@
+package app
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestRetryFunc(t *testing.T) {
+	err := retryFunc(1, func() error { return nil })
+	if err != nil {
+		t.Errorf("should not return an error: %s", err)
+	}
+
+	i := 0
+	fn := func() error {
+		i++
+		return fmt.Errorf("oh no, something went wrong!")
+	}
+	err = retryFunc(2, fn)
+	if err == nil {
+		t.Error("should return an error")
+	}
+	if i != 2 {
+		t.Errorf("The function should have been called 1 times, got %d", i)
+	}
+}
diff --git a/tests/integration/app_remove.bats b/tests/integration/app_remove.bats
index 8b53984c..8800e70a 100644
--- a/tests/integration/app_remove.bats
+++ b/tests/integration/app_remove.bats
@@ -104,10 +104,7 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
-
-  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  run $ABRA app volume rm "$TEST_APP_DOMAIN"
   assert_success
 
   run $ABRA app volume ls "$TEST_APP_DOMAIN"
@@ -132,9 +129,6 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
-
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
   assert_output --partial 'test-volume'

From e9b99fe921891eb8ef115f9e6cd6e4ac3050e1f9 Mon Sep 17 00:00:00 2001
From: basebuilder <bases@eotl.supply>
Date: Tue, 16 Jan 2024 17:38:05 +0100
Subject: [PATCH 65/76] make installer save abra-download to /tmp/ directory

the current location of download is ~/.local/bin/ but this
conflicts with some security tools
---
 scripts/installer/installer | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/installer/installer b/scripts/installer/installer
index 2a73b442..01605942 100755
--- a/scripts/installer/installer
+++ b/scripts/installer/installer
@@ -65,17 +65,19 @@ function install_abra_release {
 
   checksums=$(wget -q -O- $checksums_url)
   checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p')
+  abra_download="/tmp/abra-download"
 
   echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..."
-  wget -q "$release_url" -O "$HOME/.local/bin/.abra-download"
-  localsum=$(sha256sum $HOME/.local/bin/.abra-download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
+
+  wget -q "$release_url" -O $abra_download 
+  localsum=$(sha256sum $abra_download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
   echo "checking if checksums match..."
   if [[ "$localsum" != "$checksum" ]]; then
       print_checksum_error
       exit 1
   fi
   echo "$(tput setaf 2)check successful!$(tput sgr0)"
-  mv "$HOME/.local/bin/.abra-download" "$HOME/.local/bin/abra"
+  mv "$abra_download" "$HOME/.local/bin/abra"
   chmod +x "$HOME/.local/bin/abra"
 
   x=$(echo $PATH | grep $HOME/.local/bin)

From 0643df6d73ae5464fa1c29e52c2c5fbb5ae97e30 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Wed, 24 Jan 2024 15:01:33 +0000
Subject: [PATCH 66/76] feat: fetch all recipes when no recipe is specified
 (!401)

Closes https://git.coopcloud.tech/coop-cloud/organising/issues/530

Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/401
Reviewed-by: decentral1se <decentral1se@noreply.git.coopcloud.tech>
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 cli/recipe/fetch.go                 | 22 ++++++++++++++--------
 pkg/recipe/recipe.go                | 14 ++++++++++++++
 tests/integration/recipe_fetch.bats | 12 +++++++++++-
 3 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/cli/recipe/fetch.go b/cli/recipe/fetch.go
index acfc1708..de81cfec 100644
--- a/cli/recipe/fetch.go
+++ b/cli/recipe/fetch.go
@@ -3,6 +3,7 @@ package recipe
 import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -17,26 +18,31 @@ var recipeFetchCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
-
 		if recipeName != "" {
 			internal.ValidateRecipe(c)
+			if err := recipe.Ensure(recipeName); err != nil {
+				logrus.Fatal(err)
+			}
+			return nil
 		}
 
-		if err := recipe.EnsureExists(recipeName); err != nil {
+		catalogue, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := recipe.EnsureUpToDate(recipeName); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := recipe.EnsureLatest(recipeName); err != nil {
-			logrus.Fatal(err)
+		catlBar := formatter.CreateProgressbar(len(catalogue), "fetching latest recipes...")
+		for recipeName := range catalogue {
+			if err := recipe.Ensure(recipeName); err != nil {
+				logrus.Error(err)
+			}
+			catlBar.Add(1)
 		}
 
 		return nil
diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go
index dbd4a520..a5d6d2eb 100644
--- a/pkg/recipe/recipe.go
+++ b/pkg/recipe/recipe.go
@@ -264,6 +264,20 @@ func (r Recipe) SampleEnv() (map[string]string, error) {
 	return sampleEnv, nil
 }
 
+// Ensure makes sure the recipe exists, is up to date and has the latest version checked out.
+func Ensure(recipeName string) error {
+	if err := EnsureExists(recipeName); err != nil {
+		return err
+	}
+	if err := EnsureUpToDate(recipeName); err != nil {
+		return err
+	}
+	if err := EnsureLatest(recipeName); err != nil {
+		return err
+	}
+	return nil
+}
+
 // EnsureExists ensures that a recipe is locally cloned
 func EnsureExists(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
diff --git a/tests/integration/recipe_fetch.bats b/tests/integration/recipe_fetch.bats
index a64306ab..1cc61e7c 100644
--- a/tests/integration/recipe_fetch.bats
+++ b/tests/integration/recipe_fetch.bats
@@ -5,7 +5,17 @@ setup() {
   _common_setup
 }
 
-@test "recipe fetch" {
+@test "recipe fetch all" {
+  run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"
+
+  run $ABRA recipe fetch
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/matrix-synapse"
+}
+
+@test "recipe fetch single recipe" {
   run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"

From 40c0fb4bac3dc288aa99bba3dbedb00cb76dab7f Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Mon, 11 Mar 2024 13:27:21 +0000
Subject: [PATCH 67/76] fix-integration-tests (!403)

In preparation for the new abra release, let's fix all integration tests

After merging, this needs to be cherry-picked into the release-0-9 branch.

  - [x] app_backup.bats (skip this one)
  - [x] app_check.bats (fixed by https://git.coopcloud.tech/coop-cloud/abra/commit/bd21014fedf6f8d4fb8f1baf44fb8b0df99fc326)
  - [x] app_cmd.bats (partially fixed in https://git.coopcloud.tech/coop-cloud/abra/commit/08232b74f6d512ab2ca7c08f67bb7c227b82a2ca), has known regression https://git.coopcloud.tech/coop-cloud/organising/issues/581
  - [x] app_config.bats (no changes needed)
  - [x] app_cp.bats (no changes needed)
  - [x] app_deploy.bats
  - [x] app_errors.bats (no changes needed)
  - [x] app_list.bats (no changes needed)
  - [x] app_logs.bats (no changes needed)
  - [x] app_new.bats (no changes needed)
  - [x] app_ps.bats (no changes needed)
  - [x] app_remove.bats (fixed by [2f29fbeb2e](https://git.coopcloud.tech/coop-cloud/abra/pulls/403/commits/2f29fbeb2e018656413fa25f8615b7a98cdcb083))
  - [x] app_restart.bats (no changes needed
  - [x] app_restore.bats (fixed by [f2dd5afc38](https://git.coopcloud.tech/coop-cloud/abra/pulls/403/commits/f2dd5afc38a25a8316899fa0c6d59499445868d7))
  - [x] app_rollback.bats (partially fixed by https://git.coopcloud.tech/coop-cloud/abra/commit/6e99b74c24e92a293a5f71d7aacd552441dc8613)
  - [x] app_run.bats (no changes needed)
  - [x] app_secret.bats (fixed by https://git.coopcloud.tech/coop-cloud/abra/commit/bd069d32f6d58f12cf8b0d2a1fef0e4cdc50d94d)
  - [x] app_services.bats (no changes needed)
  - [x] app_undeploy.bats (no changes needed)
  - [x] app_upgrade.bats (no changes needed)
  - [x] app_version.bats (partially fixed by https://git.coopcloud.tech/coop-cloud/abra/commit/ad323ad2bd6184c69baf909f891a8bf8ffc168a4)
  - [x] app_volume.bats (fixed by [03c3823770](https://git.coopcloud.tech/coop-cloud/abra/pulls/403/commits/03c38237707ae795b723180eb07a7edc84a8de35))
  - [x] autocomplete.bats (no changes needed)
  - [x] catalogue.bats (no changes needed)
  - [x] dirs.bats (no changes needed)
  - [x] install.bats (failes, but is expected)
  - [x] recipe_diff.bats (no changes needed)
  - [x] recipe_fetch.bats (no changes needed)
  - [x] recipe_lint.bats (fixed by [b6b0808066](https://git.coopcloud.tech/coop-cloud/abra/pulls/403/commits/b6b0808066a11e4bcd77517ec39600d500bcb944))
  - [x] recipe_list.bats (no changes needed)
  - [x] recipe_new.bats (fixed by [0aac464ded](https://git.coopcloud.tech/coop-cloud/abra/pulls/403/commits/0aac464ded6b43afb3ec37ade2f64d6191b9838f))
  - [x] recipe_release.bats (no changes needed)
  - [x] recipe_reset.bats (no changes needed)
  - [x] recipe_sync.bats (no changes needed)
  - [x] recipe_upgrade.bats (fixed by [ab86904cf4](https://git.coopcloud.tech/coop-cloud/abra/pulls/403/commits/ab86904cf45db89c7c189ca1fd9971909bd446dd))
  - [x] recipe_version.bats (fixed by https://git.coopcloud.tech/coop-cloud/abra/commit/81897bf4daf6d37f3997fdea0deafb1d8f241c6c)
  - [x] server_add.bats
  - [x] server_list.bats
  - [x] server_prune.bats (no changes needed)
  - [x] server_remove.bats
  - [x] upgrade.bats
  - [x] version.bats (no changes needed)

Co-authored-by: decentral1se <cellarspoon@riseup.net>
Reviewed-on: https://git.coopcloud.tech/coop-cloud/abra/pulls/403
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>
---
 scripts/installer/installer           |  2 +-
 tests/integration/app_check.bats      |  8 +++----
 tests/integration/app_cmd.bats        | 32 +++++++++++++--------------
 tests/integration/app_deploy.bats     | 18 ++++++++++-----
 tests/integration/app_new.bats        | 10 ++++-----
 tests/integration/app_remove.bats     |  5 ++++-
 tests/integration/app_restore.bats    | 14 ++++++------
 tests/integration/app_rollback.bats   | 10 ++++-----
 tests/integration/app_secret.bats     |  9 +-------
 tests/integration/app_version.bats    |  4 +++-
 tests/integration/app_volume.bats     |  4 ++--
 tests/integration/helpers/app.bash    |  2 +-
 tests/integration/helpers/server.bash |  6 ++++-
 tests/integration/recipe_lint.bats    |  8 +++----
 tests/integration/recipe_upgrade.bats |  4 ++--
 tests/integration/recipe_version.bats |  2 ++
 16 files changed, 75 insertions(+), 63 deletions(-)

diff --git a/scripts/installer/installer b/scripts/installer/installer
index 01605942..8b8ee5ec 100755
--- a/scripts/installer/installer
+++ b/scripts/installer/installer
@@ -2,7 +2,7 @@
 
 ABRA_VERSION="0.8.1-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.1-beta"
+RC_VERSION="0.8.0-rc1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do
diff --git a/tests/integration/app_check.bats b/tests/integration/app_check.bats
index 2ea628c9..e296e3cb 100644
--- a/tests/integration/app_check.bats
+++ b/tests/integration/app_check.bats
@@ -70,13 +70,13 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   _reset_recipe
 }
@@ -86,7 +86,7 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 1'
+  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
 
   # NOTE(d1): we can't quite tell if this will fail or not in the future, so,
   # since it isn't an important part of what we're testing here, we don't check
@@ -94,7 +94,7 @@ setup(){
   run $ABRA app check "$TEST_APP_DOMAIN" --offline
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 1'
+  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
 
   _reset_recipe
 }
diff --git a/tests/integration/app_cmd.bats b/tests/integration/app_cmd.bats
index db2808ba..fd5e6695 100644
--- a/tests/integration/app_cmd.bats
+++ b/tests/integration/app_cmd.bats
@@ -58,7 +58,7 @@ test_cmd_export"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 
@@ -70,7 +70,7 @@ test_cmd_export"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
   assert_failure
   assert_output --partial 'locally unstaged changes'
 
@@ -83,7 +83,7 @@ test_cmd_export"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
+  run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 
@@ -96,14 +96,14 @@ test_cmd_export"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --partial "up to date"
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -113,14 +113,14 @@ test_cmd_export"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --offline
+  run $ABRA app cmd --local --offline "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -132,13 +132,13 @@ test_cmd_export"
 }
 
 @test "error if missing arguments when passing --local" {
-  run $ABRA app cmd "$TEST_APP_DOMAIN" --local
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN"
   assert_failure
   assert_output --partial 'missing arguments'
 }
 
 @test "cannot use --local and --user at same time" {
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --user root
+  run $ABRA app cmd --local --user root "$TEST_APP_DOMAIN" test_cmd
   assert_failure
   assert_output --partial 'cannot use --local & --user together'
 }
@@ -147,7 +147,7 @@ test_cmd_export"
   run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh"
   assert_success
 
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos
+  run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd
   assert_failure
   assert_output --partial "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh does not exist"
 
@@ -155,25 +155,25 @@ test_cmd_export"
 }
 
 @test "error if missing command" {
-  run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist --local
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" doesnt_exist
   assert_failure
   assert_output --partial "doesn't have a doesnt_exist function"
 }
 
 @test "run --local command" {
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 }
 
 @test "run command with single arg" {
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_arg --local -- bing
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_arg -- bing
   assert_success
   assert_output --partial 'bing'
 }
 
 @test "run command with several args" {
-  run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_args --local -- bong bang
+  run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_args -- bong bang
   assert_success
   assert_output --partial 'bong bang'
 }
diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index 5747ce6e..0df25675 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -16,6 +16,7 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
+  _reset_recipe
 }
 
 teardown(){
@@ -82,13 +83,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  refute_output --regexp 'behind .* 3 commits'
 
   _reset_recipe
   _undeploy_app
@@ -100,7 +101,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
@@ -108,7 +109,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   _undeploy_app
   _reset_recipe
@@ -116,6 +117,9 @@ teardown(){
 
 # bats test_tags=slow
 @test "deploy latest commit if no published versions and no --chaos" {
+  # TODO(d1): fix with a new test recipe which has no published versions?
+  skip "known issue, abra-test-recipe has published versions now"
+
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   _remove_tags
@@ -140,7 +144,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
@@ -273,6 +277,10 @@ teardown(){
 }
 
 @test "ensure domain is checked" {
+  if [[ "$TEST_SERVER" == "default" ]]; then
+      skip "domain checks are disabled for local server"
+  fi
+
   appDomain="custom-html.DOESNTEXIST"
 
   run $ABRA app new custom-html \
diff --git a/tests/integration/app_new.bats b/tests/integration/app_new.bats
index 885e02cd..457ee152 100644
--- a/tests/integration/app_new.bats
+++ b/tests/integration/app_new.bats
@@ -45,7 +45,7 @@ teardown(){
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is up to date with 'origin/main'."
+  assert_output --partial "up to date"
 }
 
 @test "create new app with version" {
@@ -121,7 +121,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -131,7 +131,7 @@ teardown(){
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is up to date with 'origin/main'."
+  assert_output --partial "up to date"
 
   _reset_recipe
 }
@@ -141,7 +141,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
+  assert_output --regexp 'behind .* 3 commits'
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app new "$TEST_RECIPE" \
@@ -154,7 +154,7 @@ teardown(){
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
+  assert_output --regexp 'behind .* 3 commits'
 
   _reset_recipe
 }
diff --git a/tests/integration/app_remove.bats b/tests/integration/app_remove.bats
index 8800e70a..4cf6f07a 100644
--- a/tests/integration/app_remove.bats
+++ b/tests/integration/app_remove.bats
@@ -104,7 +104,10 @@ teardown(){
 
   _undeploy_app
 
-  run $ABRA app volume rm "$TEST_APP_DOMAIN"
+  # TODO: should wait as long as volume is no longer in use
+  sleep 10
+
+  run $ABRA app volume rm "$TEST_APP_DOMAIN" --no-input
   assert_success
 
   run $ABRA app volume ls "$TEST_APP_DOMAIN"
diff --git a/tests/integration/app_restore.bats b/tests/integration/app_restore.bats
index 77a63f63..f2cf52e8 100644
--- a/tests/integration/app_restore.bats
+++ b/tests/integration/app_restore.bats
@@ -109,13 +109,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
-  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
+  run $ABRA app restore "$TEST_APP_DOMAIN" app
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --partial "up to date"
 }
 
 @test "ensure recipe not up to date if --offline" {
@@ -126,19 +126,19 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
-  run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST --offline
+  run $ABRA app restore "$TEST_APP_DOMAIN" app --offline
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --partial "HEAD detached at $latestCommit"
 }
 
 @test "error if missing service" {
diff --git a/tests/integration/app_rollback.bats b/tests/integration/app_rollback.bats
index 0439b94f..b8200bbb 100644
--- a/tests/integration/app_rollback.bats
+++ b/tests/integration/app_rollback.bats
@@ -50,13 +50,13 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --partial "up to date"
 }
 
 @test "ensure recipe not up to date if --offline" {
@@ -67,14 +67,14 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" \
     --no-input --no-converge-checks --offline
   assert_failure
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
   assert_success
@@ -131,7 +131,7 @@ teardown(){
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
-    --no-input --no-converge-checks --chaos
+    --no-input --chaos
   assert_success
   assert_output --partial "$latestCommit"
   assert_output --partial 'chaos'
diff --git a/tests/integration/app_secret.bats b/tests/integration/app_secret.bats
index 3d15e959..b745a939 100644
--- a/tests/integration/app_secret.bats
+++ b/tests/integration/app_secret.bats
@@ -8,7 +8,7 @@ setup_file(){
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 }
@@ -19,13 +19,6 @@ teardown_file(){
   _reset_recipe
 }
 
-teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
-}
-
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
diff --git a/tests/integration/app_version.bats b/tests/integration/app_version.bats
index 2a6d6319..b11f58ef 100644
--- a/tests/integration/app_version.bats
+++ b/tests/integration/app_version.bats
@@ -59,6 +59,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "error if not in catalogue" {
+  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
+
   _deploy_app
 
   run $ABRA app version "$TEST_APP_DOMAIN"
@@ -92,7 +94,7 @@ teardown(){
   assert_success
 
  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 3
+  sleep 5
 
   run $ABRA app volume remove "$appDomain" --no-input
   assert_success
diff --git a/tests/integration/app_volume.bats b/tests/integration/app_volume.bats
index 5b4ff600..99acc6c9 100644
--- a/tests/integration/app_volume.bats
+++ b/tests/integration/app_volume.bats
@@ -79,7 +79,7 @@ teardown(){
   _undeploy_app
 
   # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
+  sleep 10
 
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
@@ -93,7 +93,7 @@ teardown(){
   _undeploy_app
 
   # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
+  sleep 10
 
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
diff --git a/tests/integration/helpers/app.bash b/tests/integration/helpers/app.bash
index ee3e30f6..32208b84 100644
--- a/tests/integration/helpers/app.bash
+++ b/tests/integration/helpers/app.bash
@@ -49,7 +49,7 @@ _reset_app(){
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 }
diff --git a/tests/integration/helpers/server.bash b/tests/integration/helpers/server.bash
index 3c6c74d7..e3bcb320 100644
--- a/tests/integration/helpers/server.bash
+++ b/tests/integration/helpers/server.bash
@@ -11,7 +11,11 @@ _add_server() {
 }
 
 _rm_server() {
-  run $ABRA server remove --no-input "$TEST_SERVER"
+  if [[ "$TEST_SERVER" == "default" ]]; then
+      run rm -rf "$ABRA_DIR/servers/default"
+  else
+      run $ABRA server remove --no-input "$TEST_SERVER"
+  fi
   assert_success
   assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER"
 }
diff --git a/tests/integration/recipe_lint.bats b/tests/integration/recipe_lint.bats
index 0b2617b4..401ab8fd 100644
--- a/tests/integration/recipe_lint.bats
+++ b/tests/integration/recipe_lint.bats
@@ -66,13 +66,13 @@ setup() {
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA recipe lint "$TEST_RECIPE"
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   _reset_recipe
 }
@@ -82,13 +82,13 @@ setup() {
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA recipe lint "$TEST_RECIPE" --offline
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   _reset_recipe
 }
diff --git a/tests/integration/recipe_upgrade.bats b/tests/integration/recipe_upgrade.bats
index 183d1f22..a36efe26 100644
--- a/tests/integration/recipe_upgrade.bats
+++ b/tests/integration/recipe_upgrade.bats
@@ -61,14 +61,14 @@ setup(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input
   assert_success
   assert_output --partial 'can upgrade service: app'
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --regexp 'behind .* 3 commits'
 
   _reset_recipe
 }
diff --git a/tests/integration/recipe_version.bats b/tests/integration/recipe_version.bats
index 42d2e4d0..ebc5bad9 100644
--- a/tests/integration/recipe_version.bats
+++ b/tests/integration/recipe_version.bats
@@ -12,6 +12,8 @@ setup() {
 }
 
 @test "error if not present in catalogue" {
+  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
+
   run $ABRA recipe versions "$TEST_RECIPE"
   assert_failure
   assert_output --partial "is not published on the catalogue"

From 2c515ce70adbcb808895b8bb1efd6036e51092c1 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 12 Mar 2024 10:03:42 +0100
Subject: [PATCH 68/76] Revert "feat: backup revolution"

This reverts commit c5687dfbd7413f87ad202a4a275da495edae4730.

This is a temporary measure to facilitate a release which won't
completely explode peoples workflows (missing command logic). We
re-instate this commit after the first 0.9.x release.
---
 cli/app/backup.go              | 674 +++++++++++++++++++--------------
 cli/app/cp.go                  |  16 +-
 cli/app/restore.go             | 197 ++++++++--
 cli/app/run.go                 |   2 +-
 cli/internal/backup.go         |  86 ++---
 cli/internal/command.go        |   4 +-
 pkg/config/env.go              |   2 -
 pkg/container/container.go     |   2 +-
 pkg/service/service.go         |  64 ----
 pkg/upstream/container/exec.go |  23 +-
 10 files changed, 614 insertions(+), 456 deletions(-)

diff --git a/cli/app/backup.go b/cli/app/backup.go
index f6c8d2ae..f9a239b1 100644
--- a/cli/app/backup.go
+++ b/cli/app/backup.go
@@ -1,296 +1,414 @@
 package app
 
 import (
+	"archive/tar"
+	"context"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/klauspost/pgzip"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var snapshot string
-var snapshotFlag = &cli.StringFlag{
-	Name:        "snapshot, s",
-	Usage:       "Lists specific snapshot",
-	Destination: &snapshot,
-}
-
-var includePath string
-var includePathFlag = &cli.StringFlag{
-	Name:        "path, p",
-	Usage:       "Include path",
-	Destination: &includePath,
-}
-
-var resticRepo string
-var resticRepoFlag = &cli.StringFlag{
-	Name:        "repo, r",
-	Usage:       "Restic repository",
-	Destination: &resticRepo,
-}
-
-var appBackupListCommand = cli.Command{
-	Name:    "list",
-	Aliases: []string{"ls"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List all backups",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}
-
-var appBackupDownloadCommand = cli.Command{
-	Name:    "download",
-	Aliases: []string{"d"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "Download a backup",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		remoteBackupDir := "/tmp/backup.tar.gz"
-		currentWorkingDir := "."
-		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
-			logrus.Fatal(err)
-		}
-
-		fmt.Println("backup successfully downloaded to current working directory")
-
-		return nil
-	},
-}
-
-var appBackupCreateCommand = cli.Command{
-	Name:    "create",
-	Aliases: []string{"c"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		resticRepoFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "Create a new backup",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if resticRepo != "" {
-			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
-			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}
-
-var appBackupSnapshotsCommand = cli.Command{
-	Name:    "snapshots",
-	Aliases: []string{"s"},
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.OfflineFlag,
-		snapshotFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	Usage:        "List backup snapshots",
-	BashComplete: autocomplete.AppNameComplete,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
-					logrus.Fatal(err)
-				}
-			}
-
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
+type backupConfig struct {
+	preHookCmd  string
+	postHookCmd string
+	backupPaths []string
 }
 
 var appBackupCommand = cli.Command{
 	Name:      "backup",
-	Aliases:   []string{"b"},
-	Usage:     "Manage app backups",
-	ArgsUsage: "<domain>",
-	Subcommands: []cli.Command{
-		appBackupListCommand,
-		appBackupSnapshotsCommand,
-		appBackupDownloadCommand,
-		appBackupCreateCommand,
+	Aliases:   []string{"bk"},
+	Usage:     "Run app backup",
+	ArgsUsage: "<domain> [<service>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		internal.ChaosFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app backup.
+
+A backup command and pre/post hook commands are defined in the recipe
+configuration. Abra reads this configuration and run the comands in the context
+of the deployed services. Pass <service> if you only want to back up a single
+service. All backups are placed in the ~/.abra/backups directory.
+
+A single backup file is produced for all backup paths specified for a service.
+If we have the following backup configuration:
+
+    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
+
+And we run "abra app backup example.com app", Abra will produce a file that
+looks like:
+
+    ~/.abra/backups/example_com_app_609341138.tar.gz
+
+This file is a compressed archive which contains all backup paths. To see paths, run:
+
+    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
+
+(Make sure to change the name of the backup file)
+
+This single file can be used to restore your app. See "abra app restore" for more.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		backupConfigs := make(map[string]backupConfig)
+		for _, service := range recipe.Config.Services {
+			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
+				if backupsEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					bkConfig := backupConfig{}
+
+					logrus.Debugf("backup config detected for %s", fullServiceName)
+
+					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
+						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
+						bkConfig.backupPaths = strings.Split(paths, ",")
+					}
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						bkConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						bkConfig.postHookCmd = postHookCmd
+					}
+
+					backupConfigs[service.Name] = bkConfig
+				}
+			}
+		}
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName != "" {
+			backupConfig, ok := backupConfigs[serviceName]
+			if !ok {
+				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
+			}
+
+			logrus.Infof("running backup for the %s service", serviceName)
+
+			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			if len(backupConfigs) == 0 {
+				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
+			}
+
+			for serviceName, backupConfig := range backupConfigs {
+				logrus.Infof("running backup for the %s service", serviceName)
+
+				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
 	},
 }
+
+// TimeStamp generates a file name friendly timestamp.
+func TimeStamp() string {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	return strings.Replace(ts, ":", "-", -1)
+}
+
+// runBackup does the actual backup logic.
+func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
+	if len(bkConfig.backupPaths) == 0 {
+		return fmt.Errorf("backup paths are empty for %s?", serviceName)
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if bkConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
+	}
+
+	var tempBackupPaths []string
+	for _, remoteBackupPath := range bkConfig.backupPaths {
+		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
+		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
+
+		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
+
+		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		if err != nil {
+			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
+		}
+		defer content.Close()
+
+		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
+		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
+		if err := copyToFile(localBackupPath, preArchive); err != nil {
+			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+			if err := cleanupTempArchives(tempBackupPaths); err != nil {
+				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			}
+			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
+		}
+
+		tempBackupPaths = append(tempBackupPaths, localBackupPath)
+	}
+
+	logrus.Infof("compressing and merging archives...")
+
+	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
+		logrus.Debugf("failed to merge archive files: %s", err.Error())
+		if err := cleanupTempArchives(tempBackupPaths); err != nil {
+			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+		}
+		return fmt.Errorf("failed to merge archive files: %s", err.Error())
+	}
+
+	if err := cleanupTempArchives(tempBackupPaths); err != nil {
+		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+	}
+
+	if bkConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
+	}
+
+	return nil
+}
+
+func copyToFile(outfile string, r io.Reader) error {
+	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
+	if err != nil {
+		return err
+	}
+
+	tmpPath := tmpFile.Name()
+
+	_, err = io.Copy(tmpFile, r)
+	tmpFile.Close()
+
+	if err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	if err = os.Rename(tmpPath, outfile); err != nil {
+		os.Remove(tmpPath)
+		return err
+	}
+
+	return nil
+}
+
+func cleanupTempArchives(tarPaths []string) error {
+	for _, tarPath := range tarPaths {
+		if err := os.RemoveAll(tarPath); err != nil {
+			return err
+		}
+
+		logrus.Debugf("remove temporary archive file %s", tarPath)
+	}
+
+	return nil
+}
+
+func mergeArchives(tarPaths []string, serviceName string) error {
+	var out io.Writer
+	var cout *pgzip.Writer
+
+	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
+
+	fout, err := os.Create(localBackupPath)
+	if err != nil {
+		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
+	}
+
+	defer fout.Close()
+	out = fout
+
+	cout = pgzip.NewWriter(out)
+	out = cout
+
+	tw := tar.NewWriter(out)
+
+	for _, tarPath := range tarPaths {
+		if err := addTar(tw, tarPath); err != nil {
+			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		return fmt.Errorf("failed to close tar writer %v", err)
+	}
+
+	if cout != nil {
+		if err := cout.Flush(); err != nil {
+			return fmt.Errorf("failed to flush: %s", err)
+		} else if err = cout.Close(); err != nil {
+			return fmt.Errorf("failed to close compressed writer: %s", err)
+		}
+	}
+
+	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
+
+	return nil
+}
+
+func addTar(tw *tar.Writer, pth string) (err error) {
+	var tr *tar.Reader
+	var rc io.ReadCloser
+	var hdr *tar.Header
+
+	if tr, rc, err = openTarFile(pth); err != nil {
+		return
+	}
+
+	for {
+		if hdr, err = tr.Next(); err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			break
+		}
+		if err = tw.WriteHeader(hdr); err != nil {
+			break
+		} else if _, err = io.Copy(tw, tr); err != nil {
+			break
+		}
+	}
+	if err == nil {
+		err = rc.Close()
+	} else {
+		rc.Close()
+	}
+	return
+}
+
+func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
+	var fin *os.File
+	var n int
+	buff := make([]byte, 1024)
+
+	if fin, err = os.Open(pth); err != nil {
+		return
+	}
+
+	if n, err = fin.Read(buff); err != nil {
+		fin.Close()
+		return
+	} else if n == 0 {
+		fin.Close()
+		err = fmt.Errorf("%s is empty", pth)
+		return
+	}
+
+	if _, err = fin.Seek(0, 0); err != nil {
+		fin.Close()
+		return
+	}
+
+	rc = fin
+	tr = tar.NewReader(rc)
+
+	return tr, rc, nil
+}
diff --git a/cli/app/cp.go b/cli/app/cp.go
index f27b42ce..bfc2c789 100644
--- a/cli/app/cp.go
+++ b/cli/app/cp.go
@@ -76,9 +76,9 @@ And if you want to copy that file back to your current working directory locally
 		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 		if toContainer {
-			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
+			err = copyToContainer(cl, container.ID, srcPath, dstPath)
 		} else {
-			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
+			err = copyFromContainer(cl, container.ID, srcPath, dstPath)
 		}
 		if err != nil {
 			logrus.Fatal(err)
@@ -106,9 +106,9 @@ func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service st
 	return "", "", "", false, errServiceMissing
 }
 
-// CopyToContainer copies a file or directory from the local file system to the container.
+// copyToContainer copies a file or directory from the local file system to the container.
 // See the possible copy modes and their documentation.
-func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := os.Stat(srcPath)
 	if err != nil {
 		return fmt.Errorf("local %s ", err)
@@ -140,7 +140,7 @@ func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -179,7 +179,7 @@ func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -194,9 +194,9 @@ func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 	return nil
 }
 
-// CopyFromContainer copies a file or directory from the given container to the local file system.
+// copyFromContainer copies a file or directory from the given container to the local file system.
 // See the possible copy modes and their documentation.
-func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
 	if err != nil {
 		if errdefs.IsNotFound(err) {
diff --git a/cli/app/restore.go b/cli/app/restore.go
index c80347f5..1bf9c840 100644
--- a/cli/app/restore.go
+++ b/cli/app/restore.go
@@ -1,82 +1,223 @@
 package app
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-var targetPath string
-var targetPathFlag = &cli.StringFlag{
-	Name:        "target, t",
-	Usage:       "Target path",
-	Destination: &targetPath,
+type restoreConfig struct {
+	preHookCmd  string
+	postHookCmd string
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Restore an app backup",
-	ArgsUsage: "<domain> <service>",
+	Usage:     "Run app restore",
+	ArgsUsage: "<domain> <service> <file>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		targetPathFlag,
+		internal.ChaosFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+Run an app restore.
+
+Pre/post hook commands are defined in the recipe configuration. Abra reads this
+configuration and run the comands in the context of the service before
+restoring the backup.
+
+Unlike "abra app backup", restore must be run on a per-service basis. You can
+not restore all services in one go. Backup files produced by Abra are
+compressed archives which use absolute paths. This allows Abra to restore
+according to standard tar command logic, i.e. the backup will be restored to
+the path it was originally backed up from.
+
+Example:
+
+    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
+`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		if err := recipe.EnsureExists(app.Recipe); err != nil {
+		recipe, err := recipe.Get(app.Recipe, internal.Offline)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
+		}
+
+		backupPath := c.Args().Get(2)
+		if backupPath == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
+		}
+
+		if _, err := os.Stat(backupPath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s doesn't exist?", backupPath)
+			}
+		}
+
+		restoreConfigs := make(map[string]restoreConfig)
+		for _, service := range recipe.Config.Services {
+			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
+				if restoreEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					rsConfig := restoreConfig{}
+
+					logrus.Debugf("restore config detected for %s", fullServiceName)
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						rsConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						rsConfig.postHookCmd = postHookCmd
+					}
+
+					restoreConfigs[service.Name] = rsConfig
+				}
+			}
+		}
+
+		rsConfig, ok := restoreConfigs[serviceName]
+		if !ok {
+			rsConfig = restoreConfig{}
+		}
+
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if targetPath != "" {
-			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
-			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
+		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
+
+// runRestore does the actual restore logic.
+func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if rsConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
+	}
+
+	backupReader, err := os.Open(backupPath)
+	if err != nil {
+		return err
+	}
+
+	content, err := archive.DecompressStream(backupReader)
+	if err != nil {
+		return err
+	}
+
+	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
+	// files according to the paths set in the compressed archive
+	restorePath := "/"
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
+		return err
+	}
+
+	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
+
+	if rsConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
+	}
+
+	return nil
+}
diff --git a/cli/app/run.go b/cli/app/run.go
index b5e0a9ce..4ae68c1b 100644
--- a/cli/app/run.go
+++ b/cli/app/run.go
@@ -91,7 +91,7 @@ var appRunCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
diff --git a/cli/internal/backup.go b/cli/internal/backup.go
index 530735c9..79951810 100644
--- a/cli/internal/backup.go
+++ b/cli/internal/backup.go
@@ -1,67 +1,35 @@
 package internal
 
 import (
-	"context"
-
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	"coopcloud.tech/abra/pkg/service"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/sirupsen/logrus"
+	"strings"
 )
 
-// RetrieveBackupBotContainer gets the deployed backupbot container.
-func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) {
-	ctx := context.Background()
-	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
-	if err != nil {
-		return types.Container{}, err
+// SafeSplit splits up a string into a list of commands safely.
+func SafeSplit(s string) []string {
+	split := strings.Split(s, " ")
+
+	var result []string
+	var inquote string
+	var block string
+	for _, i := range split {
+		if inquote == "" {
+			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
+				inquote = string(i[0])
+				block = strings.TrimPrefix(i, inquote) + " "
+			} else {
+				result = append(result, i)
+			}
+		} else {
+			if !strings.HasSuffix(i, inquote) {
+				block += i + " "
+			} else {
+				block += strings.TrimSuffix(i, inquote)
+				inquote = ""
+				result = append(result, block)
+				block = ""
+			}
+		}
 	}
 
-	logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
-
-	filters := filters.NewArgs()
-	filters.Add("name", chosenService.Spec.Name)
-	targetContainer, err := containerPkg.GetContainer(
-		ctx,
-		cl,
-		filters,
-		NoInput,
-	)
-	if err != nil {
-		return types.Container{}, err
-	}
-
-	return targetContainer, nil
-}
-
-// RunBackupCmdRemote runs a backup related command on a remote backupbot container.
-func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error {
-	execBackupListOpts := types.ExecConfig{
-		AttachStderr: true,
-		AttachStdin:  true,
-		AttachStdout: true,
-		Cmd:          []string{"/usr/bin/backup", "--", backupCmd},
-		Detach:       false,
-		Env:          execEnv,
-		Tty:          true,
-	}
-
-	logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts)
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil {
-		return err
-	}
-
-	return nil
+	return result
 }
diff --git a/cli/internal/command.go b/cli/internal/command.go
index 13c007be..6f02ae1c 100644
--- a/cli/internal/command.go
+++ b/cli/internal/command.go
@@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		Tty:          false,
 	}
 
-	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
 		shell = "/bin/sh"
 	}
@@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		execCreateOpts.Tty = false
 	}
 
-	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		return err
 	}
 
diff --git a/pkg/config/env.go b/pkg/config/env.go
index adedc6f4..62f6a71d 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -36,8 +36,6 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
-var BackupbotLabel = "coop-cloud.backupbot.enabled"
-
 // envVarModifiers is a list of env var modifier strings. These are added to
 // env vars as comments and modify their processing by Abra, e.g. determining
 // how long secrets should be.
diff --git a/pkg/container/container.go b/pkg/container/container.go
index 1354b0dd..09d5703b 100644
--- a/pkg/container/container.go
+++ b/pkg/container/container.go
@@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) > 1 {
+	if len(containers) != 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")
diff --git a/pkg/service/service.go b/pkg/service/service.go
index 48cdce75..3d92d821 100644
--- a/pkg/service/service.go
+++ b/pkg/service/service.go
@@ -14,70 +14,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// GetService retrieves a service container based on a label. If prompt is true
-// and the retrievd count of service containers does not match 1, then a prompt
-// is presented to let the user choose. An error is returned when no service is
-// found.
-func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
-	services, err := cl.ServiceList(c, types.ServiceListOptions{})
-	if err != nil {
-		return swarm.Service{}, err
-	}
-
-	if len(services) == 0 {
-		return swarm.Service{}, fmt.Errorf("no services deployed?")
-	}
-
-	var matchingServices []swarm.Service
-	for _, service := range services {
-		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
-			matchingServices = append(matchingServices, service)
-		}
-	}
-
-	if len(matchingServices) == 0 {
-		return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label)
-	}
-
-	if len(matchingServices) > 1 {
-		var servicesRaw []string
-		for _, service := range matchingServices {
-			serviceName := service.Spec.Name
-			created := formatter.HumanDuration(service.CreatedAt.Unix())
-			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
-		}
-
-		if !prompt {
-			err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " "))
-			return swarm.Service{}, err
-		}
-
-		logrus.Warnf("ambiguous service list received, prompting for input")
-
-		var response string
-		prompt := &survey.Select{
-			Message: "which service are you looking for?",
-			Options: servicesRaw,
-		}
-
-		if err := survey.AskOne(prompt, &response); err != nil {
-			return swarm.Service{}, err
-		}
-
-		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
-		for _, service := range matchingServices {
-			serviceName := strings.ToLower(service.Spec.Name)
-			if serviceName == chosenService {
-				return service, nil
-			}
-		}
-
-		logrus.Panic("failed to match chosen service")
-	}
-
-	return matchingServices[0], nil
-}
-
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.
diff --git a/pkg/upstream/container/exec.go b/pkg/upstream/container/exec.go
index 82a2c570..e811481a 100644
--- a/pkg/upstream/container/exec.go
+++ b/pkg/upstream/container/exec.go
@@ -13,10 +13,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// RunExec runs a command on a remote container. io.Writer corresponds to the
-// command output.
-func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string,
-	execConfig *types.ExecConfig) (io.Writer, error) {
+func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error {
 	ctx := context.Background()
 
 	// We need to check the tty _before_ we do the ContainerExecCreate, because
@@ -24,22 +21,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 	// there's no easy way to clean those up). But also in order to make "not
 	// exist" errors take precedence we do a dummy inspect first.
 	if _, err := client.ContainerInspect(ctx, containerID); err != nil {
-		return nil, err
+		return err
 	}
 	if !execConfig.Detach {
 		if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	response, err := client.ContainerExecCreate(ctx, containerID, *execConfig)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	execID := response.ID
 	if execID == "" {
-		return nil, errors.New("exec ID empty")
+		return errors.New("exec ID empty")
 	}
 
 	if execConfig.Detach {
@@ -47,13 +44,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 			Detach: execConfig.Detach,
 			Tty:    execConfig.Tty,
 		}
-		return nil, client.ContainerExecStart(ctx, execID, execStartCheck)
+		return client.ContainerExecStart(ctx, execID, execStartCheck)
 	}
 	return interactiveExec(ctx, dockerCli, client, execConfig, execID)
 }
 
 func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client,
-	execConfig *types.ExecConfig, execID string) (io.Writer, error) {
+	execConfig *types.ExecConfig, execID string) error {
 	// Interactive exec requested.
 	var (
 		out, stderr io.Writer
@@ -79,7 +76,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 	}
 	resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck)
 	if err != nil {
-		return out, err
+		return err
 	}
 	defer resp.Close()
 
@@ -110,10 +107,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 
 	if err := <-errCh; err != nil {
 		logrus.Debugf("Error hijack: %s", err)
-		return out, err
+		return err
 	}
 
-	return out, getExecExitStatus(ctx, client, execID)
+	return getExecExitStatus(ctx, client, execID)
 }
 
 func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {

From cea56dddde49c4e4456d7b469c99ee9ffdc26ae9 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Mon, 11 Mar 2024 15:54:57 +0100
Subject: [PATCH 69/76] fix: drop deprecated stanza (goreleaser)

---
 .goreleaser.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.goreleaser.yml b/.goreleaser.yml
index 307505e5..24f6a009 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -51,12 +51,6 @@ builds:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
 
-archives:
-  - replacements:
-      386: i386
-      amd64: x86_64
-    format: binary
-
 checksum:
   name_template: "checksums.txt"
 

From 9a3a39a1851dc54dfe87a75e97c276360ce95e79 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 12 Mar 2024 10:05:31 +0100
Subject: [PATCH 70/76] chore: new 0.9.x series

---
 scripts/installer/installer | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/installer/installer b/scripts/installer/installer
index 8b8ee5ec..b4c5f787 100755
--- a/scripts/installer/installer
+++ b/scripts/installer/installer
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.8.1-beta"
+ABRA_VERSION="0.9.0-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
 RC_VERSION="0.8.0-rc1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"

From e3a0af5840553c2c5a1a5135f474afb90c7f4375 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 12 Mar 2024 10:11:14 +0100
Subject: [PATCH 71/76] build: upgrade goreleaser

Closes https://git.coopcloud.tech/coop-cloud/organising/issues/474
---
 .drone.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index 5861cd4d..08b78ccb 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -29,7 +29,7 @@ steps:
       event: tag
 
   - name: release
-    image: goreleaser/goreleaser:v1.18.2
+    image: goreleaser/goreleaser:v1.24.0
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token

From 575f9905f1e26359bcd3ac2bc1a01e77062ca4a2 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 12 Mar 2024 10:34:40 +0100
Subject: [PATCH 72/76] Revert "Revert "feat: backup revolution""

This reverts commit 2c515ce70adbcb808895b8bb1efd6036e51092c1.
---
 cli/app/backup.go              | 564 +++++++++++++--------------------
 cli/app/cp.go                  |  16 +-
 cli/app/restore.go             | 197 ++----------
 cli/app/run.go                 |   2 +-
 cli/internal/backup.go         |  86 +++--
 cli/internal/command.go        |   4 +-
 pkg/config/env.go              |   2 +
 pkg/container/container.go     |   2 +-
 pkg/service/service.go         |  64 ++++
 pkg/upstream/container/exec.go |  23 +-
 10 files changed, 401 insertions(+), 559 deletions(-)

diff --git a/cli/app/backup.go b/cli/app/backup.go
index f9a239b1..f6c8d2ae 100644
--- a/cli/app/backup.go
+++ b/cli/app/backup.go
@@ -1,414 +1,296 @@
 package app
 
 import (
-	"archive/tar"
-	"context"
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/klauspost/pgzip"
+	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type backupConfig struct {
-	preHookCmd  string
-	postHookCmd string
-	backupPaths []string
+var snapshot string
+var snapshotFlag = &cli.StringFlag{
+	Name:        "snapshot, s",
+	Usage:       "Lists specific snapshot",
+	Destination: &snapshot,
 }
 
-var appBackupCommand = cli.Command{
-	Name:      "backup",
-	Aliases:   []string{"bk"},
-	Usage:     "Run app backup",
-	ArgsUsage: "<domain> [<service>]",
+var includePath string
+var includePathFlag = &cli.StringFlag{
+	Name:        "path, p",
+	Usage:       "Include path",
+	Destination: &includePath,
+}
+
+var resticRepo string
+var resticRepoFlag = &cli.StringFlag{
+	Name:        "repo, r",
+	Usage:       "Restic repository",
+	Destination: &resticRepo,
+}
+
+var appBackupListCommand = cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		snapshotFlag,
+		includePathFlag,
 	},
 	Before:       internal.SubCommandBefore,
+	Usage:        "List all backups",
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app backup.
-
-A backup command and pre/post hook commands are defined in the recipe
-configuration. Abra reads this configuration and run the comands in the context
-of the deployed services. Pass <service> if you only want to back up a single
-service. All backups are placed in the ~/.abra/backups directory.
-
-A single backup file is produced for all backup paths specified for a service.
-If we have the following backup configuration:
-
-    - "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
-
-And we run "abra app backup example.com app", Abra will produce a file that
-looks like:
-
-    ~/.abra/backups/example_com_app_609341138.tar.gz
-
-This file is a compressed archive which contains all backup paths. To see paths, run:
-
-    tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
-
-(Make sure to change the name of the backup file)
-
-This single file can be used to restore your app. See "abra app restore" for more.
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		backupConfigs := make(map[string]backupConfig)
-		for _, service := range recipe.Config.Services {
-			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
-				if backupsEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					bkConfig := backupConfig{}
-
-					logrus.Debugf("backup config detected for %s", fullServiceName)
-
-					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
-						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
-						bkConfig.backupPaths = strings.Split(paths, ",")
-					}
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						bkConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						bkConfig.postHookCmd = postHookCmd
-					}
-
-					backupConfigs[service.Name] = bkConfig
-				}
-			}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName != "" {
-			backupConfig, ok := backupConfigs[serviceName]
-			if !ok {
-				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
-			}
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 
-			logrus.Infof("running backup for the %s service", serviceName)
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
 
-			if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			if len(backupConfigs) == 0 {
-				logrus.Fatalf("no backup configs discovered for %s?", app.Name)
-			}
-
-			for serviceName, backupConfig := range backupConfigs {
-				logrus.Infof("running backup for the %s service", serviceName)
-
-				if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
-					logrus.Fatal(err)
-				}
-			}
+		if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
 
-// TimeStamp generates a file name friendly timestamp.
-func TimeStamp() string {
-	ts := time.Now().UTC().Format(time.RFC3339)
-	return strings.Replace(ts, ":", "-", -1)
-}
+var appBackupDownloadCommand = cli.Command{
+	Name:    "download",
+	Aliases: []string{"d"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+		includePathFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Download a backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-// runBackup does the actual backup logic.
-func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
-	if len(bkConfig.backupPaths) == 0 {
-		return fmt.Errorf("backup paths are empty for %s?", serviceName)
-	}
-
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if bkConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
-	}
-
-	var tempBackupPaths []string
-	for _, remoteBackupPath := range bkConfig.backupPaths {
-		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
-		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
-		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
-
-		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
-
-		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		cl, err := client.New(app.Server)
 		if err != nil {
-			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			logrus.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if includePath != "" {
+			logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
+			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
+		}
+
+		remoteBackupDir := "/tmp/backup.tar.gz"
+		currentWorkingDir := "."
+		if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Println("backup successfully downloaded to current working directory")
+
+		return nil
+	},
+}
+
+var appBackupCreateCommand = cli.Command{
+	Name:    "create",
+	Aliases: []string{"c"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		resticRepoFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "Create a new backup",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
 			}
-			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
-		}
-		defer content.Close()
 
-		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
-		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
-		if err := copyToFile(localBackupPath, preArchive); err != nil {
-			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-			if err := cleanupTempArchives(tempBackupPaths); err != nil {
-				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
 			}
-			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
-		}
 
-		tempBackupPaths = append(tempBackupPaths, localBackupPath)
-	}
-
-	logrus.Infof("compressing and merging archives...")
-
-	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
-		logrus.Debugf("failed to merge archive files: %s", err.Error())
-		if err := cleanupTempArchives(tempBackupPaths); err != nil {
-			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-		}
-		return fmt.Errorf("failed to merge archive files: %s", err.Error())
-	}
-
-	if err := cleanupTempArchives(tempBackupPaths); err != nil {
-		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
-	}
-
-	if bkConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
-	}
-
-	return nil
-}
-
-func copyToFile(outfile string, r io.Reader) error {
-	tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
-	if err != nil {
-		return err
-	}
-
-	tmpPath := tmpFile.Name()
-
-	_, err = io.Copy(tmpFile, r)
-	tmpFile.Close()
-
-	if err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	if err = os.Rename(tmpPath, outfile); err != nil {
-		os.Remove(tmpPath)
-		return err
-	}
-
-	return nil
-}
-
-func cleanupTempArchives(tarPaths []string) error {
-	for _, tarPath := range tarPaths {
-		if err := os.RemoveAll(tarPath); err != nil {
-			return err
-		}
-
-		logrus.Debugf("remove temporary archive file %s", tarPath)
-	}
-
-	return nil
-}
-
-func mergeArchives(tarPaths []string, serviceName string) error {
-	var out io.Writer
-	var cout *pgzip.Writer
-
-	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
-
-	fout, err := os.Create(localBackupPath)
-	if err != nil {
-		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
-	}
-
-	defer fout.Close()
-	out = fout
-
-	cout = pgzip.NewWriter(out)
-	out = cout
-
-	tw := tar.NewWriter(out)
-
-	for _, tarPath := range tarPaths {
-		if err := addTar(tw, tarPath); err != nil {
-			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
-		}
-	}
-
-	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer %v", err)
-	}
-
-	if cout != nil {
-		if err := cout.Flush(); err != nil {
-			return fmt.Errorf("failed to flush: %s", err)
-		} else if err = cout.Close(); err != nil {
-			return fmt.Errorf("failed to close compressed writer: %s", err)
-		}
-	}
-
-	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
-
-	return nil
-}
-
-func addTar(tw *tar.Writer, pth string) (err error) {
-	var tr *tar.Reader
-	var rc io.ReadCloser
-	var hdr *tar.Header
-
-	if tr, rc, err = openTarFile(pth); err != nil {
-		return
-	}
-
-	for {
-		if hdr, err = tr.Next(); err != nil {
-			if err == io.EOF {
-				err = nil
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
 			}
-			break
 		}
-		if err = tw.WriteHeader(hdr); err != nil {
-			break
-		} else if _, err = io.Copy(tw, tr); err != nil {
-			break
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
 		}
-	}
-	if err == nil {
-		err = rc.Close()
-	} else {
-		rc.Close()
-	}
-	return
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if resticRepo != "" {
+			logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
+			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
 }
 
-func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
-	var fin *os.File
-	var n int
-	buff := make([]byte, 1024)
+var appBackupSnapshotsCommand = cli.Command{
+	Name:    "snapshots",
+	Aliases: []string{"s"},
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.OfflineFlag,
+		snapshotFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	Usage:        "List backup snapshots",
+	BashComplete: autocomplete.AppNameComplete,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
 
-	if fin, err = os.Open(pth); err != nil {
-		return
-	}
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
+			logrus.Fatal(err)
+		}
 
-	if n, err = fin.Read(buff); err != nil {
-		fin.Close()
-		return
-	} else if n == 0 {
-		fin.Close()
-		err = fmt.Errorf("%s is empty", pth)
-		return
-	}
+		if !internal.Chaos {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
 
-	if _, err = fin.Seek(0, 0); err != nil {
-		fin.Close()
-		return
-	}
+			if !internal.Offline {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 
-	rc = fin
-	tr = tar.NewReader(rc)
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
+				logrus.Fatal(err)
+			}
+		}
 
-	return tr, rc, nil
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+var appBackupCommand = cli.Command{
+	Name:      "backup",
+	Aliases:   []string{"b"},
+	Usage:     "Manage app backups",
+	ArgsUsage: "<domain>",
+	Subcommands: []cli.Command{
+		appBackupListCommand,
+		appBackupSnapshotsCommand,
+		appBackupDownloadCommand,
+		appBackupCreateCommand,
+	},
 }
diff --git a/cli/app/cp.go b/cli/app/cp.go
index bfc2c789..f27b42ce 100644
--- a/cli/app/cp.go
+++ b/cli/app/cp.go
@@ -76,9 +76,9 @@ And if you want to copy that file back to your current working directory locally
 		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
 
 		if toContainer {
-			err = copyToContainer(cl, container.ID, srcPath, dstPath)
+			err = CopyToContainer(cl, container.ID, srcPath, dstPath)
 		} else {
-			err = copyFromContainer(cl, container.ID, srcPath, dstPath)
+			err = CopyFromContainer(cl, container.ID, srcPath, dstPath)
 		}
 		if err != nil {
 			logrus.Fatal(err)
@@ -106,9 +106,9 @@ func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service st
 	return "", "", "", false, errServiceMissing
 }
 
-// copyToContainer copies a file or directory from the local file system to the container.
+// CopyToContainer copies a file or directory from the local file system to the container.
 // See the possible copy modes and their documentation.
-func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := os.Stat(srcPath)
 	if err != nil {
 		return fmt.Errorf("local %s ", err)
@@ -140,7 +140,7 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -179,7 +179,7 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 		if err != nil {
 			return err
 		}
-		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+		if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
 			AttachStderr: true,
 			AttachStdin:  true,
 			AttachStdout: true,
@@ -194,9 +194,9 @@ func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri
 	return nil
 }
 
-// copyFromContainer copies a file or directory from the given container to the local file system.
+// CopyFromContainer copies a file or directory from the given container to the local file system.
 // See the possible copy modes and their documentation.
-func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
 	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
 	if err != nil {
 		if errdefs.IsNotFound(err) {
diff --git a/cli/app/restore.go b/cli/app/restore.go
index 1bf9c840..c80347f5 100644
--- a/cli/app/restore.go
+++ b/cli/app/restore.go
@@ -1,223 +1,82 @@
 package app
 
 import (
-	"context"
-	"errors"
 	"fmt"
-	"os"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/recipe"
-	recipePkg "coopcloud.tech/abra/pkg/recipe"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-type restoreConfig struct {
-	preHookCmd  string
-	postHookCmd string
+var targetPath string
+var targetPathFlag = &cli.StringFlag{
+	Name:        "target, t",
+	Usage:       "Target path",
+	Destination: &targetPath,
 }
 
 var appRestoreCommand = cli.Command{
 	Name:      "restore",
 	Aliases:   []string{"rs"},
-	Usage:     "Run app restore",
-	ArgsUsage: "<domain> <service> <file>",
+	Usage:     "Restore an app backup",
+	ArgsUsage: "<domain> <service>",
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.OfflineFlag,
-		internal.ChaosFlag,
+		targetPathFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.AppNameComplete,
-	Description: `
-Run an app restore.
-
-Pre/post hook commands are defined in the recipe configuration. Abra reads this
-configuration and run the comands in the context of the service before
-restoring the backup.
-
-Unlike "abra app backup", restore must be run on a per-service basis. You can
-not restore all services in one go. Backup files produced by Abra are
-compressed archives which use absolute paths. This allows Abra to restore
-according to standard tar command logic, i.e. the backup will be restored to
-the path it was originally backed up from.
-
-Example:
-
-    abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
-`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
-		recipe, err := recipe.Get(app.Recipe, internal.Offline)
-		if err != nil {
+		if err := recipe.EnsureExists(app.Recipe); err != nil {
 			logrus.Fatal(err)
 		}
 
 		if !internal.Chaos {
-			if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
+			if err := recipe.EnsureIsClean(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 
 			if !internal.Offline {
-				if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
+				if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
 					logrus.Fatal(err)
 				}
 			}
 
-			if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
+			if err := recipe.EnsureLatest(app.Recipe); err != nil {
 				logrus.Fatal(err)
 			}
 		}
 
-		serviceName := c.Args().Get(1)
-		if serviceName == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
-		}
-
-		backupPath := c.Args().Get(2)
-		if backupPath == "" {
-			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
-		}
-
-		if _, err := os.Stat(backupPath); err != nil {
-			if os.IsNotExist(err) {
-				logrus.Fatalf("%s doesn't exist?", backupPath)
-			}
-		}
-
-		restoreConfigs := make(map[string]restoreConfig)
-		for _, service := range recipe.Config.Services {
-			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
-				if restoreEnabled == "true" {
-					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
-					rsConfig := restoreConfig{}
-
-					logrus.Debugf("restore config detected for %s", fullServiceName)
-
-					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
-						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
-						rsConfig.preHookCmd = preHookCmd
-					}
-
-					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
-						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
-						rsConfig.postHookCmd = postHookCmd
-					}
-
-					restoreConfigs[service.Name] = rsConfig
-				}
-			}
-		}
-
-		rsConfig, ok := restoreConfigs[serviceName]
-		if !ok {
-			rsConfig = restoreConfig{}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		if snapshot != "" {
+			logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if targetPath != "" {
+			logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
+			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
+		}
+
+		if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
 			logrus.Fatal(err)
 		}
 
 		return nil
 	},
 }
-
-// runRestore does the actual restore logic.
-func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
-	// FIXME: avoid instantiating a new CLI
-	dcli, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
-
-	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
-	if err != nil {
-		return err
-	}
-
-	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
-	if rsConfig.preHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
-
-		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
-
-		preHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
-	}
-
-	backupReader, err := os.Open(backupPath)
-	if err != nil {
-		return err
-	}
-
-	content, err := archive.DecompressStream(backupReader)
-	if err != nil {
-		return err
-	}
-
-	// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
-	// files according to the paths set in the compressed archive
-	restorePath := "/"
-
-	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
-		return err
-	}
-
-	logrus.Infof("restored %s to %s", backupPath, fullServiceName)
-
-	if rsConfig.postHookCmd != "" {
-		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
-
-		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
-
-		postHookExecOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          splitCmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
-			return err
-		}
-
-		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
-	}
-
-	return nil
-}
diff --git a/cli/app/run.go b/cli/app/run.go
index 4ae68c1b..b5e0a9ce 100644
--- a/cli/app/run.go
+++ b/cli/app/run.go
@@ -91,7 +91,7 @@ var appRunCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+		if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 			logrus.Fatal(err)
 		}
 
diff --git a/cli/internal/backup.go b/cli/internal/backup.go
index 79951810..530735c9 100644
--- a/cli/internal/backup.go
+++ b/cli/internal/backup.go
@@ -1,35 +1,67 @@
 package internal
 
 import (
-	"strings"
+	"context"
+
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
 )
 
-// SafeSplit splits up a string into a list of commands safely.
-func SafeSplit(s string) []string {
-	split := strings.Split(s, " ")
-
-	var result []string
-	var inquote string
-	var block string
-	for _, i := range split {
-		if inquote == "" {
-			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
-				inquote = string(i[0])
-				block = strings.TrimPrefix(i, inquote) + " "
-			} else {
-				result = append(result, i)
-			}
-		} else {
-			if !strings.HasSuffix(i, inquote) {
-				block += i + " "
-			} else {
-				block += strings.TrimSuffix(i, inquote)
-				inquote = ""
-				result = append(result, block)
-				block = ""
-			}
-		}
+// RetrieveBackupBotContainer gets the deployed backupbot container.
+func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) {
+	ctx := context.Background()
+	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
+	if err != nil {
+		return types.Container{}, err
 	}
 
-	return result
+	logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
+
+	filters := filters.NewArgs()
+	filters.Add("name", chosenService.Spec.Name)
+	targetContainer, err := containerPkg.GetContainer(
+		ctx,
+		cl,
+		filters,
+		NoInput,
+	)
+	if err != nil {
+		return types.Container{}, err
+	}
+
+	return targetContainer, nil
+}
+
+// RunBackupCmdRemote runs a backup related command on a remote backupbot container.
+func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error {
+	execBackupListOpts := types.ExecConfig{
+		AttachStderr: true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		Cmd:          []string{"/usr/bin/backup", "--", backupCmd},
+		Detach:       false,
+		Env:          execEnv,
+		Tty:          true,
+	}
+
+	logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts)
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil {
+		return err
+	}
+
+	return nil
 }
diff --git a/cli/internal/command.go b/cli/internal/command.go
index 6f02ae1c..13c007be 100644
--- a/cli/internal/command.go
+++ b/cli/internal/command.go
@@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		Tty:          false,
 	}
 
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name)
 		shell = "/bin/sh"
 	}
@@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName,
 		execCreateOpts.Tty = false
 	}
 
-	if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
+	if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil {
 		return err
 	}
 
diff --git a/pkg/config/env.go b/pkg/config/env.go
index 62f6a71d..adedc6f4 100644
--- a/pkg/config/env.go
+++ b/pkg/config/env.go
@@ -36,6 +36,8 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+var BackupbotLabel = "coop-cloud.backupbot.enabled"
+
 // envVarModifiers is a list of env var modifier strings. These are added to
 // env vars as comments and modify their processing by Abra, e.g. determining
 // how long secrets should be.
diff --git a/pkg/container/container.go b/pkg/container/container.go
index 09d5703b..1354b0dd 100644
--- a/pkg/container/container.go
+++ b/pkg/container/container.go
@@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 		return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
 	}
 
-	if len(containers) != 1 {
+	if len(containers) > 1 {
 		var containersRaw []string
 		for _, container := range containers {
 			containerName := strings.Join(container.Names, " ")
diff --git a/pkg/service/service.go b/pkg/service/service.go
index 3d92d821..48cdce75 100644
--- a/pkg/service/service.go
+++ b/pkg/service/service.go
@@ -14,6 +14,70 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// GetService retrieves a service container based on a label. If prompt is true
+// and the retrievd count of service containers does not match 1, then a prompt
+// is presented to let the user choose. An error is returned when no service is
+// found.
+func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
+	services, err := cl.ServiceList(c, types.ServiceListOptions{})
+	if err != nil {
+		return swarm.Service{}, err
+	}
+
+	if len(services) == 0 {
+		return swarm.Service{}, fmt.Errorf("no services deployed?")
+	}
+
+	var matchingServices []swarm.Service
+	for _, service := range services {
+		if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
+			matchingServices = append(matchingServices, service)
+		}
+	}
+
+	if len(matchingServices) == 0 {
+		return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label)
+	}
+
+	if len(matchingServices) > 1 {
+		var servicesRaw []string
+		for _, service := range matchingServices {
+			serviceName := service.Spec.Name
+			created := formatter.HumanDuration(service.CreatedAt.Unix())
+			servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
+		}
+
+		if !prompt {
+			err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " "))
+			return swarm.Service{}, err
+		}
+
+		logrus.Warnf("ambiguous service list received, prompting for input")
+
+		var response string
+		prompt := &survey.Select{
+			Message: "which service are you looking for?",
+			Options: servicesRaw,
+		}
+
+		if err := survey.AskOne(prompt, &response); err != nil {
+			return swarm.Service{}, err
+		}
+
+		chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
+		for _, service := range matchingServices {
+			serviceName := strings.ToLower(service.Spec.Name)
+			if serviceName == chosenService {
+				return service, nil
+			}
+		}
+
+		logrus.Panic("failed to match chosen service")
+	}
+
+	return matchingServices[0], nil
+}
+
 // GetService retrieves a service container. If prompt is true and the retrievd
 // count of service containers does not match 1, then a prompt is presented to
 // let the user choose. A count of 0 is handled gracefully.
diff --git a/pkg/upstream/container/exec.go b/pkg/upstream/container/exec.go
index e811481a..82a2c570 100644
--- a/pkg/upstream/container/exec.go
+++ b/pkg/upstream/container/exec.go
@@ -13,7 +13,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error {
+// RunExec runs a command on a remote container. io.Writer corresponds to the
+// command output.
+func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string,
+	execConfig *types.ExecConfig) (io.Writer, error) {
 	ctx := context.Background()
 
 	// We need to check the tty _before_ we do the ContainerExecCreate, because
@@ -21,22 +24,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 	// there's no easy way to clean those up). But also in order to make "not
 	// exist" errors take precedence we do a dummy inspect first.
 	if _, err := client.ContainerInspect(ctx, containerID); err != nil {
-		return err
+		return nil, err
 	}
 	if !execConfig.Detach {
 		if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	response, err := client.ContainerExecCreate(ctx, containerID, *execConfig)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	execID := response.ID
 	if execID == "" {
-		return errors.New("exec ID empty")
+		return nil, errors.New("exec ID empty")
 	}
 
 	if execConfig.Detach {
@@ -44,13 +47,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string
 			Detach: execConfig.Detach,
 			Tty:    execConfig.Tty,
 		}
-		return client.ContainerExecStart(ctx, execID, execStartCheck)
+		return nil, client.ContainerExecStart(ctx, execID, execStartCheck)
 	}
 	return interactiveExec(ctx, dockerCli, client, execConfig, execID)
 }
 
 func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client,
-	execConfig *types.ExecConfig, execID string) error {
+	execConfig *types.ExecConfig, execID string) (io.Writer, error) {
 	// Interactive exec requested.
 	var (
 		out, stderr io.Writer
@@ -76,7 +79,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 	}
 	resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck)
 	if err != nil {
-		return err
+		return out, err
 	}
 	defer resp.Close()
 
@@ -107,10 +110,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie
 
 	if err := <-errCh; err != nil {
 		logrus.Debugf("Error hijack: %s", err)
-		return err
+		return out, err
 	}
 
-	return getExecExitStatus(ctx, client, execID)
+	return out, getExecExitStatus(ctx, client, execID)
 }
 
 func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {

From 72c20e0039181a4add20aa89dc04420a5203f1f3 Mon Sep 17 00:00:00 2001
From: 3wc <3wc@doesthisthing.work>
Date: Tue, 26 Mar 2024 21:07:38 -0300
Subject: [PATCH 73/76] fix: make installer work again

---
 scripts/installer/installer | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scripts/installer/installer b/scripts/installer/installer
index b4c5f787..b6e0e543 100755
--- a/scripts/installer/installer
+++ b/scripts/installer/installer
@@ -45,7 +45,9 @@ function install_abra_release {
   fi
 
   ARCH=$(uname -m)
-  if [[ $ARCH =~ "aarch64" ]]; then
+  if [[ $ARCH =~ "x86_64" ]]; then
+      ARCH="amd64"
+  elif [[ $ARCH =~ "aarch64" ]]; then
       ARCH="arm64"
   elif [[ $ARCH =~ "armv5l" ]]; then
       ARCH="armv5"
@@ -55,7 +57,7 @@ function install_abra_release {
       ARCH="armv7"
   fi
   PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')_$ARCH
-  FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM""
+  FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM".tar.gz"
   sed_command_rel='s/.*"assets":\[\{[^]]*"name":"'$FILENAME'"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
   sed_command_checksums='s/.*"assets":\[\{[^\]*"name":"checksums.txt"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
 
@@ -65,7 +67,7 @@ function install_abra_release {
 
   checksums=$(wget -q -O- $checksums_url)
   checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p')
-  abra_download="/tmp/abra-download"
+  abra_download="/tmp/abra-download.tar.gz"
 
   echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..."
 
@@ -77,7 +79,10 @@ function install_abra_release {
       exit 1
   fi
   echo "$(tput setaf 2)check successful!$(tput sgr0)"
-  mv "$abra_download" "$HOME/.local/bin/abra"
+  cd /tmp/
+  tar xf abra-download.tar.gz
+  mv abra "$HOME/.local/bin/abra"
+  tar tf abra-download.tar.gz | xargs rm -f
   chmod +x "$HOME/.local/bin/abra"
 
   x=$(echo $PATH | grep $HOME/.local/bin)

From d5ac3958a4543ca7ee94f5c39bf2147d4a8089c4 Mon Sep 17 00:00:00 2001
From: p4u1 <p4u1_f4u1@riseup.net>
Date: Mon, 11 Mar 2024 15:24:41 +0100
Subject: [PATCH 74/76] feat: add retries to app volume remove

---
 cli/app/remove.go                             | 45 ++++---------------
 cli/app/volume.go                             |  7 +--
 pkg/client/volumes.go                         | 33 +++++++++++---
 .../client/volumes_test.go                    |  2 +-
 tests/integration/app_remove.bats             |  3 --
 tests/integration/app_volume.bats             |  6 ---
 6 files changed, 40 insertions(+), 56 deletions(-)
 rename cli/app/remove_test.go => pkg/client/volumes_test.go (96%)

diff --git a/cli/app/remove.go b/cli/app/remove.go
index 82499fd1..f9167d82 100644
--- a/cli/app/remove.go
+++ b/cli/app/remove.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -13,7 +12,6 @@ import (
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/volume"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -112,28 +110,19 @@ flag.
 			logrus.Fatal(err)
 		}
 
-		volumeListOptions := volume.ListOptions{fs}
-		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
-		volumeList := volumeListOKBody.Volumes
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, fs)
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		volumeNames := client.GetVolumeNames(volumeList)
 
-		var vols []string
-		for _, vol := range volumeList {
-			vols = append(vols, vol.Name)
-		}
-
-		if len(vols) > 0 {
-			for _, vol := range vols {
-				err = retryFunc(5, func() error {
-					return cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
-				})
-				if err != nil {
-					log.Fatalf("removing volumes failed: %s", err)
-				}
-				logrus.Info(fmt.Sprintf("volume %s removed", vol))
+		if len(volumeNames) > 0 {
+			err := client.RemoveVolumes(cl, context.Background(), volumeNames, internal.Force, 5)
+			if err != nil {
+				log.Fatalf("removing volumes failed: %s", err)
 			}
+
+			logrus.Infof("%d volumes removed successfully", len(volumeNames))
 		} else {
 			logrus.Info("no volumes to remove")
 		}
@@ -147,21 +136,3 @@ flag.
 		return nil
 	},
 }
-
-// retryFunc retries the given function for the given retries. After the nth
-// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
-// It returns an error if the function still failed after the last retry.
-func retryFunc(retries int, fn func() error) error {
-	for i := 0; i < retries; i++ {
-		err := fn()
-		if err == nil {
-			return nil
-		}
-		if i+1 < retries {
-			sleep := time.Duration(i+1) * time.Duration(i+1)
-			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
-			time.Sleep(sleep * time.Second)
-		}
-	}
-	return fmt.Errorf("%d retries failed", retries)
-}
diff --git a/cli/app/volume.go b/cli/app/volume.go
index b64ce69c..4fb72c6b 100644
--- a/cli/app/volume.go
+++ b/cli/app/volume.go
@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"log"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -131,12 +132,12 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		}
 
 		if len(volumesToRemove) > 0 {
-			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
+			err := client.RemoveVolumes(cl, context.Background(), volumesToRemove, internal.Force, 5)
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatalf("removing volumes failed: %s", err)
 			}
 
-			logrus.Info("volumes removed successfully")
+			logrus.Infof("%d volumes removed successfully", len(volumesToRemove))
 		} else {
 			logrus.Info("no volumes removed")
 		}
diff --git a/pkg/client/volumes.go b/pkg/client/volumes.go
index 4afd60c2..f295b081 100644
--- a/pkg/client/volumes.go
+++ b/pkg/client/volumes.go
@@ -2,15 +2,17 @@ package client
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
 )
 
 func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
-	volumeListOptions := volume.ListOptions{fs}
-	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
+	volumeListOKBody, err := cl.VolumeList(ctx, volume.ListOptions{Filters: fs})
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
 		return volumeList, err
@@ -29,13 +31,32 @@ func GetVolumeNames(volumes []*volume.Volume) []string {
 	return volumeNames
 }
 
-func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
+func RemoveVolumes(cl *client.Client, ctx context.Context, volumeNames []string, force bool, retries int) error {
 	for _, volName := range volumeNames {
-		err := cl.VolumeRemove(ctx, volName, force)
+		err := retryFunc(5, func() error {
+			return cl.VolumeRemove(context.Background(), volName, force)
+		})
 		if err != nil {
-			return err
+			return fmt.Errorf("volume %s: %s", volName, err)
 		}
 	}
-
 	return nil
 }
+
+// retryFunc retries the given function for the given retries. After the nth
+// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
+// It returns an error if the function still failed after the last retry.
+func retryFunc(retries int, fn func() error) error {
+	for i := 0; i < retries; i++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		if i+1 < retries {
+			sleep := time.Duration(i+1) * time.Duration(i+1)
+			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
+			time.Sleep(sleep * time.Second)
+		}
+	}
+	return fmt.Errorf("%d retries failed", retries)
+}
diff --git a/cli/app/remove_test.go b/pkg/client/volumes_test.go
similarity index 96%
rename from cli/app/remove_test.go
rename to pkg/client/volumes_test.go
index c3c9f8a0..792af002 100644
--- a/cli/app/remove_test.go
+++ b/pkg/client/volumes_test.go
@@ -1,4 +1,4 @@
-package app
+package client
 
 import (
 	"fmt"
diff --git a/tests/integration/app_remove.bats b/tests/integration/app_remove.bats
index 4cf6f07a..c0be90da 100644
--- a/tests/integration/app_remove.bats
+++ b/tests/integration/app_remove.bats
@@ -104,9 +104,6 @@ teardown(){
 
   _undeploy_app
 
-  # TODO: should wait as long as volume is no longer in use
-  sleep 10
-
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --no-input
   assert_success
 
diff --git a/tests/integration/app_volume.bats b/tests/integration/app_volume.bats
index 99acc6c9..6327821c 100644
--- a/tests/integration/app_volume.bats
+++ b/tests/integration/app_volume.bats
@@ -78,9 +78,6 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 10
-
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'
@@ -92,9 +89,6 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 10
-
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'

From 2ecace3e90e9874140d1b8f8a2ce583847daa645 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sun, 31 Mar 2024 01:39:00 +0100
Subject: [PATCH 75/76] fix: add missing packages on final layer

Closes https://git.coopcloud.tech/coop-cloud/organising/issues/598
---
 Dockerfile | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 21b0f52e..ad13b47a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,23 +1,29 @@
+# Build image
 FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
 RUN apk add --no-cache \
-  ca-certificates \
   gcc \
   git \
   make \
   musl-dev
 
-RUN update-ca-certificates
-
 COPY . /app
 
 WORKDIR /app
 
 RUN CGO_ENABLED=0 make build
 
-FROM scratch
+# Release image ("slim")
+FROM alpine:3.19.1
+
+RUN apk add --no-cache \
+  ca-certificates \
+  git \
+  openssh
+
+RUN update-ca-certificates
 
 COPY --from=build /app/abra /abra
 

From 63ea58ffaae1caff67a6f49aa8c20de4f123e30c Mon Sep 17 00:00:00 2001
From: Mayel de Borniol <git@mayel.space>
Date: Mon, 1 Apr 2024 18:51:53 +0100
Subject: [PATCH 76/76] add relevant command to error message

---
 pkg/upstream/stack/stack.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/upstream/stack/stack.go b/pkg/upstream/stack/stack.go
index c2d1e214..d8e7bdd9 100644
--- a/pkg/upstream/stack/stack.go
+++ b/pkg/upstream/stack/stack.go
@@ -233,7 +233,7 @@ func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAP
 		network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
 		switch {
 		case dockerClient.IsErrNotFound(err):
-			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
+			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed, which you can do by running this on the server: docker network create -d overlay proxy", networkName)
 		case err != nil:
 			return err
 		case network.Scope != "swarm":