feat: abra app cp enhancements

This implements proper modifier support in the env file using this new fork of the godotenv library. The modifier implementation is quite basic for but can be improved later if needed. See this commit for the actual implementation.

Because we are now using proper modifer parsing, it does not affect the parsing of value, so this is possible again:
```
MY_VAR="#foo"
```
Closes coop-cloud/organising#535

cli/app/cp.go

@@ -2,19 +2,24 @@ package app
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"os"
+	"path"
+	"path/filepath"
 	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/client"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/container"
+	containerPkg "coopcloud.tech/abra/pkg/container"
 	"coopcloud.tech/abra/pkg/formatter"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -49,46 +54,14 @@ And if you want to copy that file back to your current working directory locally
 		dst := c.Args().Get(2)
 		if src == "" {
 			logrus.Fatal("missing <src> argument")
-		} else if dst == "" {
+		}
+		if dst == "" {
 			logrus.Fatal("missing <dest> argument")
 		}
 
-		parsedSrc := strings.SplitN(src, ":", 2)
-		parsedDst := strings.SplitN(dst, ":", 2)
-		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
-		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
-			logrus.Fatal(errorMsg)
-		} else if len(parsedSrc) != 2 {
-			if len(parsedDst) != 2 {
-				logrus.Fatal(errorMsg)
-			}
-		} else if len(parsedDst) != 2 {
-			if len(parsedSrc) != 2 {
-				logrus.Fatal(errorMsg)
-			}
-		}
-
-		var service string
-		var srcPath string
-		var dstPath string
-		isToContainer := false // <container:src> <dst>
-		if len(parsedSrc) == 2 {
-			service = parsedSrc[0]
-			srcPath = parsedSrc[1]
-			dstPath = dst
-			logrus.Debugf("assuming transfer is coming FROM the container")
-		} else if len(parsedDst) == 2 {
-			service = parsedDst[0]
-			dstPath = parsedDst[1]
-			srcPath = src
-			isToContainer = true // <src> <container:dst>
-			logrus.Debugf("assuming transfer is going TO the container")
-		}
-
-		if isToContainer {
-			if _, err := os.Stat(srcPath); os.IsNotExist(err) {
-				logrus.Fatalf("%s does not exist locally?", srcPath)
-			}
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst)
+		if err != nil {
+			logrus.Fatal(err)
 		}
 
 		cl, err := client.New(app.Server)
@@ -96,7 +69,18 @@ And if you want to copy that file back to your current working directory locally
 			logrus.Fatal(err)
 		}
 
-		if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
+		container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+
+		if toContainer {
+			err = copyToContainer(cl, container.ID, srcPath, dstPath)
+		} else {
+			err = copyFromContainer(cl, container.ID, srcPath, dstPath)
+		}
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
@@ -104,46 +88,292 @@ And if you want to copy that file back to your current working directory locally
 	},
 }
 
-func configureAndCp(
-	c *cli.Context,
-	cl *dockerClient.Client,
-	app config.App,
-	srcPath string,
-	dstPath string,
-	service string,
-	isToContainer bool) error {
-	filters := filters.NewArgs()
-	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))
+var errServiceMissing = errors.New("one of <src>/<dest> arguments must take $SERVICE:$PATH form")
 
-	container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput)
+// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH
+func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) {
+	parsedSrc := strings.SplitN(src, ":", 2)
+	parsedDst := strings.SplitN(dst, ":", 2)
+	if len(parsedSrc)+len(parsedDst) != 3 {
+		return "", "", "", false, errServiceMissing
+	}
+	if len(parsedSrc) == 2 {
+		return parsedSrc[1], dst, parsedSrc[0], false, nil
+	}
+	if len(parsedDst) == 2 {
+		return src, parsedDst[1], parsedDst[0], true, nil
+	}
+	return "", "", "", false, errServiceMissing
+}
+
+// copyToContainer copies a file or directory from the local file system to the container.
+// See the possible copy modes and their documentation.
+func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := os.Stat(srcPath)
 	if err != nil {
-		logrus.Fatal(err)
+		return fmt.Errorf("local %s ", err)
 	}
 
-	logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server)
+	dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath)
+	dstExists := true
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			dstExists = false
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	}
 
-	if isToContainer {
-		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		content, err := archive.TarWithOptions(srcPath, toTarOpts)
-		if err != nil {
-			logrus.Fatal(err)
-		}
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists)
+	if err != nil {
+		return err
+	}
+	movePath := ""
+	switch mode {
+	case CopyModeDirToDir:
+		// Add the src directory to the destination path
+		_, srcDir := path.Split(srcPath)
+		dstPath = path.Join(dstPath, srcDir)
 
-		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-		if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil {
-			logrus.Fatal(err)
-		}
-	} else {
-		content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath)
+		// Make sure the dst directory exits.
+		dcli, err := command.NewDockerCli()
 		if err != nil {
-			logrus.Fatal(err)
+			return err
 		}
-		defer content.Close()
-		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-			logrus.Fatal(err)
+		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mkdir", "-p", dstPath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return fmt.Errorf("create remote directory: %s", err)
+		}
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		movePath = dstPath
+		dstPath = "/tmp"
+	}
+
+	toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+	content, err := archive.TarWithOptions(srcPath, toTarOpts)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("copy %s from local to %s on container", srcPath, dstPath)
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil {
+		return err
+	}
+
+	if movePath != "" {
+		_, srcFile := path.Split(srcPath)
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			return err
+		}
+		if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          []string{"mv", path.Join("/tmp", srcFile), movePath},
+			Detach:       false,
+			Tty:          true,
+		}); err != nil {
+			return fmt.Errorf("create remote directory: %s", err)
 		}
 	}
 
 	return nil
 }
+
+// copyFromContainer copies a file or directory from the given container to the local file system.
+// See the possible copy modes and their documentation.
+func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error {
+	srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return fmt.Errorf("remote: %s does not exist", srcPath)
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	}
+
+	dstStat, err := os.Stat(dstPath)
+	dstExists := true
+	var dstMode os.FileMode
+	if err != nil {
+		if os.IsNotExist(err) {
+			dstExists = false
+		} else {
+			return fmt.Errorf("remote path: %s", err)
+		}
+	} else {
+		dstMode = dstStat.Mode()
+	}
+
+	mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists)
+	if err != nil {
+		return err
+	}
+
+	moveDstDir := ""
+	moveDstFile := ""
+	switch mode {
+	case CopyModeFileToFile:
+		// Remove the file component from the path, since docker can only copy
+		// to a directory.
+		dstPath, _ = path.Split(dstPath)
+	case CopyModeFileToFileRename:
+		// Copy the file to the temp directory and move it to its dstPath
+		// afterwards.
+		moveDstFile = dstPath
+		dstPath = "/tmp"
+	case CopyModeFilesToDir:
+		// Copy the directory to the temp directory and move it to its
+		// dstPath afterwards.
+		moveDstDir = path.Join(dstPath, "/")
+		dstPath = "/tmp"
+
+		// Make sure the temp directory always gets removed
+		defer os.Remove(path.Join("/tmp"))
+	}
+
+	content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath)
+	if err != nil {
+		return fmt.Errorf("copy: %s", err)
+	}
+	defer content.Close()
+	if err := archive.Untar(content, dstPath, &archive.TarOptions{
+		NoOverwriteDirNonDir: true,
+		Compression:          archive.Gzip,
+		NoLchown:             true,
+	}); err != nil {
+		return fmt.Errorf("untar: %s", err)
+	}
+
+	if moveDstFile != "" {
+		_, srcFile := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil {
+			return err
+		}
+	}
+	if moveDstDir != "" {
+		_, srcDir := path.Split(strings.TrimSuffix(srcPath, "/"))
+		if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var (
+	ErrCopyDirToFile  = fmt.Errorf("can't copy dir to file")
+	ErrDstDirNotExist = fmt.Errorf("destination directory does not exist")
+)
+
+type CopyMode int
+
+const (
+	// Copy a src file to a dest file. The src and dest file names are the same.
+	//  <dir_src>/<file> + <dir_dst>/<file> -> <dir_dst>/<file>
+	CopyModeFileToFile = CopyMode(iota)
+	// Copy a src file to a dest file. The src and dest file names are  not the same.
+	//  <dir_src>/<file_src> + <dir_dst>/<file_dst> -> <dir_dst>/<file_dst>
+	CopyModeFileToFileRename
+	// Copy a src file to dest directory. The dest file gets created in the dest
+	// folder with the src filename.
+	//  <dir_src>/<file> + <dir_dst> -> <dir_dst>/<file>
+	CopyModeFileToDir
+	// Copy a src directory to dest directory.
+	//  <dir_src> + <dir_dst> -> <dir_dst>/<dir_src>
+	CopyModeDirToDir
+	// Copy all files in the src directory to the dest directory. This works recursively.
+	//  <dir_src>/ + <dir_dst> -> <dir_dst>/<files_from_dir_src>
+	CopyModeFilesToDir
+)
+
+// copyMode takes a src and dest path and file mode to determine the copy mode.
+// See the possible copy modes and their documentation.
+func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) {
+	_, srcFile := path.Split(srcPath)
+	_, dstFile := path.Split(dstPath)
+	if srcMode.IsDir() {
+		if !dstExists {
+			return -1, ErrDstDirNotExist
+		}
+		if dstMode.IsDir() {
+			if strings.HasSuffix(srcPath, "/") {
+				return CopyModeFilesToDir, nil
+			}
+			return CopyModeDirToDir, nil
+		}
+		return -1, ErrCopyDirToFile
+	}
+
+	if dstMode.IsDir() {
+		return CopyModeFileToDir, nil
+	}
+
+	if srcFile != dstFile {
+		return CopyModeFileToFileRename, nil
+	}
+
+	return CopyModeFileToFile, nil
+}
+
+// moveDir moves all files from a source path to the destination path recursively.
+func moveDir(sourcePath, destPath string) error {
+	return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath))
+		if info.IsDir() {
+			err := os.Mkdir(newPath, info.Mode())
+			if err != nil {
+				if os.IsExist(err) {
+					return nil
+				}
+				return err
+			}
+		}
+		if info.Mode().IsRegular() {
+			return moveFile(p, newPath)
+		}
+		return nil
+	})
+}
+
+// moveFile moves a file from a source path to a destination path.
+func moveFile(sourcePath, destPath string) error {
+	inputFile, err := os.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	outputFile, err := os.Create(destPath)
+	if err != nil {
+		inputFile.Close()
+		return err
+	}
+	defer outputFile.Close()
+	_, err = io.Copy(outputFile, inputFile)
+	inputFile.Close()
+	if err != nil {
+		return err
+	}
+
+	// Remove file after succesfull copy.
+	err = os.Remove(sourcePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}

cli/app/cp_test.go

@@ -0,0 +1,113 @@
+package app
+
+import (
+	"os"
+	"testing"
+)
+
+func TestParse(t *testing.T) {
+	tests := []struct {
+		src         string
+		dst         string
+		srcPath     string
+		dstPath     string
+		service     string
+		toContainer bool
+		err         error
+	}{
+		{src: "foo", dst: "bar", err: errServiceMissing},
+		{src: "app:foo", dst: "app:bar", err: errServiceMissing},
+		{src: "app:foo", dst: "bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: false},
+		{src: "foo", dst: "app:bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: true},
+	}
+
+	for i, tc := range tests {
+		srcPath, dstPath, service, toContainer, err := parseSrcAndDst(tc.src, tc.dst)
+		if srcPath != tc.srcPath {
+			t.Errorf("[%d] srcPath: want (%s), got(%s)", i, tc.srcPath, srcPath)
+		}
+		if dstPath != tc.dstPath {
+			t.Errorf("[%d] dstPath: want (%s), got(%s)", i, tc.dstPath, dstPath)
+		}
+		if service != tc.service {
+			t.Errorf("[%d] service: want (%s), got(%s)", i, tc.service, service)
+		}
+		if toContainer != tc.toContainer {
+			t.Errorf("[%d] toConainer: want (%t), got(%t)", i, tc.toContainer, toContainer)
+		}
+		if err == nil && tc.err != nil && err.Error() != tc.err.Error() {
+			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
+		}
+	}
+}
+
+func TestCopyMode(t *testing.T) {
+	tests := []struct {
+		srcPath   string
+		dstPath   string
+		srcMode   os.FileMode
+		dstMode   os.FileMode
+		dstExists bool
+		mode      CopyMode
+		err       error
+	}{
+		{
+			srcPath:   "foo.txt",
+			dstPath:   "foo.txt",
+			srcMode:   os.ModePerm,
+			dstMode:   os.ModePerm,
+			dstExists: true,
+			mode:      CopyModeFileToFile,
+		},
+		{
+			srcPath:   "foo.txt",
+			dstPath:   "bar.txt",
+			srcMode:   os.ModePerm,
+			dstExists: true,
+			mode:      CopyModeFileToFileRename,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModeDir,
+			dstExists: true,
+			mode:      CopyModeDirToDir,
+		},
+		{
+			srcPath:   "foo/",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModeDir,
+			dstExists: true,
+			mode:      CopyModeFilesToDir,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstExists: false,
+			mode:      -1,
+			err:       ErrDstDirNotExist,
+		},
+		{
+			srcPath:   "foo",
+			dstPath:   "foo",
+			srcMode:   os.ModeDir,
+			dstMode:   os.ModePerm,
+			dstExists: true,
+			mode:      -1,
+			err:       ErrCopyDirToFile,
+		},
+	}
+
+	for i, tc := range tests {
+		mode, err := copyMode(tc.srcPath, tc.dstPath, tc.srcMode, tc.dstMode, tc.dstExists)
+		if mode != tc.mode {
+			t.Errorf("[%d] mode: want (%d), got(%d)", i, tc.mode, mode)
+		}
+		if err != tc.err {
+			t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err)
+		}
+	}
+}

cli/app/new.go

@@ -97,7 +97,7 @@ var appNewCommand = cli.Command{
 		var secrets AppSecrets
 		var secretTable *jsontable.JSONTable
 		if internal.Secrets {
-			sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
+			sampleEnv, err := recipe.SampleEnv()
 			if err != nil {
 				logrus.Fatal(err)
 			}
@@ -168,7 +168,7 @@ var appNewCommand = cli.Command{
 type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
-func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) {
+func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
 	// NOTE(d1): trim to match app.StackName() implementation
 	if len(sanitisedAppName) > 45 {
 		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
@@ -217,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
 }
 
 // promptForSecrets asks if we should generate secrets for a new app.
-func promptForSecrets(recipeName string, secretsConfig map[string]string) error {
+func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
 	if len(secretsConfig) == 0 {
 		logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
 		return nil

cli/app/secret.go

@@ -20,19 +20,23 @@ import (
 	"github.com/urfave/cli"
 )
 
-var allSecrets bool
-var allSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &allSecrets,
-	Usage:       "Generate all secrets",
-}
+var (
+	allSecrets     bool
+	allSecretsFlag = &cli.BoolFlag{
+		Name:        "all, a",
+		Destination: &allSecrets,
+		Usage:       "Generate all secrets",
+	}
+)
 
-var rmAllSecrets bool
-var rmAllSecretsFlag = &cli.BoolFlag{
-	Name:        "all, a",
-	Destination: &rmAllSecrets,
-	Usage:       "Remove all secrets",
-}
+var (
+	rmAllSecrets     bool
+	rmAllSecretsFlag = &cli.BoolFlag{
+		Name:        "all, a",
+		Destination: &rmAllSecrets,
+		Usage:       "Remove all secrets",
+	}
+)
 
 var appSecretGenerateCommand = cli.Command{
 	Name:      "generate",
@@ -87,28 +91,22 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		secretsToCreate := make(map[string]string)
-		if allSecrets {
-			secretsToCreate = secretsConfig
-		} else {
+		if !allSecrets {
 			secretName := c.Args().Get(1)
 			secretVersion := c.Args().Get(2)
-			matches := false
-			for name := range secretsConfig {
-				if secretName == name {
-					secretsToCreate[name] = secretVersion
-					matches = true
-				}
-			}
-
-			if !matches {
+			s, ok := secrets[secretName]
+			if !ok {
 				logrus.Fatalf("%s doesn't exist in the env config?", secretName)
 			}
+			s.Version = secretVersion
+			secrets = map[string]secret.SecretValue{
+				secretName: s,
+			}
 		}
 
 		cl, err := client.New(app.Server)
@@ -116,7 +114,7 @@ var appSecretGenerateCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
+		secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -276,7 +274,7 @@ Example:
 			logrus.Fatal(err)
 		}
 
-		secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
+		secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -311,12 +309,7 @@ Example:
 
 		match := false
 		secretToRm := c.Args().Get(1)
-		for secretName, secretValue := range secretsConfig {
-			val, err := secret.ParseSecretValue(secretValue)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
+		for secretName, val := range secrets {
 			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 			if _, ok := remoteSecretNames[secretRemoteName]; ok {
 				if secretToRm != "" {

go.mod

@@ -4,8 +4,8 @@ go 1.21
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd
 	github.com/AlecAivazis/survey/v2 v2.3.7
-	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/docker/cli v24.0.7+incompatible
 	github.com/docker/distribution v2.8.3+incompatible

go.sum

@@ -51,12 +51,12 @@ coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd h1:dctCkMhcsgIWMrkB1Br8S0RJF17eG+LKiqcXXVr3mdU=
+git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
 github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
-github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg=
-github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -1315,6 +1315,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

pkg/compose/compose.go

@@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
 			return false, err
 		}
@@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
 		opts := stack.Deploy{Composefiles: []string{composeFile}}
 
 		envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-		sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+		sampleEnv, err := config.ReadEnv(envSamplePath)
 		if err != nil {
 			return err
 		}

pkg/config/app.go

@@ -25,6 +25,9 @@ import (
 // AppEnv is a map of the values in an apps env config
 type AppEnv = map[string]string
 
+// AppModifiers is a map of modifiers in an apps env config
+type AppModifiers = map[string]map[string]string
+
 // AppName is AppName
 type AppName = string
 
@@ -150,7 +153,7 @@ func (a ByName) Less(i, j int) bool {
 }
 
 func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
-	env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
+	env, err := ReadEnv(appFile.Path)
 	if err != nil {
 		return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
 	}

pkg/config/env.go

@@ -12,7 +12,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/Autonomic-Cooperative/godotenv"
+	"git.coopcloud.tech/coop-cloud/godotenv"
 	"github.com/sirupsen/logrus"
 )
 
@@ -55,45 +55,34 @@ func GetServers() ([]string, error) {
 	return servers, nil
 }
 
-// ReadEnvOptions modifies the ReadEnv processing of env vars.
-type ReadEnvOptions struct {
-	IncludeModifiers bool
-}
-
-// ContainsEnvVarModifier determines if an env var contains a modifier.
-func ContainsEnvVarModifier(envVar string) bool {
-	for _, mod := range envVarModifiers {
-		if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
-			return true
-		}
-	}
-	return false
-}
-
 // ReadEnv loads an app envivornment into a map.
-func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
+func ReadEnv(filePath string) (AppEnv, error) {
 	var envVars AppEnv
 
-	envVars, err := godotenv.Read(filePath)
+	envVars, _, err := godotenv.Read(filePath)
 	if err != nil {
 		return nil, err
 	}
 
-	//	for idx, envVar := range envVars {
-	//		if strings.Contains(envVar, "#") {
-	//			if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
-	//				continue
-	//			}
-	//			vals := strings.Split(envVar, "#")
-	//			envVars[idx] = strings.TrimSpace(vals[0])
-	//		}
-	//	}
-
 	logrus.Debugf("read %s from %s", envVars, filePath)
 
 	return envVars, nil
 }
 
+// ReadEnv loads an app envivornment and their modifiers in two different maps.
+func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) {
+	var envVars AppEnv
+
+	envVars, mods, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, mods, err
+	}
+
+	logrus.Debugf("read %s from %s", envVars, filePath)
+
+	return envVars, mods, nil
+}
+
 // ReadServerNames retrieves all server names.
 func ReadServerNames() ([]string, error) {
 	serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
@@ -227,7 +216,7 @@ func CheckEnv(app App) ([]EnvVar, error) {
 		return envVars, err
 	}
 
-	envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
+	envSample, err := ReadEnv(envSamplePath)
 	if err != nil {
 		return envVars, err
 	}

pkg/config/env_test.go

@@ -13,15 +13,21 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 )
 
-var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
-var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+var (
+	TestFolder    = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+	ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+)
 
 // make sure these are in alphabetical order
-var TFolders = []string{"folder1", "folder2"}
-var TFiles = []string{"bar.env", "foo.env"}
+var (
+	TFolders = []string{"folder1", "folder2"}
+	TFiles   = []string{"bar.env", "foo.env"}
+)
 
-var AppName = "ecloud"
-var ServerName = "evil.corp"
+var (
+	AppName    = "ecloud"
+	ServerName = "evil.corp"
+)
 
 var ExpectedAppEnv = config.AppEnv{
 	"DOMAIN": "ecloud.evil.corp",
@@ -71,7 +77,7 @@ func TestGetAllFilesInDirectory(t *testing.T) {
 }
 
 func TestReadEnv(t *testing.T) {
-	env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
+	env, err := config.ReadEnv(ExpectedAppFile.Path)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -149,7 +155,7 @@ func TestCheckEnv(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	envSample, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -183,7 +189,7 @@ func TestCheckEnvError(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	envSample, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,19 +217,7 @@ func TestCheckEnvError(t *testing.T) {
 	}
 }
 
-func TestContainsEnvVarModifier(t *testing.T) {
-	if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
-		t.Fatal("FOO contains no env var modifier")
-	}
-
-	if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
-		t.Fatal("FOO contains an env var modifier (length)")
-	}
-}
-
 func TestEnvVarCommentsRemoved(t *testing.T) {
-	t.Skip("https://git.coopcloud.tech/coop-cloud/organising/issues/535")
-
 	offline := true
 	r, err := recipe.Get("abra-test-recipe", offline)
 	if err != nil {
@@ -231,7 +225,7 @@ func TestEnvVarCommentsRemoved(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	envSample, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -263,12 +257,19 @@ func TestEnvVarModifiersIncluded(t *testing.T) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
+	envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
-		t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
+	if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "v1") {
+		t.Errorf("value should be 'v1', got: '%s'", envSample["SECRET_TEST_PASS_TWO_VERSION"])
+	}
+	if modifiers == nil || modifiers["SECRET_TEST_PASS_TWO_VERSION"] == nil {
+		t.Errorf("no modifiers included")
+	} else {
+		if modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"] != "10" {
+			t.Errorf("length modifier should be '10', got: '%s'", modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"])
+		}
 	}
 }

pkg/container/container.go

@@ -68,3 +68,15 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
 
 	return containers[0], nil
 }
+
+// GetContainerFromStackAndService retrieves the container for the given stack and service.
+func GetContainerFromStackAndService(cl *client.Client, stack, service string) (types.Container, error) {
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", stack, service))
+
+	container, err := GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return types.Container{}, err
+	}
+	return container, nil
+}

pkg/lint/recipe.go

@@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
 // therefore no matching traefik deploy label will be present.
 func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
 	}

pkg/recipe/recipe.go

@@ -227,7 +227,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}
 
 	envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
+	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		return Recipe{}, err
 	}
@@ -255,9 +255,9 @@ func Get(recipeName string, offline bool) (Recipe, error) {
 	}, nil
 }
 
-func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
+func (r Recipe) SampleEnv() (map[string]string, error) {
 	envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
-	sampleEnv, err := config.ReadEnv(envSamplePath, opts)
+	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
 		return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
 	}

pkg/secret/secret.go

@@ -21,9 +21,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
+// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
 // secret definition.
-type secretValue struct {
+type SecretValue struct {
 	Version string
 	Length  int
 }
@@ -35,7 +35,6 @@ func GeneratePasswords(count, length uint) ([]string, error) {
 		length,
 		passgen.AlphabetDefault,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -54,7 +53,6 @@ func GeneratePassphrases(count uint) ([]string, error) {
 		passgen.PassphraseCasingDefault,
 		passgen.WordListDefault,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -69,18 +67,20 @@ func GeneratePassphrases(count uint) ([]string, error) {
 // and some times you don't (as the caller). We need to be able to handle the
 // "app new" case where we pass in the .env.sample and the "secret generate"
 // case where the app is created.
-func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
-	secretConfigs := make(map[string]string)
-
-	appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
+func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
+	appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
 	if err != nil {
-		return secretConfigs, err
+		return nil, err
 	}
 
 	opts := stack.Deploy{Composefiles: composeFiles}
 	config, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
-		return secretConfigs, err
+		return nil, err
+	}
+	configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
+	if err != nil {
+		return nil, err
 	}
 
 	var enabledSecrets []string
@@ -92,12 +92,13 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 
 	if len(enabledSecrets) == 0 {
 		logrus.Debugf("not generating app secrets, none enabled in recipe config")
-		return secretConfigs, nil
+		return nil, nil
 	}
 
+	secretValues := map[string]SecretValue{}
 	for secretId, secretConfig := range config.Secrets {
 		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
-			return secretConfigs, fmt.Errorf("missing version for secret? (%s)", secretId)
+			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
 		}
 
 		if !(slices.Contains(enabledSecrets, secretId)) {
@@ -107,60 +108,47 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
 
 		lastIdx := strings.LastIndex(secretConfig.Name, "_")
 		secretVersion := secretConfig.Name[lastIdx+1:]
-		secretConfigs[secretId] = secretVersion
+		value := SecretValue{Version: secretVersion}
+
+		// Check if the length modifier is set for this secret.
+		for k, v := range appModifiers {
+			// configWithoutEnv contains the raw name as defined in the compose.yaml
+			if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
+				continue
+			}
+			lengthRaw, ok := v["length"]
+			if ok {
+				length, err := strconv.Atoi(lengthRaw)
+				if err != nil {
+					return nil, err
+				}
+				value.Length = length
+			}
+			break
+		}
+		secretValues[secretId] = value
 	}
 
-	return secretConfigs, nil
-}
-
-func ParseSecretValue(secret string) (secretValue, error) {
-	values := strings.Split(secret, "#")
-	if len(values) == 0 {
-		return secretValue{}, fmt.Errorf("unable to parse %s", secret)
-	}
-
-	if len(values) == 1 {
-		return secretValue{Version: values[0], Length: 0}, nil
-	}
-
-	split := strings.Split(values[1], "=")
-	parsed := split[len(split)-1]
-	stripped := strings.ReplaceAll(parsed, " ", "")
-	length, err := strconv.Atoi(stripped)
-	if err != nil {
-		return secretValue{}, err
-	}
-	version := strings.ReplaceAll(values[0], " ", "")
-
-	logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
-
-	return secretValue{Version: version, Length: length}, nil
+	return secretValues, nil
 }
 
 // GenerateSecrets generates secrets locally and sends them to a remote server for storage.
-func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, appName, server string) (map[string]string, error) {
-	secrets := make(map[string]string)
-
+func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
+	secretsGenerated := map[string]string{}
 	var mutex sync.Mutex
 	var wg sync.WaitGroup
-	ch := make(chan error, len(secretsFromConfig))
-	for n, v := range secretsFromConfig {
+	ch := make(chan error, len(secrets))
+	for n, v := range secrets {
 		wg.Add(1)
 
-		go func(secretName, secretValue string) {
+		go func(secretName string, secret SecretValue) {
 			defer wg.Done()
 
-			parsedSecretValue, err := ParseSecretValue(secretValue)
-			if err != nil {
-				ch <- err
-				return
-			}
-
-			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version)
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
 			logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
 
-			if parsedSecretValue.Length > 0 {
-				passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length))
+			if secret.Length > 0 {
+				passwords, err := GeneratePasswords(1, uint(secret.Length))
 				if err != nil {
 					ch <- err
 					return
@@ -178,7 +166,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secrets[secretName] = passwords[0]
+				secretsGenerated[secretName] = passwords[0]
 			} else {
 				passphrases, err := GeneratePassphrases(1)
 				if err != nil {
@@ -198,7 +186,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 				mutex.Lock()
 				defer mutex.Unlock()
-				secrets[secretName] = passphrases[0]
+				secretsGenerated[secretName] = passphrases[0]
 			}
 			ch <- nil
 		}(n, v)
@@ -206,16 +194,16 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
 
 	wg.Wait()
 
-	for range secretsFromConfig {
+	for range secrets {
 		err := <-ch
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	logrus.Debugf("generated and stored %s on %s", secrets, server)
+	logrus.Debugf("generated and stored %v on %s", secrets, server)
 
-	return secrets, nil
+	return secretsGenerated, nil
 }
 
 type secretStatus struct {
@@ -257,14 +245,9 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
 		remoteSecretNames[cont.Spec.Annotations.Name] = true
 	}
 
-	for secretName, secretValue := range secretsConfig {
+	for secretName, val := range secretsConfig {
 		createdRemote := false
 
-		val, err := ParseSecretValue(secretValue)
-		if err != nil {
-			return secStats, err
-		}
-
 		secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
 		if _, ok := remoteSecretNames[secretRemoteName]; ok {
 			createdRemote = true

pkg/secret/secret_test.go

@@ -18,7 +18,7 @@ func TestReadSecretsConfig(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
+	sampleEnv, err := recipe.SampleEnv()
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/upstream/commandconn/connection.go

@@ -18,7 +18,7 @@ import (
 //
 // ssh://<user>@<host> URL requires Docker 18.09 or later on the remote host.
 func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
-	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=5"})
+	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=60"})
 }
 
 func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) {

pkg/upstream/stack/loader.go

@@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) {
 	opts.SkipValidation = false
 }
 
+// SkipInterpolation skip interpolating environment variables.
+func SkipInterpolation(opts *loader.Options) {
+	opts.SkipInterpolation = true
+}
+
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
-func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
+func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
 	if err != nil {
 		return nil, err
 	}
 
+	if options == nil {
+		options = []func(*loader.Options){DontSkipValidation}
+	}
+
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails, DontSkipValidation)
+	config, err := loader.Load(configDetails, options...)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
 			return nil, fmt.Errorf("compose file contains unsupported options: %s",

tests/integration/app_cp.bats

@@ -5,9 +5,11 @@ setup_file(){
   _common_setup
   _add_server
   _new_app
+  _deploy_app
 }
 
 teardown_file(){
+  _undeploy_app
   _rm_app
   _rm_server
 }
@@ -17,11 +19,29 @@ setup(){
   _common_setup
 }
 
-teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+_mkfile() {
+  run bash -c "echo $2 > $1"
+  assert_success
+}
+
+_mkfile_remote() {
+  run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\""
+  assert_success
+}
+
+_mkdir() {
+  run bash -c "mkdir -p $1"
+  assert_success
+}
+
+_rm() {
+  run rm -rf "$1"
+  assert_success
+}
+
+_rm_remote() {
+  run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1"
+  assert_success
 }
 
 @test "validate app argument" {
@@ -54,68 +74,120 @@ teardown(){
   assert_output --partial 'arguments must take $SERVICE:$PATH form'
 }
 
-@test "detect 'coming FROM' syntax" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug
-  assert_failure
-  assert_output --partial 'coming FROM the container'
-}
-
-@test "detect 'going TO' syntax" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug
-  assert_failure
-  assert_output --partial 'going TO the container'
-}
-
 @test "error if local file missing" {
-  run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere
+  run $ABRA app cp "$TEST_APP_DOMAIN" thisfileshouldnotexist.txt app:/somewhere
   assert_failure
-  assert_output --partial 'myfile.txt does not exist locally?'
+  assert_output --partial 'local stat thisfileshouldnotexist.txt: no such file or directory'
 }
 
 # bats test_tags=slow
 @test "error if service doesn't exist" {
-  _deploy_app
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
 
-  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
-  assert_success
-
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ --debug
   assert_failure
   assert_output --partial 'no containers matching'
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
-  assert_success
-
-  _undeploy_app
+  _rm "$BATS_TMPDIR/myfile.txt"
 }
 
 # bats test_tags=slow
-@test "copy to container" {
-  _deploy_app
-
-  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
-  assert_success
+@test "copy local file to container directory" {
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
 
   run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
   assert_success
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
   assert_success
+  assert_output --partial "foo"
 
-  _undeploy_app
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
-@test "copy from container" {
-  _deploy_app
+@test "copy local file to container file (and override on remote)" {
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
 
-  run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt"
+  # create
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
   assert_success
 
-  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  assert_success
+  assert_output --partial "foo"
+
+  _mkfile "$BATS_TMPDIR/myfile.txt" "bar"
+
+  # override
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt
   assert_success
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
+  assert_success
+  assert_output --partial "bar"
+
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy local file to container file (and rename)" {
+  _mkfile "$BATS_TMPDIR/myfile.txt" "foo"
+
+  # rename
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile2.txt
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile2.txt
+  assert_success
+  assert_output --partial "foo"
+
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile2.txt"
+}
+
+# bats test_tags=slow
+@test "copy local directory to container directory (and creates missing directory)" {
+  _mkdir "$BATS_TMPDIR/mydir"
+  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir" app:/etc
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/mydir
+  assert_success
+  assert_output --partial "myfile.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/mydir"
+}
+
+# bats test_tags=slow
+@test "copy local files to container directory" {
+  _mkdir "$BATS_TMPDIR/mydir"
+  _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo"
+  _mkfile "$BATS_TMPDIR/mydir/myfile2.txt" "foo"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir/" app:/etc
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile.txt
+  assert_success
+  assert_output --partial "myfile.txt"
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile2.txt
+  assert_success
+  assert_output --partial "myfile2.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/myfile*"
+}
+
+# bats test_tags=slow
+@test "copy container file to local directory" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
   assert_success
 
   run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR"
@@ -123,8 +195,76 @@ teardown(){
   assert_exists "$BATS_TMPDIR/myfile.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
 
-  run rm -rf "$BATS_TMPDIR/myfile.txt"
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy container file to local file" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
   assert_success
 
-  _undeploy_app
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/myfile.txt"
+  assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
+
+  _rm "$BATS_TMPDIR/myfile.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy container file to local file and rename" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  assert_success
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile2.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/myfile2.txt"
+  assert bash -c "cat $BATS_TMPDIR/myfile2.txt | grep -q foo"
+
+  _rm "$BATS_TMPDIR/myfile2.txt"
+  _rm_remote "/etc/myfile.txt"
+}
+
+# bats test_tags=slow
+@test "copy container directory to local directory" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
+  assert_success
+
+  mkdir "$BATS_TMPDIR/mydir"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc "$BATS_TMPDIR/mydir"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/etc/myfile.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/etc/myfile2.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/myfile.txt"
+  _rm_remote "/etc/myfile2.txt"
+}
+
+# bats test_tags=slow
+@test "copy container files to local directory" {
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt"
+  assert_success
+
+  run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt"
+  assert_success
+
+  mkdir "$BATS_TMPDIR/mydir"
+
+  run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/ "$BATS_TMPDIR/mydir"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/myfile.txt"
+  assert_success
+  assert_exists "$BATS_TMPDIR/mydir/myfile2.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+  _rm_remote "/etc/myfile.txt"
+  _rm_remote "/etc/myfile2.txt"
 }