secret: allow inserting secret from file and add trim flag

See coop-cloud/organising#601
See coop-cloud/organising#482

.drone.yml

@@ -29,7 +29,7 @@ steps:
       event: tag
 
   - name: release
-    image: goreleaser/goreleaser:v1.18.2
+    image: goreleaser/goreleaser:v1.24.0
     environment:
       GITEA_TOKEN:
         from_secret: goreleaser_gitea_token

.goreleaser.yml

@@ -29,6 +29,8 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
+      - "-s"
+      - "-w"
 
   - id: kadabra
     binary: kadabra
@@ -50,12 +52,8 @@ builds:
     ldflags:
       - "-X 'main.Commit={{ .Commit }}'"
       - "-X 'main.Version={{ .Version }}'"
-
+      - "-s"
-archives:
+      - "-w"
-  - replacements:
-      386: i386
-      amd64: x86_64
-    format: binary
 
 checksum:
   name_template: "checksums.txt"

AUTHORS.md

@@ -7,6 +7,7 @@
 - cassowary
 - codegod100
 - decentral1se
+- fauno
 - frando
 - kawaiipunk
 - knoflook

Dockerfile

@@ -1,23 +1,29 @@
+# Build image
 FROM golang:1.21-alpine AS build
 
 ENV GOPRIVATE coopcloud.tech
 
 RUN apk add --no-cache \
-  ca-certificates \
   gcc \
   git \
   make \
   musl-dev
 
-RUN update-ca-certificates
-
 COPY . /app
 
 WORKDIR /app
 
 RUN CGO_ENABLED=0 make build
 
-FROM scratch
+# Release image ("slim")
+FROM alpine:3.19.1
+
+RUN apk add --no-cache \
+  ca-certificates \
+  git \
+  openssh
+
+RUN update-ca-certificates
 
 COPY --from=build /app/abra /abra
 

Makefile

@@ -53,3 +53,6 @@ test:
 
 loc:
 	@find . -name "*.go" | xargs wc -l
+
+deps:
+	@go get -t -u ./...

cli/app/deploy.go

@@ -97,6 +97,19 @@ recipes.
 			logrus.Fatal(err)
 		}
 
+		// NOTE(d1): check out specific version before dealing with secrets. This
+		// is because we need to deal with GetComposeFiles under the hood and these
+		// files change from version to version which therefore affects which
+		// secrets might be generated
+		version := deployedVersion
+		if specificVersion != "" {
+			version = specificVersion
+			logrus.Debugf("choosing %s as version to deploy", version)
+			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
 		secStats, err := secret.PollSecretsStatus(cl, app)
 		if err != nil {
 			logrus.Fatal(err)
@@ -116,15 +129,6 @@ recipes.
 			}
 		}
 
-		version := deployedVersion
-		if specificVersion != "" {
-			version = specificVersion
-			logrus.Debugf("choosing %s as version to deploy", version)
-			if err := recipe.EnsureVersion(app.Recipe, version); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		if !internal.Chaos && specificVersion == "" {
 			catl, err := recipe.ReadRecipeCatalogue(internal.Offline)
 			if err != nil {

cli/app/errors.go

@@ -13,7 +13,7 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/recipe"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
@@ -97,7 +97,7 @@ func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error
 		filters := filters.NewArgs()
 		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
 
-		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
 		if err != nil {
 			return err
 		}

cli/app/logs.go

@@ -15,6 +15,7 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/swarm"
 	dockerClient "github.com/docker/docker/client"
@@ -110,7 +111,7 @@ func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) er
 		// collected in parallel.
 		wg.Add(1)
 		go func(serviceID string) {
-			logs, err := cl.ServiceLogs(context.Background(), serviceID, types.ContainerLogsOptions{
+			logs, err := cl.ServiceLogs(context.Background(), serviceID, containerTypes.LogsOptions{
 				ShowStderr: true,
 				ShowStdout: !internal.StdErrOnly,
 				Since:      internal.SinceLogs,

cli/app/new.go

@@ -10,7 +10,6 @@ import (
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/jsontable"
-	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/secret"
 	"github.com/AlecAivazis/survey/v2"
@@ -78,9 +77,29 @@ var appNewCommand = cli.Command{
 				}
 			}
 			if c.Args().Get(1) == "" {
-				if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+				var version string
+
+				recipeVersions, err := recipePkg.GetRecipeVersions(recipe.Name, internal.Offline)
+				if err != nil {
 					logrus.Fatal(err)
 				}
+
+				// NOTE(d1): determine whether recipe versions exist or not and check
+				// out the latest version or current HEAD
+				if len(recipeVersions) > 0 {
+					latest := recipeVersions[len(recipeVersions)-1]
+					for tag := range latest {
+						version = tag
+					}
+
+					if err := recipePkg.EnsureVersion(recipe.Name, version); err != nil {
+						logrus.Fatal(err)
+					}
+				} else {
+					if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
+						logrus.Fatal(err)
+					}
+				}
 			} else {
 				if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil {
 					logrus.Fatal(err)
@@ -183,6 +202,12 @@ type AppSecrets map[string]string
 
 // createSecrets creates all secrets for a new app.
 func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) {
+	// NOTE(d1): trim to match app.StackName() implementation
+	if len(sanitisedAppName) > config.MAX_SANITISED_APP_NAME_LENGTH {
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH])
+		sanitisedAppName = sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH]
+	}
+
 	secrets, err := secret.GenerateSecrets(cl, secretsConfig, internal.NewAppServer)
 	if err != nil {
 		return nil, err
@@ -206,7 +231,7 @@ func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secr
 }
 
 // ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
-func ensureDomainFlag(recipe recipe.Recipe, server string) error {
+func ensureDomainFlag(recipe recipePkg.Recipe, server string) error {
 	if internal.Domain == "" && !internal.NoInput {
 		prompt := &survey.Input{
 			Message: "Specify app domain",

cli/app/ps.go

@@ -2,7 +2,8 @@ package app
 
 import (
 	"context"
-	"strings"
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"coopcloud.tech/abra/cli/internal"
@@ -10,11 +11,13 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	"coopcloud.tech/abra/pkg/service"
+	"coopcloud.tech/abra/pkg/recipe"
+	abraService "coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/buger/goterm"
 	dockerFormatter "github.com/docker/cli/cli/command/formatter"
-	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -27,6 +30,7 @@ var appPsCommand = cli.Command{
 	ArgsUsage:   "<domain>",
 	Description: "Show a more detailed status output of a specific deployed app",
 	Flags: []cli.Flag{
+		internal.MachineReadableFlag,
 		internal.WatchFlag,
 		internal.DebugFlag,
 	},
@@ -40,7 +44,7 @@ var appPsCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		isDeployed, _, err := stack.IsDeployed(context.Background(), cl, app.StackName())
+		isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())
 		if err != nil {
 			logrus.Fatal(err)
 		}
@@ -49,6 +53,15 @@ var appPsCommand = cli.Command{
 			logrus.Fatalf("%s is not deployed?", app.Name)
 		}
 
+		statuses, err := config.GetAppStatuses([]config.App{app}, true)
+		if statusMeta, ok := statuses[app.StackName()]; ok {
+			if _, exists := statusMeta["chaos"]; !exists {
+				if err := recipe.EnsureVersion(app.Recipe, deployedVersion); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
 		if !internal.Watch {
 			showPSOutput(c, app, cl)
 			return nil
@@ -66,36 +79,77 @@ var appPsCommand = cli.Command{
 
 // showPSOutput renders ps output.
 func showPSOutput(c *cli.Context, app config.App, cl *dockerClient.Client) {
-	filters, err := app.Filters(true, true)
+	composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env)
 	if err != nil {
 		logrus.Fatal(err)
+		return
 	}
 
-	containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+	deployOpts := stack.Deploy{
+		Composefiles: composeFiles,
+		Namespace:    app.StackName(),
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+	compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
 	if err != nil {
 		logrus.Fatal(err)
+		return
 	}
 
-	tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
+	var tablerows [][]string
-	table := formatter.CreateTable(tableCol)
+	allContainerStats := make(map[string]map[string]string)
+	for _, service := range compose.Services {
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service.Name))
 
-	for _, container := range containers {
+		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
-		var containerNames []string
+		if err != nil {
-		for _, containerName := range container.Names {
+			logrus.Fatal(err)
-			trimmed := strings.TrimPrefix(containerName, "/")
+			return
-			containerNames = append(containerNames, trimmed)
 		}
 
-		tableRow := []string{
+		var containerStats map[string]string
-			service.ContainerToServiceName(container.Names, app.StackName()),
+
-			formatter.RemoveSha(container.Image),
+		if len(containers) == 0 {
-			formatter.HumanDuration(container.Created),
+			containerStats = map[string]string{
-			container.Status,
+				"service name": service.Name,
-			container.State,
+				"image":        "unknown",
-			dockerFormatter.DisplayablePorts(container.Ports),
+				"created":      "unknown",
+				"status":       "unknown",
+				"state":        "unknown",
+				"ports":        "unknown",
+			}
+		} else {
+			container := containers[0]
+			containerStats = map[string]string{
+				"service name": abraService.ContainerToServiceName(container.Names, app.StackName()),
+				"image":        formatter.RemoveSha(container.Image),
+				"created":      formatter.HumanDuration(container.Created),
+				"status":       container.Status,
+				"state":        container.State,
+				"ports":        dockerFormatter.DisplayablePorts(container.Ports),
+			}
 		}
-		table.Append(tableRow)
+		allContainerStats[containerStats["service name"]] = containerStats
+
+		var tablerow []string = []string{containerStats["service name"], containerStats["image"], containerStats["created"], containerStats["status"], containerStats["state"], containerStats["ports"]}
+		tablerows = append(tablerows, tablerow)
+	}
+	if internal.MachineReadable {
+		jsonstring, err := json.Marshal(allContainerStats)
+		if err != nil {
+			logrus.Fatal(err)
+		} else {
+			fmt.Println(string(jsonstring))
+		}
+		return
+	} else {
+		tableCol := []string{"service name", "image", "created", "status", "state", "ports"}
+		table := formatter.CreateTable(tableCol)
+		for _, row := range tablerows {
+			table.Append(row)
+		}
+		table.Render()
 	}
-
-	table.Render()
 }

cli/app/remove.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -13,7 +12,6 @@ import (
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/volume"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -112,28 +110,19 @@ flag.
 			logrus.Fatal(err)
 		}
 
-		volumeListOptions := volume.ListOptions{fs}
+		volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, fs)
-		volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions)
-		volumeList := volumeListOKBody.Volumes
 		if err != nil {
 			logrus.Fatal(err)
 		}
+		volumeNames := client.GetVolumeNames(volumeList)
 
-		var vols []string
+		if len(volumeNames) > 0 {
-		for _, vol := range volumeList {
+			err := client.RemoveVolumes(cl, context.Background(), volumeNames, internal.Force, 5)
-			vols = append(vols, vol.Name)
+			if err != nil {
-		}
+				log.Fatalf("removing volumes failed: %s", err)
-
-		if len(vols) > 0 {
-			for _, vol := range vols {
-				err = retryFunc(5, func() error {
-					return cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
-				})
-				if err != nil {
-					log.Fatalf("removing volumes failed: %s", err)
-				}
-				logrus.Info(fmt.Sprintf("volume %s removed", vol))
 			}
+
+			logrus.Infof("%d volumes removed successfully", len(volumeNames))
 		} else {
 			logrus.Info("no volumes to remove")
 		}
@@ -147,21 +136,3 @@ flag.
 		return nil
 	},
 }
-
-// retryFunc retries the given function for the given retries. After the nth
-// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
-// It returns an error if the function still failed after the last retry.
-func retryFunc(retries int, fn func() error) error {
-	for i := 0; i < retries; i++ {
-		err := fn()
-		if err == nil {
-			return nil
-		}
-		if i+1 < retries {
-			sleep := time.Duration(i+1) * time.Duration(i+1)
-			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
-			time.Sleep(sleep * time.Second)
-		}
-	}
-	return fmt.Errorf("%d retries failed", retries)
-}

cli/app/secret.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -156,6 +157,8 @@ var appSecretInsertCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.PassFlag,
+		internal.FileFlag,
+		internal.TrimFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	ArgsUsage:    "<domain> <secret-name> <version> <data>",
@@ -188,6 +191,18 @@ Example:
 		version := c.Args().Get(2)
 		data := c.Args().Get(3)
 
+		if internal.File {
+			raw, err := os.ReadFile(data)
+			if err != nil {
+				logrus.Fatalf("reading secret from file: %s", err)
+			}
+			data = string(raw)
+		}
+
+		if internal.Trim {
+			data = strings.TrimSpace(data)
+		}
+
 		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
 		if err := client.StoreSecret(cl, secretName, data, app.Server); err != nil {
 			logrus.Fatal(err)

cli/app/services.go

@@ -11,7 +11,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/service"
 	stack "coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -48,7 +48,7 @@ var appServicesCommand = cli.Command{
 			logrus.Fatal(err)
 		}
 
-		containers, err := cl.ContainerList(context.Background(), types.ContainerListOptions{Filters: filters})
+		containers, err := cl.ContainerList(context.Background(), containerTypes.ListOptions{Filters: filters})
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/upgrade.go

@@ -31,6 +31,7 @@ var appUpgradeCommand = cli.Command{
 		internal.NoDomainChecksFlag,
 		internal.DontWaitConvergeFlag,
 		internal.OfflineFlag,
+		internal.ReleaseNotesFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -193,23 +194,24 @@ recipes.
 		// check out the tag and then they'll appear to be missing. this covers
 		// when we obviously will forget to write release notes before publishing
 		var releaseNotes string
-		for _, version := range versions {
+		if chosenUpgrade != "" {
-			parsedVersion, err := tagcmp.Parse(version)
-			if err != nil {
-				logrus.Fatal(err)
-			}
 			parsedChosenUpgrade, err := tagcmp.Parse(chosenUpgrade)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-
+			for _, version := range versions {
-			if !(parsedVersion.Equals(parsedDeployedVersion)) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
+				parsedVersion, err := tagcmp.Parse(version)
-				note, err := internal.GetReleaseNotes(app.Recipe, version)
 				if err != nil {
-					return err
+					logrus.Fatal(err)
 				}
-				if note != "" {
+				if parsedVersion.IsGreaterThan(parsedDeployedVersion) && parsedVersion.IsLessThan(parsedChosenUpgrade) {
-					releaseNotes += fmt.Sprintf("%s\n", note)
+					note, err := internal.GetReleaseNotes(app.Recipe, version)
+					if err != nil {
+						return err
+					}
+					if note != "" {
+						releaseNotes += fmt.Sprintf("%s\n", note)
+					}
 				}
 			}
 		}
@@ -269,6 +271,12 @@ recipes.
 			}
 		}
 
+		if internal.ReleaseNotes {
+			fmt.Println()
+			fmt.Print(releaseNotes)
+			return nil
+		}
+
 		if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil {
 			logrus.Fatal(err)
 		}

cli/app/version.go

@@ -10,7 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
-	"github.com/docker/distribution/reference"
+	"github.com/distribution/reference"
 	"github.com/olekukonko/tablewriter"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"

cli/app/volume.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"log"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -131,12 +132,12 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
 		}
 
 		if len(volumesToRemove) > 0 {
-			err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
+			err := client.RemoveVolumes(cl, context.Background(), volumesToRemove, internal.Force, 5)
 			if err != nil {
-				logrus.Fatal(err)
+				log.Fatalf("removing volumes failed: %s", err)
 			}
 
-			logrus.Info("volumes removed successfully")
+			logrus.Infof("%d volumes removed successfully", len(volumesToRemove))
 		} else {
 			logrus.Info("no volumes removed")
 		}

cli/internal/cli.go

@@ -38,6 +38,20 @@ var PassRemoveFlag = &cli.BoolFlag{
 	Destination: &PassRemove,
 }
 
+var File bool
+var FileFlag = &cli.BoolFlag{
+	Name:        "file, f",
+	Usage:       "Treat input as a file",
+	Destination: &File,
+}
+
+var Trim bool
+var TrimFlag = &cli.BoolFlag{
+	Name:        "trim, t",
+	Usage:       "Trim input",
+	Destination: &Trim,
+}
+
 // Force force functionality without asking.
 var Force bool
 
@@ -95,6 +109,16 @@ var OfflineFlag = &cli.BoolFlag{
 	Usage:       "Prefer offline & filesystem access when possible",
 }
 
+// ReleaseNotes stores the variable from ReleaseNotesFlag.
+var ReleaseNotes bool
+
+// ReleaseNotesFlag turns on/off printing only release notes when upgrading.
+var ReleaseNotesFlag = &cli.BoolFlag{
+	Name:        "releasenotes, r",
+	Destination: &ReleaseNotes,
+	Usage:       "Only show release notes",
+}
+
 // MachineReadable stores the variable from MachineReadableFlag
 var MachineReadable bool
 

cli/internal/recipe.go

@@ -6,7 +6,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/distribution/reference"
+	"github.com/distribution/reference"
 	"github.com/sirupsen/logrus"
 )
 

cli/recipe/release.go

@@ -17,7 +17,7 @@ import (
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/distribution/reference"
+	"github.com/distribution/reference"
 	"github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"

cli/recipe/upgrade.go

@@ -18,7 +18,7 @@ import (
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/docker/distribution/reference"
+	"github.com/distribution/reference"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )

cli/recipe/version.go

@@ -54,8 +54,9 @@ var recipeVersionCommand = cli.Command{
 			logrus.Fatalf("%s has no catalogue published versions?", recipe.Name)
 		}
 
+		tableCols := []string{"version", "service", "image", "tag"}
+		aggregated_table := formatter.CreateTable(tableCols)
 		for i := len(recipeMeta.Versions) - 1; i >= 0; i-- {
-			tableCols := []string{"version", "service", "image", "tag"}
 			table := formatter.CreateTable(tableCols)
 			for version, meta := range recipeMeta.Versions[i] {
 				var versions [][]string
@@ -67,11 +68,10 @@ var recipeVersionCommand = cli.Command{
 
 				for _, version := range versions {
 					table.Append(version)
+					aggregated_table.Append(version)
 				}
 
-				if internal.MachineReadable {
+				if !internal.MachineReadable {
-					table.JSONRender()
-				} else {
 					table.SetAutoMergeCellsByColumnIndex([]int{0})
 					table.SetAlignment(tablewriter.ALIGN_LEFT)
 					table.Render()
@@ -79,6 +79,9 @@ var recipeVersionCommand = cli.Command{
 				}
 			}
 		}
+		if internal.MachineReadable {
+			aggregated_table.JSONRender()
+		}
 
 		return nil
 	},

cli/server/add.go

@@ -53,7 +53,7 @@ func cleanUp(domainName string) {
 // Docker manages SSH connection details. These are stored to disk in
 // ~/.docker. Abra can manage this completely for the user, so it's an
 // implementation detail.
-func newContext(c *cli.Context, domainName, username, port string) error {
+func newContext(c *cli.Context, domainName string) error {
 	store := contextPkg.NewDefaultDockerContextStore()
 	contexts, err := store.Store.List()
 	if err != nil {
@@ -67,9 +67,9 @@ func newContext(c *cli.Context, domainName, username, port string) error {
 		}
 	}
 
-	logrus.Debugf("creating context with domain %s, username %s and port %s", domainName, username, port)
+	logrus.Debugf("creating context with domain %s", domainName)
 
-	if err := client.CreateContext(domainName, username, port); err != nil {
+	if err := client.CreateContext(domainName); err != nil {
 		return err
 	}
 
@@ -158,12 +158,7 @@ developer machine.
 			logrus.Fatal(err)
 		}
 
-		hostConfig, err := sshPkg.GetHostConfig(domainName)
+		if err := newContext(c, domainName); err != nil {
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
 			logrus.Fatal(err)
 		}
 

go.mod

@@ -3,118 +3,131 @@ module coopcloud.tech/abra
 go 1.21
 
 require (
-	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
+	coopcloud.tech/tagcmp v0.0.0-20230809071031-eb3e7758d4eb
 	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v24.0.7+incompatible
+	github.com/distribution/distribution v2.8.3+incompatible
-	github.com/docker/distribution v2.8.3+incompatible
+	github.com/docker/cli v26.1.4+incompatible
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v26.1.4+incompatible
 	github.com/docker/go-units v0.5.0
-	github.com/go-git/go-git/v5 v5.10.0
+	github.com/go-git/go-git/v5 v5.12.0
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/moby/sys/signal v0.7.0
 	github.com/moby/term v0.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.14.1
+	github.com/schollz/progressbar/v3 v3.14.4
 	github.com/sirupsen/logrus v1.9.3
 	gotest.tools/v3 v3.5.1
 )
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
-	github.com/BurntSushi/toml v1.0.0 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/Microsoft/hcsshim v0.9.2 // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
-	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/cloudflare/circl v1.3.9 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/distribution/reference v0.5.0 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
-	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/miekg/pkcs11 v1.1.1 // indirect
-	github.com/miekg/pkcs11 v1.0.3 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.1.0 // indirect
+	github.com/opencontainers/runc v1.1.13 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/common v0.54.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/skeema/knownhosts v1.2.0 // indirect
+	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	go.opentelemetry.io/otel v1.27.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
-	golang.org/x/term v0.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
-	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	go.opentelemetry.io/otel/sdk v1.27.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	go.opentelemetry.io/otel/trace v1.27.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/term v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
+	google.golang.org/grpc v1.64.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
-	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/buger/goterm v1.0.4
-	github.com/containerd/containerd v1.5.9 // indirect
+	github.com/containerd/containerd v1.7.18 // indirect
 	github.com/containers/image v3.0.2+incompatible
 	github.com/containers/storage v1.38.2 // indirect
 	github.com/decentral1se/passgen v1.0.1
-	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/docker/docker-credential-helpers v0.8.2 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/fvbommel/sortorder v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.5
+	github.com/hashicorp/go-retryablehttp v0.7.7
-	github.com/klauspost/pgzip v1.2.6
+	github.com/moby/patternmatcher v0.6.0 // indirect
-	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_golang v1.19.1 // indirect
-	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
-	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/theupdateframework/notary v0.7.0 // indirect
-	github.com/urfave/cli v1.22.9
+	github.com/urfave/cli v1.22.15
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
-	golang.org/x/sys v0.14.0
+	golang.org/x/sys v0.21.0
 )

pkg/client/context.go

@@ -14,19 +14,16 @@ import (
 
 type Context = contextStore.Metadata
 
-func CreateContext(contextName string, user string, port string) error {
+// CreateContext creates a new Docker context.
-	host := contextName
+func CreateContext(contextName string) error {
-	if user != "" {
+	host := fmt.Sprintf("ssh://%s", contextName)
-		host = fmt.Sprintf("%s@%s", user, host)
+
-	}
-	if port != "" {
-		host = fmt.Sprintf("%s:%s", host, port)
-	}
-	host = fmt.Sprintf("ssh://%s", host)
 	if err := createContext(contextName, host); err != nil {
 		return err
 	}
+
 	logrus.Debugf("created the %s context", contextName)
+
 	return nil
 }
 

pkg/client/registry.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/containers/image/docker"
 	"github.com/containers/image/types"
-	"github.com/docker/distribution/reference"
+	"github.com/distribution/reference"
 )
 
 // GetRegistryTags retrieves all tags of an image from a container registry.

pkg/client/volumes.go

@@ -2,15 +2,17 @@ package client
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
 )
 
 func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) {
-	volumeListOptions := volume.ListOptions{fs}
+	volumeListOKBody, err := cl.VolumeList(ctx, volume.ListOptions{Filters: fs})
-	volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions)
 	volumeList := volumeListOKBody.Volumes
 	if err != nil {
 		return volumeList, err
@@ -29,13 +31,32 @@ func GetVolumeNames(volumes []*volume.Volume) []string {
 	return volumeNames
 }
 
-func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
+func RemoveVolumes(cl *client.Client, ctx context.Context, volumeNames []string, force bool, retries int) error {
 	for _, volName := range volumeNames {
-		err := cl.VolumeRemove(ctx, volName, force)
+		err := retryFunc(5, func() error {
+			return cl.VolumeRemove(context.Background(), volName, force)
+		})
 		if err != nil {
-			return err
+			return fmt.Errorf("volume %s: %s", volName, err)
 		}
 	}
-
 	return nil
 }
+
+// retryFunc retries the given function for the given retries. After the nth
+// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
+// It returns an error if the function still failed after the last retry.
+func retryFunc(retries int, fn func() error) error {
+	for i := 0; i < retries; i++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		if i+1 < retries {
+			sleep := time.Duration(i+1) * time.Duration(i+1)
+			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
+			time.Sleep(sleep * time.Second)
+		}
+	}
+	return fmt.Errorf("%d retries failed", retries)
+}

pkg/client/volumes_test.go

@@ -1,4 +1,4 @@
-package app
+package client
 
 import (
 	"fmt"

pkg/compose/compose.go

@@ -11,8 +11,8 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
+	"github.com/distribution/reference"
 	composetypes "github.com/docker/cli/cli/compose/types"
-	"github.com/docker/distribution/reference"
 	"github.com/sirupsen/logrus"
 )
 

pkg/config/app.go

@@ -69,9 +69,9 @@ func (a App) StackName() string {
 func StackName(appName string) string {
 	stackName := SanitiseAppName(appName)
 
-	if len(stackName) > 45 {
+	if len(stackName) > MAX_SANITISED_APP_NAME_LENGTH {
-		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:45])
+		logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:MAX_SANITISED_APP_NAME_LENGTH])
-		stackName = stackName[:45]
+		stackName = stackName[:MAX_SANITISED_APP_NAME_LENGTH]
 	}
 
 	return stackName

pkg/config/env.go

@@ -36,6 +36,9 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
 var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
 
+const MAX_SANITISED_APP_NAME_LENGTH = 45
+const MAX_DOCKER_SECRET_LENGTH = 64
+
 var BackupbotLabel = "coop-cloud.backupbot.enabled"
 
 // envVarModifiers is a list of env var modifier strings. These are added to

pkg/container/container.go

@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/pkg/formatter"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
+	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
@@ -17,7 +18,7 @@ import (
 // count of containers does not match 1, then a prompt is presented to let the
 // user choose. A count of 0 is handled gracefully.
 func GetContainer(c context.Context, cl *client.Client, filters filters.Args, noInput bool) (types.Container, error) {
-	containerOpts := types.ContainerListOptions{Filters: filters}
+	containerOpts := containerTypes.ListOptions{Filters: filters}
 	containers, err := cl.ContainerList(c, containerOpts)
 	if err != nil {
 		return types.Container{}, err

pkg/lint/recipe.go

@@ -10,7 +10,7 @@ import (
 	"coopcloud.tech/abra/pkg/recipe"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
-	"github.com/docker/distribution/reference"
+	"github.com/distribution/reference"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
@@ -115,6 +115,13 @@ var LintRules = map[string][]LintRule{
 			HowToResolve: "upload your recipe to git.coopcloud.tech/coop-cloud/...",
 			Function:     LintHasRecipeRepo,
 		},
+		{
+			Ref:          "R015",
+			Level:        "warn",
+			Description:  "long secret names",
+			HowToResolve: "reduce length of secret names to 12 chars",
+			Function:     LintSecretLengths,
+		},
 	},
 	"error": {
 		{
@@ -401,6 +408,16 @@ func LintHasRecipeRepo(recipe recipe.Recipe) (bool, error) {
 	return true, nil
 }
 
+func LintSecretLengths(recipe recipe.Recipe) (bool, error) {
+	for name := range recipe.Config.Secrets {
+		if len(name) > 12 {
+			return false, fmt.Errorf("secret %s is longer than 12 characters", name)
+		}
+	}
+
+	return true, nil
+}
+
 func LintValidTags(recipe recipe.Recipe) (bool, error) {
 	recipeDir := path.Join(config.RECIPES_DIR, recipe.Name)
 

pkg/recipe/recipe.go

@@ -22,8 +22,8 @@ import (
 	loader "coopcloud.tech/abra/pkg/upstream/stack"
 	"coopcloud.tech/abra/pkg/web"
 	"coopcloud.tech/tagcmp"
+	"github.com/distribution/reference"
 	composetypes "github.com/docker/cli/cli/compose/types"
-	"github.com/docker/distribution/reference"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
@@ -945,6 +945,7 @@ func GetRecipeVersions(recipeName string, offline bool) (RecipeVersions, error)
 	if err != nil {
 		return versions, err
 	}
+
 	sortRecipeVersions(versions)
 
 	logrus.Debugf("collected %s for %s", versions, recipeName)

pkg/secret/secret.go

@@ -89,7 +89,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 	appEnv["STACK_NAME"] = stackName
 
 	opts := stack.Deploy{Composefiles: composeFiles}
-	config, err := loader.LoadComposefile(opts, appEnv)
+	composeConfig, err := loader.LoadComposefile(opts, appEnv)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +100,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 	}
 
 	var enabledSecrets []string
-	for _, service := range config.Services {
+	for _, service := range composeConfig.Services {
 		for _, secret := range service.Secrets {
 			enabledSecrets = append(enabledSecrets, secret.Source)
 		}
@@ -112,7 +112,7 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 	}
 
 	secretValues := map[string]Secret{}
-	for secretId, secretConfig := range config.Secrets {
+	for secretId, secretConfig := range composeConfig.Secrets {
 		if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
 			return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
 		}
@@ -126,6 +126,10 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName strin
 		secretVersion := secretConfig.Name[lastIdx+1:]
 		value := Secret{Version: secretVersion, RemoteName: secretConfig.Name}
 
+		if len(value.RemoteName) > config.MAX_DOCKER_SECRET_LENGTH {
+			return nil, fmt.Errorf("secret %s is > %d chars when combined with %s", secretId, config.MAX_DOCKER_SECRET_LENGTH, stackName)
+		}
+
 		// Check if the length modifier is set for this secret.
 		for envName, modifierValues := range appModifiers {
 			// configWithoutEnv contains the raw name as defined in the compose.yaml

pkg/secret/secret_test.go

@@ -28,3 +28,12 @@ func TestReadSecretsConfig(t *testing.T) {
 	assert.Equal(t, "v2", secretsFromConfig["test_pass_three"].Version)
 	assert.Equal(t, 0, secretsFromConfig["test_pass_three"].Length)
 }
+
+func TestReadSecretsConfigWithLongDomain(t *testing.T) {
+	composeFiles := []string{"./testdir/compose.yaml"}
+	_, err := ReadSecretsConfig("./testdir/.env.sample", composeFiles, "should_break_on_forty_eight_char_stack_nameeeeee")
+	if err == nil {
+		t.Fatal("expected failure, stack name is too long")
+	}
+	assert.Contains(t, err.Error(), "is > 64 chars")
+}

pkg/ssh/ssh.go

@@ -2,73 +2,14 @@ package ssh
 
 import (
 	"fmt"
-	"os/exec"
 	"strings"
-
-	"github.com/sirupsen/logrus"
 )
 
-// HostConfig is a SSH host config.
-type HostConfig struct {
-	Host         string
-	IdentityFile string
-	Port         string
-	User         string
-}
-
-// String presents a human friendly output for the HostConfig.
-func (h HostConfig) String() string {
-	return fmt.Sprintf(
-		"{host: %s, username: %s, port: %s, identityfile: %s}",
-		h.Host,
-		h.User,
-		h.Port,
-		h.IdentityFile,
-	)
-}
-
-// GetHostConfig retrieves a ~/.ssh/config config for a host using /usr/bin/ssh
-// directly. We therefore maintain consistent interop with this standard
-// tooling. This is useful because SSH confuses a lot of people and having to
-// learn how two tools (`ssh` and `abra`) handle SSH connection details instead
-// of one (just `ssh`) is Not Cool. Here's to less bug reports on this topic!
-func GetHostConfig(hostname string) (HostConfig, error) {
-	var hostConfig HostConfig
-
-	out, err := exec.Command("ssh", "-G", hostname).Output()
-	if err != nil {
-		return hostConfig, err
-	}
-
-	for _, line := range strings.Split(string(out), "\n") {
-		entries := strings.Split(line, " ")
-		for idx, entry := range entries {
-			if entry == "hostname" {
-				hostConfig.Host = entries[idx+1]
-			}
-			if entry == "user" {
-				hostConfig.User = entries[idx+1]
-			}
-			if entry == "port" {
-				hostConfig.Port = entries[idx+1]
-			}
-			if entry == "identityfile" {
-				if hostConfig.IdentityFile == "" {
-					hostConfig.IdentityFile = entries[idx+1]
-				}
-			}
-		}
-	}
-
-	logrus.Debugf("retrieved ssh config for %s: %s", hostname, hostConfig.String())
-
-	return hostConfig, nil
-}
-
 // Fatal is a error output wrapper which aims to make SSH failures easier to
 // parse through re-wording.
 func Fatal(hostname string, err error) error {
 	out := err.Error()
+
 	if strings.Contains(out, "Host key verification failed.") {
 		return fmt.Errorf("SSH host key verification failed for %s", hostname)
 	} else if strings.Contains(out, "Could not resolve hostname") {
@@ -79,7 +20,7 @@ func Fatal(hostname string, err error) error {
 		return fmt.Errorf("ssh auth: permission denied for %s", hostname)
 	} else if strings.Contains(out, "Network is unreachable") {
 		return fmt.Errorf("unable to connect to %s, network is unreachable?", hostname)
-	} else {
-		return err
 	}
+
+	return err
 }

pkg/upstream/commandconn/connection.go

@@ -16,12 +16,12 @@ import (
 // GetConnectionHelper returns Docker-specific connection helper for the given URL.
 // GetConnectionHelper returns nil without error when no helper is registered for the scheme.
 //
-// ssh://<user>@<host> URL requires Docker 18.09 or later on the remote host.
+// ssh://<host> URL requires Docker 18.09 or later on the remote host.
 func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
-	return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=60"})
+	return getConnectionHelper(daemonURL)
 }
 
-func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) {
+func getConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
 	url, err := url.Parse(daemonURL)
 	if err != nil {
 		return nil, err
@@ -35,7 +35,7 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)
+				return New(ctx, "ssh", ctxConnDetails.Args("docker", "system", "dial-stdio")...)
 			},
 			Host: "http://docker.example.com",
 		}, nil
@@ -45,6 +45,7 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 	return nil, err
 }
 
+// NewConnectionHelper creates new connection helper for a remote docker daemon.
 func NewConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) {
 	helper, err := GetConnectionHelper(daemonURL)
 	if err != nil {
@@ -73,6 +74,7 @@ func getDockerEndpoint(host string) (docker.Endpoint, error) {
 	return ep, nil
 }
 
+// GetDockerEndpointMetadataAndTLS retrieves the docker endpoint and TLS info for a remote host.
 func GetDockerEndpointMetadataAndTLS(host string) (docker.EndpointMeta, *dCliContextStore.EndpointTLSData, error) {
 	ep, err := getDockerEndpoint(host)
 	if err != nil {

pkg/upstream/container/tty.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
 	apiclient "github.com/docker/docker/client"
 	"github.com/moby/sys/signal"
@@ -22,7 +22,7 @@ func resizeTtyTo(ctx context.Context, client client.ContainerAPIClient, id strin
 		return nil
 	}
 
-	options := types.ResizeOptions{
+	options := container.ResizeOptions{
 		Height: height,
 		Width:  width,
 	}

pkg/upstream/stack/stack.go

@@ -233,7 +233,7 @@ func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAP
 		network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
 		switch {
 		case dockerClient.IsErrNotFound(err):
-			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
+			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed, which you can do by running this on the server: docker network create -d overlay proxy", networkName)
 		case err != nil:
 			return err
 		case network.Scope != "swarm":

scripts/installer/installer

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.8.1-beta"
+ABRA_VERSION="0.9.0-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
 RC_VERSION="0.8.0-rc1-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
@@ -45,7 +45,9 @@ function install_abra_release {
   fi
 
   ARCH=$(uname -m)
-  if [[ $ARCH =~ "aarch64" ]]; then
+  if [[ $ARCH =~ "x86_64" ]]; then
+      ARCH="amd64"
+  elif [[ $ARCH =~ "aarch64" ]]; then
       ARCH="arm64"
   elif [[ $ARCH =~ "armv5l" ]]; then
       ARCH="armv5"
@@ -55,7 +57,7 @@ function install_abra_release {
       ARCH="armv7"
   fi
   PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')_$ARCH
-  FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM""
+  FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM".tar.gz"
   sed_command_rel='s/.*"assets":\[\{[^]]*"name":"'$FILENAME'"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
   sed_command_checksums='s/.*"assets":\[\{[^\]*"name":"checksums.txt"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p'
 
@@ -65,7 +67,7 @@ function install_abra_release {
 
   checksums=$(wget -q -O- $checksums_url)
   checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p')
-  abra_download="/tmp/abra-download"
+  abra_download="/tmp/abra-download.tar.gz"
 
   echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..."
 
@@ -77,7 +79,10 @@ function install_abra_release {
       exit 1
   fi
   echo "$(tput setaf 2)check successful!$(tput sgr0)"
-  mv "$abra_download" "$HOME/.local/bin/abra"
+  cd /tmp/
+  tar xf abra-download.tar.gz
+  mv abra "$HOME/.local/bin/abra"
+  tar tf abra-download.tar.gz | xargs rm -f
   chmod +x "$HOME/.local/bin/abra"
 
   x=$(echo $PATH | grep $HOME/.local/bin)

tests/integration/app_deploy.bats

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 setup_file(){
+  load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/common"
   _common_setup
   _add_server
@@ -362,6 +363,7 @@ teardown(){
   _reset_app
 }
 
+# bats test_tags=slow
 @test "recipe config comments not present in values" {
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
   assert_success
@@ -370,3 +372,36 @@ teardown(){
   assert_success
   refute_output --partial 'should be removed'
 }
+
+# bats test_tags=slow
+@test "deploy specific version with incompatible HEAD" {
+  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
+  assert_success
+  assert_output --partial 'extra_pass'
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/compose.extra_secret.yml"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/compose.extra_secret.yml"
+
+  _git_commit
+
+  # NOTE(d1): 0.1.1+1.20.2 is a previous version which includes compose.extra_secret.yml
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
+  assert_success
+  refute_output --partial 'no such file or directory'
+
+  _undeploy_app
+  _reset_app
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  assert_success
+
+  _reset_recipe
+}

tests/integration/app_new.bats

@@ -13,6 +13,7 @@ teardown_file(){
 
 setup(){
   load "$PWD/tests/integration/helpers/common"
+  load "$PWD/tests/integration/helpers/git"
   _common_setup
   _fetch_recipe
 }
@@ -26,14 +27,6 @@ teardown(){
   run $ABRA app new --generate-bash-completion
   assert_success
   assert_output --partial "traefik"
-  assert_output --partial "abra-test-recipe"
-
-  # Note: this test needs to be updated when a new version of the test recipe is published.
-  run $ABRA app new abra-test-recipe --generate-bash-completion
-  assert_success
-  assert_output "0.1.0+1.20.0
-0.1.1+1.20.2
-0.2.0+1.21.0"
 }
 
 @test "create new app" {
@@ -44,8 +37,9 @@ teardown(){
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  _get_head_hash
-  assert_output --partial "up to date"
+  _get_current_hash
+  assert_equal "$headHash" "$currentHash"
 }
 
 @test "create new app with version" {
@@ -56,8 +50,9 @@ teardown(){
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" log -1
+  _get_tag_hash 0.1.1+1.20.2
-  assert_output --partial "453db7121c0a56a7a8f15378f18fe3bf21ccfdef"
+  _get_current_hash
+  assert_equal "$tagHash" "$currentHash"
 }
 
 @test "does not overwrite existing env files" {
@@ -117,11 +112,14 @@ teardown(){
 }
 
 @test "ensure recipe up to date if no --offline" {
+  _reset_recipe
+  _get_n_hash 3
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  _get_current_hash
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal "$currentHash" "$nHash"
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -130,18 +128,22 @@ teardown(){
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  _get_head_hash
-  assert_output --partial "up to date"
+  _get_current_hash
+  assert_equal "$HEAD_HASH" "$CURRENT_HASH"
 
   _reset_recipe
 }
 
 @test "ensure recipe not up to date if --offline" {
+  _reset_recipe
+  _get_n_hash 3
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  _get_current_hash
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal "$currentHash" "$nHash"
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app new "$TEST_RECIPE" \
@@ -153,12 +155,13 @@ teardown(){
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  _get_current_hash
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal "$currentHash" "$nHash"
 
   _reset_recipe
 }
 
+# bats test_tags=slow
 @test "generate secrets" {
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \

tests/integration/app_remove.bats

@@ -104,9 +104,6 @@ teardown(){
 
   _undeploy_app
 
-  # TODO: should wait as long as volume is no longer in use
-  sleep 10
-
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --no-input
   assert_success
 

tests/integration/app_secret.bats

@@ -19,6 +19,13 @@ teardown_file(){
   _reset_recipe
 }
 
+teardown() {
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  _reset_app
+  _reset_recipe
+  _checkout_recipe
+}
+
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
@@ -77,9 +84,6 @@ setup(){
   assert_output --partial 'test_pass_one'
   assert_output --partial 'test_pass_two'
   refute_output --partial 'extra_pass'
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
 }
 
 @test "generate: broken if missing version" {
@@ -91,7 +95,6 @@ setup(){
   assert_failure
   assert_output --partial 'missing version'
 
-  _reset_app
 }
 
 @test "generate: use version from app env" {
@@ -108,11 +111,6 @@ setup(){
   assert_success
   assert_output --partial 'v2'
   refute_output --partial 'v1'
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
-
-  _reset_app
 }
 
 @test "generate: generate extra secret based on COMPOSE_FILE" {
@@ -131,11 +129,6 @@ setup(){
   run docker -c "$TEST_SERVER" secret ls
   assert_success
   assert_output --partial "$TEST_APP_DOMAIN_extra_pass_v1"
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
-
-  _reset_app
 }
 
 @test "generate: bail if unstaged changes and no --chaos" {
@@ -162,8 +155,6 @@ setup(){
 
   run $ABRA app secret rm "$TEST_APP_DOMAIN" --all --chaos
   assert_success
-
-  _checkout_recipe
 }
 
 @test "generate: ensure secret name uses trimmed stack name" {
@@ -228,9 +219,22 @@ setup(){
   run $ABRA app secret ls "$TEST_APP_DOMAIN"
   assert_success
   assert_output --partial 'true'
+}
 
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" test_pass_one
+@test "insert: create secret from file" {
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
   assert_success
+  assert_output --partial 'false'
+
+  run bash -c "echo bar >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
+
+  run $ABRA app secret insert --file "$TEST_APP_DOMAIN" test_pass_one v1 "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_success
+  assert_output --partial 'successfully stored on server'
+
+  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  assert_success
+  assert_output --partial 'true'
 }
 
 @test "rm: validate arguments" {
@@ -314,9 +318,6 @@ setup(){
   run $ABRA app secret ls "$TEST_APP_DOMAIN"
   assert_success
   assert_output --partial 'true'
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
 }
 
 @test "ls: show secrets as machine readable" {
@@ -330,9 +331,6 @@ setup(){
   run $ABRA app secret ls "$TEST_APP_DOMAIN" --machine
   assert_success
   assert_output --partial '"created-on-server":"true"'
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
 }
 
 @test "ls: bail if unstaged changes and no --chaos" {

tests/integration/app_volume.bats

@@ -78,9 +78,6 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 10
-
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'
@@ -92,9 +89,6 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 10
-
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_success
   assert_output --partial 'volumes removed successfully'

tests/integration/helpers/git.bash

@@ -32,6 +32,35 @@ _reset_tags() {
 _set_git_author() {
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.email test@example.com
   assert_success
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.name test
   assert_success
 }
+
+_git_commit() {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add .
+  assert_success
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "test: helpers/git.bash: _git_commit"
+  assert_success
+}
+
+_get_tag_hash() {
+  tagHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-list -n 1 "$1")
+  assert_success
+}
+
+_get_head_hash() {
+  headHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" HEAD)
+  assert_success
+}
+
+_get_current_hash() {
+  currentHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H")
+  assert_success
+}
+
+_get_n_hash() {
+  nHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" "HEAD~$1")
+  assert_success
+}