NOT WORKING: WIP on adding server-wide logs functionality

Closes coop-cloud/organising#266.

README.md

@@ -9,6 +9,20 @@ The Co-op Cloud utility belt 🎩🐇
 
 `abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create apps, deploy them, run backup and restore operations and a whole lot of other things. Please see [docs.coopcloud.tech](https://docs.coopcloud.tech) for more extensive documentation.
 
+## Quick install
+
+```bash
+curl https://install.abra.autonomic.zone | bash
+```
+
+Or using the latest release candidate (extra experimental!):
+
+```bash
+curl https://install.abra.autonomic.zone | bash -s -- --rc
+```
+
+Source for this script is in [scripts/installer/installer](./scripts/installer/installer).
+
 ## Hacking
 
 ### Getting started

cli/app/cp.go

@@ -5,13 +5,8 @@ import (
 	"os"
 	"strings"
 
-	"coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	"github.com/docker/docker/pkg/archive"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -21,6 +16,18 @@ var appCpCommand = &cli.Command{
 	Aliases:   []string{"c"},
 	ArgsUsage: "<src> <dst>",
 	Usage:     "Copy files to/from a running app service",
+	Description: `
+This command supports copying files to and from any app service file system.
+
+If you want to copy a myfile.txt to the root of the app service:
+
+    abra app cp <app> myfile.txt app:/
+
+And if you want to copy that file back to your current working directory locally:
+
+		abra app cp <app> app:/myfile.txt .
+
+`,
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
@@ -64,61 +71,28 @@ var appCpCommand = &cli.Command{
 			logrus.Debugf("assuming transfer is going TO the container")
 		}
 
-		appFiles, err := config.LoadAppFiles("")
+		if !isToContainer {
+			if _, err := os.Stat(dstPath); os.IsNotExist(err) {
+				logrus.Fatalf("%s does not exist locally?", dstPath)
+			}
+		}
+		err := internal.ConfigureAndCp(c, app, srcPath, dstPath, service, isToContainer)
 		if err != nil {
 			logrus.Fatal(err)
 		}
-
-		appEnv, err := config.GetApp(appFiles, app.Name)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		filters := filters.NewArgs()
-		filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
-		containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if len(containers) != 1 {
-			logrus.Fatalf("expected 1 container but got %v", len(containers))
-		}
-		container := containers[0]
-
-		logrus.Debugf("retrieved '%s' as target container on '%s'", formatter.ShortenID(container.ID), app.Server)
-
-		if isToContainer {
-			if _, err := os.Stat(srcPath); err != nil {
-				logrus.Fatalf("'%s' does not exist?", srcPath)
-			}
-
-			toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-			content, err := archive.TarWithOptions(srcPath, toTarOpts)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
-			if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
-				logrus.Fatal(err)
-			}
-		} else {
-			content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			defer content.Close()
-			fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
-			if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
-				logrus.Fatal(err)
-			}
-		}
 		return nil
+
+	},
+	BashComplete: func(c *cli.Context) {
+		appNames, err := config.GetAppNames()
+		if err != nil {
+			logrus.Warn(err)
+		}
+		if c.NArg() > 0 {
+			return
+		}
+		for _, a := range appNames {
+			fmt.Println(a)
+		}
 	},
 }

cli/app/list.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/ssh"
@@ -70,14 +71,18 @@ can take some time.
 		}
 		sort.Sort(config.ByServerAndType(apps))
 
+		alreadySeen := make(map[string]bool)
 		for _, app := range apps {
-			if err := ssh.EnsureHostKey(app.Server); err != nil {
+			if _, ok := alreadySeen[app.Server]; !ok {
-				logrus.Fatal(err)
+				if err := ssh.EnsureHostKey(app.Server); err != nil {
+					logrus.Fatal(fmt.Sprintf(internal.SSHFailMsg, app.Server))
+				}
+				alreadySeen[app.Server] = true
 			}
 		}
 
 		statuses := make(map[string]map[string]string)
-		tableCol := []string{"Server", "Type", "Domain"}
+		tableCol := []string{"Server", "Type", "App Name", "Domain"}
 		if status {
 			tableCol = append(tableCol, "Status", "Version", "Updates")
 			statuses, err = config.GetAppStatuses(appFiles)
@@ -96,11 +101,19 @@ can take some time.
 			canUpgradeCount      int
 		)
 
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var appsCount int
 		for _, app := range apps {
 			var tableRow []string
 			if app.Type == appType || appType == "" {
+				appsCount++
+
 				// If type flag is set, check for it, if not, Type == ""
-				tableRow = []string{app.Server, app.Type, app.Domain}
+				tableRow = []string{app.Server, app.Type, app.StackName(), app.Domain}
 				if status {
 					stackName := app.StackName()
 					status := "unknown"
@@ -121,7 +134,8 @@ can take some time.
 
 					var newUpdates []string
 					if version != "unknown" {
-						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+
+						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 						if err != nil {
 							logrus.Fatal(err)
 						}
@@ -163,14 +177,19 @@ can take some time.
 			table.Append(tableRow)
 		}
 
-		stats := fmt.Sprintf(
+		var stats string
-			"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
+		if status {
-			len(apps),
+			stats = fmt.Sprintf(
-			versionedAppsCount,
+				"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
-			unversionedAppsCount,
+				appsCount,
-			onLatestCount,
+				versionedAppsCount,
-			canUpgradeCount,
+				unversionedAppsCount,
-		)
+				onLatestCount,
+				canUpgradeCount,
+			)
+		} else {
+			stats = fmt.Sprintf("Total apps: %v", appsCount)
+		}
 
 		table.SetCaption(true, stats)
 		table.Render()

cli/app/logs.go

@@ -4,74 +4,54 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"sync"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
-// stackLogs lists logs for all stack services
-func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
-	filters := filters.NewArgs()
-	filters.Add("name", stackName)
-	serviceOpts := types.ServiceListOptions{Filters: filters}
-	services, err := client.ServiceList(c.Context, serviceOpts)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	for _, service := range services {
-		wg.Add(1)
-		go func(s string) {
-			logOpts := types.ContainerLogsOptions{
-				Details:    true,
-				Follow:     true,
-				ShowStderr: true,
-				ShowStdout: true,
-				Tail:       "20",
-				Timestamps: true,
-			}
-			logs, err := client.ServiceLogs(c.Context, s, logOpts)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			// defer after err check as any err returns a nil io.ReadCloser
-			defer logs.Close()
-
-			_, err = io.Copy(os.Stdout, logs)
-			if err != nil && err != io.EOF {
-				logrus.Fatal(err)
-			}
-		}(service.ID)
-	}
-	wg.Wait()
-	os.Exit(0)
-}
-
 var appLogsCommand = &cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
 	ArgsUsage: "[<service>]",
 	Usage:     "Tail app logs",
+	Flags: []cli.Flag{
+		internal.StderrFlag,
+		internal.StdoutFlag,
+		internal.HealthcheckFlag,
+	},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		if !internal.Stderr && !internal.Stdout && !internal.Healthcheck {
+			internal.Stderr = true
+			internal.Stdout = true
+			internal.Healthcheck = true
+		}
+
+		logrus.Debugf("flags parsed. --stderr: %t, --stdout: %t, --healthcheck: %t", internal.Stderr, internal.Stdout, internal.Healthcheck)
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		logOpts := types.ContainerLogsOptions{
+			Details:    false,
+			Follow:     true,
+			ShowStderr: internal.Stderr,
+			ShowStdout: internal.Stdout,
+			Tail:       "20",
+			Timestamps: true,
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			logrus.Debug("tailing logs for all app services")
-			stackLogs(c, app.StackName(), cl)
+			internal.StackLogs(c, app.StackName(), logOpts, cl)
 		}
 		logrus.Debugf("tailing logs for '%s'", serviceName)
 
@@ -87,14 +67,6 @@ var appLogsCommand = &cli.Command{
 			logrus.Fatalf("expected 1 service but got %v", len(services))
 		}
 
-		logOpts := types.ContainerLogsOptions{
-			Details:    true,
-			Follow:     true,
-			ShowStderr: true,
-			ShowStdout: true,
-			Tail:       "20",
-			Timestamps: true,
-		}
 		logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
 		if err != nil {
 			logrus.Fatal(err)

cli/app/ps.go

@@ -26,9 +26,10 @@ var watchFlag = &cli.BoolFlag{
 }
 
 var appPsCommand = &cli.Command{
-	Name:    "ps",
+	Name:        "ps",
-	Usage:   "Check app status",
+	Usage:       "Check app status",
-	Aliases: []string{"p"},
+	Description: "This command shows a more detailed status output of a specific deployed app.",
+	Aliases:     []string{"p"},
 	Flags: []cli.Flag{
 		watchFlag,
 	},
@@ -75,7 +76,7 @@ func showPSOutput(c *cli.Context) {
 		logrus.Fatal(err)
 	}
 
-	tableCol := []string{"image", "created", "status", "ports", "names"}
+	tableCol := []string{"image", "created", "status", "ports", "app name", "services"}
 	table := abraFormatter.CreateTable(tableCol)
 
 	for _, container := range containers {
@@ -90,6 +91,7 @@ func showPSOutput(c *cli.Context) {
 			abraFormatter.HumanDuration(container.Created),
 			container.Status,
 			formatter.DisplayablePorts(container.Ports),
+			app.StackName(),
 			strings.Join(containerNames, "\n"),
 		}
 		table.Append(tableRow)

cli/app/remove.go

@@ -7,6 +7,7 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -48,23 +49,18 @@ var appRemoveCommand = &cli.Command{
 			}
 		}
 
-		appFiles, err := config.LoadAppFiles("")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
+
 		if !internal.Force {
-			// FIXME: only query for app we are interested in, not all of them!
+			isDeployed, _, err := stack.IsDeployed(c.Context, cl, app.StackName())
-			statuses, err := config.GetAppStatuses(appFiles)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			if statuses[app.Name]["status"] == "deployed" {
+			if isDeployed {
-				logrus.Fatalf("'%s' is still deployed. Run \"abra app %s undeploy\" or pass --force", app.Name, app.Name)
+				logrus.Fatalf("'%s' is still deployed. Run \"abra app undeploy %s \" or pass --force", app.Name, app.Name)
 			}
 		}
 

cli/app/rollback.go

@@ -70,7 +70,12 @@ recipes.
 			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/upgrade.go

@@ -62,7 +62,12 @@ recipes.
 			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/autocomplete.go

@@ -56,7 +56,7 @@ Example:
 
 Supported shells are as follows:
 
-		fish
+		fizsh
 		zsh
     bash
 `,
@@ -69,16 +69,16 @@ Supported shells are as follows:
 		}
 
 		supportedShells := map[string]bool{
-			"bash": true,
+			"bash":  true,
-			"zsh":  true,
+			"zsh":   true,
-			"fish": true,
+			"fizsh": true,
 		}
 
 		if _, ok := supportedShells[shellType]; !ok {
 			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
 		}
 
-		if shellType == "fish" {
+		if shellType == "fizsh" {
 			shellType = "zsh" // handled the same on the autocompletion side
 		}
 

cli/catalogue/generate.go

@@ -22,9 +22,9 @@ import (
 // CatalogueSkipList is all the repos that are not recipes.
 var CatalogueSkipList = map[string]bool{
 	"abra":                  true,
-	"abra-bash":             true,
 	"abra-apps":             true,
 	"abra-aur":              true,
+	"abra-bash":             true,
 	"abra-capsul":           true,
 	"abra-gandi":            true,
 	"abra-hetzner":          true,
@@ -34,20 +34,24 @@ var CatalogueSkipList = map[string]bool{
 	"auto-mirror":           true,
 	"backup-bot":            true,
 	"backup-bot-two":        true,
+	"comrade-renovate-bot":  true,
 	"coopcloud.tech":        true,
 	"coturn":                true,
 	"docker-cp-deploy":      true,
 	"docker-dind-bats-kcov": true,
 	"docs.coopcloud.tech":   true,
+	"drone-abra":            true,
 	"example":               true,
 	"gardening":             true,
 	"go-abra":               true,
 	"organising":            true,
 	"pyabra":                true,
 	"radicle-seed-node":     true,
+	"recipes":               true,
 	"stack-ssh-deploy":      true,
 	"swarm-cronjob":         true,
 	"tagcmp":                true,
+	"traefik-cert-dumper":   true,
 	"tyop":                  true,
 }
 
@@ -90,6 +94,9 @@ A new catalogue copy can be published to the recipes repository by passing the
 	ArgsUsage: "[<recipe>]",
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
 
 		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
@@ -129,6 +136,15 @@ A new catalogue copy can be published to the recipes repository by passing the
 					logrus.Fatal(err)
 				}
 
+				isClean, err := gitPkg.IsClean(rm.Name)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				if !isClean {
+					logrus.Fatalf("'%s' has locally unstaged changes", rm.Name)
+				}
+
 				if err := gitPkg.EnsureUpToDate(recipeDir); err != nil {
 					logrus.Fatal(err)
 				}
@@ -160,6 +176,11 @@ A new catalogue copy can be published to the recipes repository by passing the
 				logrus.Fatal(err)
 			}
 
+			features, category, err := catalogue.GetRecipeFeaturesAndCategory(recipeMeta.Name)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
 			catl[recipeMeta.Name] = catalogue.RecipeMeta{
 				Name:          recipeMeta.Name,
 				Repository:    recipeMeta.CloneURL,
@@ -168,8 +189,8 @@ A new catalogue copy can be published to the recipes repository by passing the
 				Description:   recipeMeta.Description,
 				Website:       recipeMeta.Website,
 				Versions:      versions,
-				// Category:      ..., // FIXME: parse & load
+				Category:      category,
-				// Features:      ..., // FIXME: parse & load
+				Features:      features,
 			}
 			catlBar.Add(1)
 		}

cli/cli.go

@@ -30,18 +30,6 @@ var VerboseFlag = &cli.BoolFlag{
 	Usage:       "Show INFO messages",
 }
 
-// Debug stores the variable from DebugFlag.
-var Debug bool
-
-// DebugFlag turns on/off verbose logging down to the DEBUG level.
-var DebugFlag = &cli.BoolFlag{
-	Name:        "debug",
-	Aliases:     []string{"d"},
-	Value:       false,
-	Destination: &Debug,
-	Usage:       "Show DEBUG messages",
-}
-
 func newAbraApp(version, commit string) *cli.App {
 	app := &cli.App{
 		Name: "abra",
@@ -66,7 +54,7 @@ func newAbraApp(version, commit string) *cli.App {
 		},
 		Flags: []cli.Flag{
 			VerboseFlag,
-			DebugFlag,
+			internal.DebugFlag,
 			internal.NoInputFlag,
 		},
 		Authors: []*cli.Author{
@@ -80,7 +68,7 @@ func newAbraApp(version, commit string) *cli.App {
 	app.EnableBashCompletion = true
 
 	app.Before = func(c *cli.Context) error {
-		if Debug {
+		if internal.Debug {
 			logrus.SetLevel(logrus.DebugLevel)
 			logrus.SetFormatter(&logrus.TextFormatter{})
 			logrus.SetOutput(os.Stderr)

cli/internal/common.go

@@ -259,3 +259,60 @@ var HetznerCloudAPITokenFlag = &cli.StringFlag{
 	EnvVars:     []string{"HCLOUD_TOKEN"},
 	Destination: &HetznerCloudAPIToken,
 }
+
+// Debug stores the variable from DebugFlag.
+var Debug bool
+
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
+var DebugFlag = &cli.BoolFlag{
+	Name:        "debug",
+	Aliases:     []string{"d"},
+	Value:       false,
+	Destination: &Debug,
+	Usage:       "Show DEBUG messages",
+}
+
+// SSHFailMsg is a hopefully helpful SSH failure message
+var SSHFailMsg = `
+Woops, Abra is unable to connect to connect to %s.
+
+Here are a few tips for debugging your local SSH config. Abra uses plain 'ol
+SSH to make connections to servers, so if your SSH config is working, Abra is
+working.
+
+In the first place, Abra will always try to read your Docker context connection
+string for SSH connection details. You can view your server context configs
+with the following command. Are they correct?
+
+    abra server ls
+
+Is your ssh-agent running? You can start it by running the following command:
+
+    eval "$(ssh-agent)"
+
+If your SSH private key loaded? You can check by running the following command:
+
+    ssh-add -L
+
+If you are using a non-default public/private key, you can configure this in
+your ~/.ssh/config file which Abra will read in order to figure out connection
+details:
+
+Host foo.coopcloud.tech
+  Hostname foo.coopcloud.tech
+  User bar
+  Port 12345
+  IdentityFile ~/.ssh/bar@foo.coopcloud.tech
+
+If you're only using password authentication, you can use the following config:
+
+Host foo.coopcloud.tech
+  Hostname foo.coopcloud.tech
+  User bar
+  Port 12345
+  PreferredAuthentications=password
+  PubkeyAuthentication=no
+
+Good luck!
+
+`

cli/internal/copy.go

@@ -0,0 +1,74 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+func ConfigureAndCp(c *cli.Context, app config.App, srcPath string, dstPath string, service string, isToContainer bool) error {
+	appFiles, err := config.LoadAppFiles("")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	appEnv, err := config.GetApp(appFiles, app.Name)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
+	containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if len(containers) != 1 {
+		logrus.Fatalf("expected 1 container but got %v", len(containers))
+	}
+	container := containers[0]
+
+	logrus.Debugf("retrieved '%s' as target container on '%s'", formatter.ShortenID(container.ID), app.Server)
+
+	if isToContainer {
+		if _, err := os.Stat(srcPath); err != nil {
+			logrus.Fatalf("'%s' does not exist?", srcPath)
+		}
+
+		toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		content, err := archive.TarWithOptions(srcPath, toTarOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+		if err := cl.CopyToContainer(c.Context, container.ID, dstPath, content, copyOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		content, _, err := cl.CopyFromContainer(c.Context, container.ID, srcPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		defer content.Close()
+		fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+		if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+	return nil
+}

cli/internal/deploy.go

@@ -8,6 +8,7 @@ import (
 	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
@@ -25,7 +26,7 @@ func DeployAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("checking whether '%s' is already deployed", stackName)
+	logrus.Debugf("checking whether %s is already deployed", stackName)
 
 	isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 	if err != nil {
@@ -33,24 +34,26 @@ func DeployAction(c *cli.Context) error {
 	}
 
 	if isDeployed {
-		if Force {
+		if Force || Chaos {
-			logrus.Warnf("'%s' already deployed but continuing (--force)", stackName)
+			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", stackName)
-		} else if Chaos {
-			logrus.Warnf("'%s' already deployed but continuing (--chaos)", stackName)
 		} else {
-			logrus.Fatalf("'%s' is already deployed", stackName)
+			logrus.Fatalf("%s is already deployed", stackName)
 		}
 	}
 
 	version := deployedVersion
 	if version == "" && !Chaos {
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		if len(versions) > 0 {
 			version = versions[len(versions)-1]
-			logrus.Debugf("choosing '%s' as version to deploy", version)
+			logrus.Debugf("choosing %s as version to deploy", version)
 			if err := recipe.EnsureVersion(app.Type, version); err != nil {
 				logrus.Fatal(err)
 			}
@@ -64,7 +67,13 @@ func DeployAction(c *cli.Context) error {
 	}
 
 	if version == "" && !Chaos {
-		logrus.Debugf("choosing '%s' as version to deploy", version)
+		logrus.Debugf("choosing %s as version to deploy", version)
+		if err := recipe.EnsureVersion(app.Type, version); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if version != "" && !Chaos {
 		if err := recipe.EnsureVersion(app.Type, version); err != nil {
 			logrus.Fatal(err)
 		}
@@ -107,6 +116,16 @@ func DeployAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
+	domainName := app.Env["DOMAIN"]
+	ipv4, err := dns.EnsureIPv4(domainName)
+	if err != nil || ipv4 == "" {
+		logrus.Fatalf("could not find an IP address assigned to %s?", domainName)
+	}
+
+	if _, err = dns.EnsureDomainsResolveSameIPv4(domainName, app.Server); err != nil {
+		logrus.Fatal(err)
+	}
+
 	if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
 		logrus.Fatal(err)
 	}

cli/internal/logs.go

@@ -0,0 +1,68 @@
+package internal
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Stderr bool
+var StderrFlag = &cli.BoolFlag{
+	Name:        "stderr",
+	Aliases:     []string{"e"},
+	Value:       false,
+	Destination: &Stderr,
+}
+
+var Stdout bool
+var StdoutFlag = &cli.BoolFlag{
+	Name:        "stdout",
+	Aliases:     []string{"o"},
+	Value:       false,
+	Destination: &Stdout,
+}
+
+var Healthcheck bool
+var HealthcheckFlag = &cli.BoolFlag{
+	Name:        "healthcheck",
+	Aliases:     []string{"c"},
+	Value:       false,
+	Destination: &Healthcheck,
+}
+
+// StackLogs lists logs for all stack services
+func StackLogs(c *cli.Context, stackName string, logOpts types.ContainerLogsOptions, client *dockerClient.Client) {
+	filters := filters.NewArgs()
+	filters.Add("name", stackName)
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(c.Context, serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			logs, err := client.ServiceLogs(c.Context, s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			// defer after err check as any err returns a nil io.ReadCloser
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+	wg.Wait()
+	os.Exit(0)
+}

cli/internal/validate.go

@@ -27,7 +27,11 @@ func ValidateRecipe(c *cli.Context) recipe.Recipe {
 
 	recipe, err := recipe.Get(recipeName)
 	if err != nil {
-		logrus.Fatal(err)
+		if c.Command.Name == "generate" {
+			logrus.Warn(err)
+		} else {
+			logrus.Fatal(err)
+		}
 	}
 
 	logrus.Debugf("validated '%s' as recipe argument", recipeName)
@@ -108,6 +112,35 @@ func ValidateApp(c *cli.Context) config.App {
 	return app
 }
 
+// ValidateAppByName ensures the app is valid and takes an app name as an argument, not context.
+func ValidateAppByName(c *cli.Context, appName string) config.App {
+	if AppName != "" {
+		appName = AppName
+		logrus.Debugf("programmatically setting app name to %s", appName)
+	}
+
+	if appName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+	}
+
+	app, err := app.Get(appName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := recipe.EnsureExists(app.Type); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ssh.EnsureHostKey(app.Server); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated '%s' as app argument", appName)
+
+	return app
+}
+
 // ValidateDomain ensures the domain name arg is valid.
 func ValidateDomain(c *cli.Context) (string, error) {
 	domainName := c.Args().First()

cli/recipe/new.go

@@ -82,6 +82,11 @@ The new example repository is cloned to ~/.abra/apps/<recipe>.
 			}
 		}
 
+		newGitRepo := path.Join(config.APPS_DIR, recipeName)
+		if err := git.Init(newGitRepo, true); err != nil {
+			logrus.Fatal(err)
+		}
+
 		logrus.Infof(
 			"new recipe '%s' created in %s, happy hacking!\n",
 			recipeName, path.Join(config.APPS_DIR, recipeName),

cli/server/add.go

@@ -1,23 +1,21 @@
 package server
 
 import (
-	"context"
 	"errors"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
 	"os/user"
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	abraFormatter "coopcloud.tech/abra/cli/formatter"
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	contextPkg "coopcloud.tech/abra/pkg/context"
+	"coopcloud.tech/abra/pkg/dns"
 	"coopcloud.tech/abra/pkg/server"
 	"coopcloud.tech/abra/pkg/ssh"
 	"github.com/AlecAivazis/survey/v2"
@@ -238,7 +236,7 @@ func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, d
 		}
 	}
 
-	logrus.Infof("docker is installed on %s", domainName)
+	logrus.Infof("docker is already installed on %s", domainName)
 
 	return nil
 }
@@ -268,39 +266,18 @@ func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string)
 }
 
 func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error {
-	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
+	ipv4, err := dns.EnsureIPv4(domainName)
-	freifunkDNS := "5.1.66.255:53"
-
-	resolver := &net.Resolver{
-		PreferGo: false,
-		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-			d := net.Dialer{
-				Timeout: time.Millisecond * time.Duration(10000),
-			}
-			return d.DialContext(ctx, "udp", freifunkDNS)
-		},
-	}
-
-	logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
-
-	ips, err := resolver.LookupIPAddr(c.Context, domainName)
 	if err != nil {
 		return err
 	}
 
-	if len(ips) == 0 {
-		return fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
-	}
-
-	ipv4 := ips[0].IP.To4().String()
-	logrus.Debugf("discovered the following ipv4 addr: %s", ipv4)
-
 	initReq := swarm.InitRequest{
 		ListenAddr:    "0.0.0.0:2377",
 		AdvertiseAddr: ipv4,
 	}
 	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
-		if !strings.Contains(err.Error(), "is already part of a swarm") {
+		if !strings.Contains(err.Error(), "is already part of a swarm") ||
+			!strings.Contains(err.Error(), "must specify a listening address") {
 			return err
 		}
 		logrus.Infof("swarm mode already initialised on %s", domainName)

cli/server/logs.go

@@ -0,0 +1,101 @@
+package server
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Taillen string
+var TaillenFlag = &cli.StringFlag{
+	Name:        "tail",
+	Aliases:     []string{"t"},
+	Value:       "5",
+	Destination: &Taillen,
+	Usage:       "change how many lines are shown",
+}
+
+var serverLogsCommand = &cli.Command{
+	Name:      "logs",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<server>",
+	Usage:     "show logs from all apps from server",
+	Flags: []cli.Flag{
+		TaillenFlag,
+		internal.StderrFlag,
+		internal.StdoutFlag,
+		internal.HealthcheckFlag,
+	},
+	Action: func(c *cli.Context) error {
+		serverName, err := internal.ValidateServer(c)
+		serviceName := ""
+		if !internal.Stderr && !internal.Stdout && !internal.Healthcheck {
+			internal.Stderr = true
+			internal.Stdout = true
+			internal.Healthcheck = true
+		}
+
+		logrus.Debugf("flags parsed. --stderr: %t, --stdout: %t, --healthcheck: %t", internal.Stderr, internal.Stdout, internal.Healthcheck)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		appMap, err := config.LoadAppFiles(serverName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logOpts := types.ContainerLogsOptions{
+			Details:    false,
+			Follow:     false,
+			ShowStderr: internal.Stderr,
+			ShowStdout: internal.Stdout,
+			Tail:       Taillen,
+			Timestamps: true,
+		}
+
+		var appFiles []config.App
+		for appname, _ := range appMap {
+			app := internal.ValidateAppByName(c, appname)
+			appFiles = append(appFiles, app)
+		}
+		for _, app := range appFiles {
+			fmt.Println(app)
+			logrus.Debugf("checking logs for: %s", app.Name)
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("tailing logs for all services")
+			filters := filters.NewArgs()
+			filters.Add("name", service)
+			serviceOpts := types.ServiceListOptions{Filters: filters}
+			services, err := cl.ServiceList(c.Context, serviceOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info(app.StackName())
+			for {
+				_, err = io.Copy(os.Stdout, logs)
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			logs.Close()
+		}
+		return nil
+	},
+}

cli/server/server.go

@@ -23,5 +23,6 @@ apps, see available flags on "server add" for more.
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,
+		serverLogsCommand,
 	},
 }

cli/upgrade.go

@@ -8,12 +8,24 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+var RC bool
+var RCFlag = &cli.BoolFlag{
+	Name:        "rc",
+	Value:       false,
+	Destination: &RC,
+	Usage:       "Insatll the latest Release Candidate",
+}
+
 // UpgradeCommand upgrades abra in-place.
 var UpgradeCommand = &cli.Command{
 	Name:  "upgrade",
 	Usage: "Upgrade abra",
+	Flags: []cli.Flag{RCFlag},
 	Action: func(c *cli.Context) error {
 		cmd := exec.Command("bash", "-c", "curl -s https://install.abra.coopcloud.tech | bash")
+		if RC {
+			cmd = exec.Command("bash", "-c", "curl -s https://git.coopcloud.tech/coop-cloud/abra/raw/branch/main/scripts/installer/installer | bash -s -- --rc")
+		}
 		logrus.Debugf("attempting to run '%s'", cmd)
 		if err := internal.RunCmd(cmd); err != nil {
 			logrus.Fatal(err)

go.mod

@@ -4,16 +4,16 @@ go 1.16
 
 require (
 	coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
-	github.com/AlecAivazis/survey/v2 v2.3.1
+	github.com/AlecAivazis/survey/v2 v2.3.2
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.8+incompatible
+	github.com/docker/cli v20.10.11+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/docker v20.10.11+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/go-git/go-git/v5 v5.4.2
-	github.com/hetznercloud/hcloud-go v1.32.0
+	github.com/hetznercloud/hcloud-go v1.33.1
-	github.com/moby/sys/signal v0.5.0
+	github.com/moby/sys/signal v0.6.0
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
@@ -31,10 +31,11 @@ require (
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/gliderlabs/ssh v0.2.2
+	github.com/gliderlabs/ssh v0.3.3
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351
+	github.com/hashicorp/go-retryablehttp v0.7.0
+	github.com/kevinburke/ssh_config v1.1.0
 	github.com/libdns/gandi v1.0.2
 	github.com/libdns/libdns v0.2.1
 	github.com/moby/sys/mount v0.2.0 // indirect
@@ -43,5 +44,5 @@ require (
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
-	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0
+	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
 )

go.sum

@@ -26,8 +26,8 @@ coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e/go.mod h1:HEQ9pSJRsD
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52 h1:cyFFOl0tKe+dVHt8saejG8xoff33eQiHxFCVzRpPUjM=
 coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi06jItF3rI7enf4Jt2PvbkWpDDHk1DQ=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/AlecAivazis/survey/v2 v2.3.1 h1:lzkuHA60pER7L4eYL8qQJor4bUWlJe4V0gqAT19tdOA=
+github.com/AlecAivazis/survey/v2 v2.3.2 h1:TqTB+aDDCLYhf9/bD2TwSO8u8jDSmMUd2SUVO4gCnU8=
-github.com/AlecAivazis/survey/v2 v2.3.1/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg=
+github.com/AlecAivazis/survey/v2 v2.3.2/go.mod h1:TH2kPCDU3Kqq7pLbnCWwZXDBjnhZtmsCle5EiYDJ2fg=
 github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4 h1:aYUdiI42a4fWfPoUr25XlaJrFEICv24+o/gWhqYS/jk=
 github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -88,8 +88,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
@@ -259,14 +260,14 @@ github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/cli v20.10.8+incompatible h1:/zO/6y9IOpcehE49yMRTV9ea0nBpb8OeqSskXLNfH1E=
+github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc=
-github.com/docker/cli v20.10.8+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.8+incompatible h1:RVqD337BgQicVCzYrrlhLDWhq6OAD2PJDUg2LsEUvKM=
+github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
-github.com/docker/docker v20.10.8+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
@@ -302,7 +303,6 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -315,8 +315,9 @@ github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui72
 github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/gliderlabs/ssh v0.3.3 h1:mBQ8NiOgDkINJrZtoizkC3nDNYgSaWtxyem6S2XHBtA=
+github.com/gliderlabs/ssh v0.3.3/go.mod h1:ZSS+CUoKHDrqVakTfTWUlKSr9MtMFkC4UvtQKD7O914=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
@@ -443,14 +444,20 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
+github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hetznercloud/hcloud-go v1.32.0 h1:7zyN2V7hMlhm3HZdxOarmOtvzKvkcYKjM0hcwYMQZz0=
+github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ=
-github.com/hetznercloud/hcloud-go v1.32.0/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
+github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
 github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ=
 github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -489,8 +496,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kevinburke/ssh_config v1.1.0 h1:pH/t1WS9NzT8go394IqZeJTMHVm6Cr6ZJ6AQ+mdNo/o=
+github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -563,8 +571,8 @@ github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7s
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/sys/signal v0.5.0 h1:MzpEFrMxugDynb1gkTIThU1O3wEmrAkOY+G9dHcHnCc=
+github.com/moby/sys/signal v0.6.0 h1:aDpY94H8VlhTGa9sNYUFCFsMZIUh5wm0B6XkIoJj/iY=
-github.com/moby/sys/signal v0.5.0/go.mod h1:JwObcMnOrUy2VTP5swPKWwywH0Mbgk8Y5qua9iwtIRM=
+github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
@@ -820,6 +828,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -972,8 +981,9 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
 golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=

pkg/catalogue/catalogue.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -46,6 +45,7 @@ type features struct {
 	Image       image  `json:"image"`
 	Status      int    `json:"status"`
 	Tests       string `json:"tests"`
+	SSO         string `json:"sso"`
 }
 
 // tag represents a git tag.
@@ -122,7 +122,7 @@ func (r ByRecipeName) Less(i, j int) bool {
 // recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
 // is up to date.
 func recipeCatalogueFSIsLatest() (bool, error) {
-	httpClient := &http.Client{Timeout: web.Timeout}
+	httpClient := web.NewHTTPRetryClient()
 	res, err := httpClient.Head(RecipeCatalogueURL)
 	if err != nil {
 		return false, err
@@ -374,6 +374,127 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 	return reposMeta, nil
 }
 
+func GetStringInBetween(str, start, end string) (result string, err error) {
+	// GetStringInBetween returns empty string if no start or end string found
+	s := strings.Index(str, start)
+	if s == -1 {
+		return "", fmt.Errorf("marker string '%s' not found", start)
+	}
+	s += len(start)
+	e := strings.Index(str[s:], end)
+	if e == -1 {
+		return "", fmt.Errorf("end marker '%s' not found", end)
+	}
+	return str[s : s+e], nil
+}
+
+func GetImageMetadata(imageRowString string) (image, error) {
+	img := image{}
+
+	imgFields := strings.Split(imageRowString, ",")
+
+	for i, elem := range imgFields {
+		imgFields[i] = strings.TrimSpace(elem)
+	}
+
+	if len(imgFields) < 3 {
+		logrus.Warnf("image string has incorrect format: %s", imageRowString)
+		return img, nil
+	}
+
+	img.Rating = imgFields[1]
+	img.Source = imgFields[2]
+
+	imgString := imgFields[0]
+
+	imageName, err := GetStringInBetween(imgString, "[", "]")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	img.Image = strings.ReplaceAll(imageName, "`", "")
+
+	imageURL, err := GetStringInBetween(imgString, "(", ")")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	img.URL = imageURL
+
+	return img, nil
+}
+
+func GetRecipeFeaturesAndCategory(recipeName string) (features, string, error) {
+	feat := features{}
+
+	var category string
+
+	readmePath := path.Join(config.ABRA_DIR, "apps", recipeName, "README.md")
+
+	logrus.Debugf("attempting to open '%s'", readmePath)
+
+	readmeFS, err := ioutil.ReadFile(readmePath)
+	if err != nil {
+		return feat, category, err
+	}
+
+	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
+		string(readmeFS),
+		"<!-- metadata -->", "<!-- endmetadata -->",
+	)
+	if err != nil {
+		return feat, category, err
+	}
+
+	readmeLines := strings.Split( // Array item from lines
+		strings.ReplaceAll( // Remove \t tabs
+			readmeMetadata, "\t", "",
+		),
+		"\n")
+
+	for _, val := range readmeLines {
+		if strings.Contains(val, "**Category**") {
+			category = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Category**:"),
+			)
+		}
+		if strings.Contains(val, "**Backups**") {
+			feat.Backups = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Backups**:"),
+			)
+		}
+		if strings.Contains(val, "**Email**") {
+			feat.Email = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Email**:"),
+			)
+		}
+		if strings.Contains(val, "**SSO**") {
+			feat.SSO = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **SSO**:"),
+			)
+		}
+		if strings.Contains(val, "**Healthcheck**") {
+			feat.Healthcheck = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Healthcheck**:"),
+			)
+		}
+		if strings.Contains(val, "**Tests**") {
+			feat.Tests = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Tests**:"),
+			)
+		}
+		if strings.Contains(val, "**Image**") {
+			imageMetadata, err := GetImageMetadata(strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Image**:"),
+			))
+			if err != nil {
+				continue
+			}
+			feat.Image = imageMetadata
+		}
+	}
+
+	return feat, category, nil
+}
+
 // GetRecipeVersions retrieves all recipe versions.
 func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 	versions := RecipeVersions{}
@@ -404,7 +525,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 
 		checkOutOpts := &git.CheckoutOptions{
 			Create: false,
-			Force:  true,
+			Keep:   true,
 			Branch: plumbing.ReferenceName(ref.Name()),
 		}
 		if err := worktree.Checkout(checkOutOpts); err != nil {
@@ -432,9 +553,21 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 				path = strings.Split(path, "/")[1]
 			}
 
+			var tag string
+			switch img.(type) {
+			case reference.NamedTagged:
+				tag = img.(reference.NamedTagged).Tag()
+			case reference.Named:
+				logrus.Warnf("%s service is missing image tag?", path)
+				continue
+			}
+
+			logrus.Debugf("looking up image: '%s' from '%s'", img, path)
+
 			digest, err := client.GetTagDigest(img)
 			if err != nil {
-				return err
+				logrus.Warn(err)
+				continue
 			}
 
 			versionMeta[service.Name] = ServiceMeta{
@@ -465,7 +598,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {
@@ -480,14 +613,9 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 }
 
 // GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
-func GetRecipeCatalogueVersions(recipeName string) ([]string, error) {
+func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
 	var versions []string
 
-	catl, err := ReadRecipeCatalogue()
-	if err != nil {
-		return versions, err
-	}
-
 	if recipeMeta, exists := catl[recipeName]; exists {
 		for _, versionMeta := range recipeMeta.Versions {
 			for tag := range versionMeta {

pkg/client/registry.go

@@ -9,6 +9,7 @@ import (
 
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/docker/distribution/reference"
+	"github.com/hashicorp/go-retryablehttp"
 )
 
 type RawTag struct {
@@ -35,12 +36,12 @@ func GetRegistryTags(image string) (RawTags, error) {
 func getRegv2Token(image reference.Named) (string, error) {
 	img := reference.Path(image)
 	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
-	req, err := http.NewRequest("GET", authTokenURL, nil)
+	req, err := retryablehttp.NewRequest("GET", authTokenURL, nil)
 	if err != nil {
 		return "", err
 	}
 
-	client := &http.Client{Timeout: web.Timeout}
+	client := web.NewHTTPRetryClient()
 	res, err := client.Do(req)
 	if err != nil {
 		return "", err
@@ -78,7 +79,7 @@ func GetTagDigest(image reference.Named) (string, error) {
 	tag := image.(reference.NamedTagged).Tag()
 	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
 
-	req, err := http.NewRequest("GET", manifestURL, nil)
+	req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
 	if err != nil {
 		return "", err
 	}
@@ -96,7 +97,7 @@ func GetTagDigest(image reference.Named) (string, error) {
 		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
 	}
 
-	client := &http.Client{Timeout: web.Timeout}
+	client := web.NewHTTPRetryClient()
 	res, err := client.Do(req)
 	if err != nil {
 		return "", err

pkg/dns/common.go

@@ -1,8 +1,11 @@
 package dns
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"os"
+	"time"
 
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/sirupsen/logrus"
@@ -26,3 +29,68 @@ func NewToken(provider, providerTokenEnvVar string) (string, error) {
 
 	return token, nil
 }
+
+// EnsureIPv4 ensures that an ipv4 address is set for a domain name
+func EnsureIPv4(domainName string) (string, error) {
+	var ipv4 string
+
+	// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
+	freifunkDNS := "5.1.66.255:53"
+
+	resolver := &net.Resolver{
+		PreferGo: false,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: time.Millisecond * time.Duration(10000),
+			}
+			return d.DialContext(ctx, "udp", freifunkDNS)
+		},
+	}
+
+	logrus.Debugf("created DNS resolver via '%s'", freifunkDNS)
+
+	ctx := context.Background()
+	ips, err := resolver.LookupIPAddr(ctx, domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if len(ips) == 0 {
+		return ipv4, fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
+	}
+
+	ipv4 = ips[0].IP.To4().String()
+	logrus.Debugf("discovered the following ipv4 addr: %s", ipv4)
+
+	return ipv4, nil
+}
+
+// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address
+func EnsureDomainsResolveSameIPv4(domainName, server string) (string, error) {
+	var ipv4 string
+
+	domainIPv4, err := EnsureIPv4(domainName)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if domainIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", domainName)
+	}
+
+	serverIPv4, err := EnsureIPv4(server)
+	if err != nil {
+		return ipv4, err
+	}
+
+	if serverIPv4 == "" {
+		return ipv4, fmt.Errorf("cannot resolve ipv4 for %s?", server)
+	}
+
+	if domainIPv4 != serverIPv4 {
+		err := "app domain %s (%s) does not appear to resolve to app server %s (%s)?"
+		return ipv4, fmt.Errorf(err, domainName, domainIPv4, server, serverIPv4)
+	}
+
+	return ipv4, nil
+}

pkg/git/clone.go

@@ -47,6 +47,16 @@ func EnsureUpToDate(dir string) error {
 		return err
 	}
 
+	recipeName := filepath.Base(dir)
+	isClean, err := IsClean(recipeName)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
+	}
+
 	branch := "master"
 	if _, err := repo.Branch("master"); err != nil {
 		if _, err := repo.Branch("main"); err != nil {
@@ -66,7 +76,7 @@ func EnsureUpToDate(dir string) error {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {

pkg/git/common.go

@@ -0,0 +1,14 @@
+package git
+
+import (
+	"fmt"
+	"os"
+)
+
+// EnsureGitRepo ensures a git repo .git folder exists
+func EnsureGitRepo(repoPath string) error {
+	if _, err := os.Stat(repoPath); os.IsNotExist(err) {
+		return fmt.Errorf("no .git directory in %s?", repoPath)
+	}
+	return nil
+}

pkg/git/init.go

@@ -0,0 +1,38 @@
+package git
+
+import (
+	"github.com/go-git/go-git/v5"
+	gitPkg "github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+)
+
+// Init inits a new repo and commits all the stuff if you want
+func Init(repoPath string, commit bool) error {
+	if _, err := gitPkg.PlainInit(repoPath, false); err != nil {
+		logrus.Fatal(err)
+	}
+	logrus.Debugf("initialised new git repo in %s", repoPath)
+
+	if commit {
+		commitRepo, err := git.PlainOpen(repoPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		commitWorktree, err := commitRepo.Worktree()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := commitWorktree.AddGlob("**"); err != nil {
+			return err
+		}
+
+		if _, err = commitWorktree.Commit("init", &git.CommitOptions{}); err != nil {
+			return err
+		}
+		logrus.Debugf("init committed all files for new git repo in %s", repoPath)
+	}
+
+	return nil
+}

pkg/git/read.go

@@ -1,11 +1,17 @@
 package git
 
 import (
+	"io/ioutil"
+	"os/user"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/go-git/go-git/v5"
+	gitConfigPkg "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,6 +46,12 @@ func IsClean(recipeName string) (bool, error) {
 		return false, err
 	}
 
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return false, err
+	}
+	worktree.Excludes = append(patterns, worktree.Excludes...)
+
 	status, err := worktree.Status()
 	if err != nil {
 		return false, err
@@ -53,3 +65,95 @@ func IsClean(recipeName string) (bool, error) {
 
 	return status.IsClean(), nil
 }
+
+// GetExcludesFiles reads the exlude files from a global git ignore
+func GetExcludesFiles() ([]gitignore.Pattern, error) {
+	var err error
+	var patterns []gitignore.Pattern
+
+	cfg, err := parseGitConfig()
+	if err != nil {
+		return patterns, err
+	}
+
+	excludesfile := getExcludesFile(cfg)
+	patterns, err = parseExcludesFile(excludesfile)
+	if err != nil {
+		return patterns, err
+	}
+
+	return patterns, nil
+}
+
+func parseGitConfig() (*gitConfigPkg.Config, error) {
+	cfg := gitConfigPkg.NewConfig()
+
+	usr, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := ioutil.ReadFile(usr.HomeDir + "/.gitconfig")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cfg.Unmarshal(b); err != nil {
+		return nil, err
+	}
+
+	return cfg, err
+}
+
+func getExcludesFile(cfg *gitConfigPkg.Config) string {
+	for _, sec := range cfg.Raw.Sections {
+		if sec.Name == "core" {
+			for _, opt := range sec.Options {
+				if opt.Key == "excludesfile" {
+					return opt.Value
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func parseExcludesFile(excludesfile string) ([]gitignore.Pattern, error) {
+	excludesfile, err := expandTilde(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadFile(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	var ps []gitignore.Pattern
+	for _, s := range strings.Split(string(data), "\n") {
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			ps = append(ps, gitignore.ParsePattern(s, nil))
+		}
+	}
+
+	return ps, nil
+}
+
+func expandTilde(path string) (string, error) {
+	if !strings.HasPrefix(path, "~") {
+		return path, nil
+	}
+	var paths []string
+	u, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+	for _, p := range strings.Split(path, string(filepath.Separator)) {
+		if p == "~" {
+			paths = append(paths, u.HomeDir)
+		} else {
+			paths = append(paths, p)
+		}
+	}
+	return "/" + filepath.Join(paths...), nil
+}

pkg/recipe/recipe.go

@@ -2,6 +2,7 @@ package recipe
 
 import (
 	"fmt"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -83,7 +84,7 @@ func Get(recipeName string) (Recipe, error) {
 	envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
 	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
-		logrus.Fatal(err)
+		return Recipe{}, err
 	}
 
 	opts := stack.Deploy{Composefiles: composeFiles}
@@ -95,13 +96,22 @@ func Get(recipeName string) (Recipe, error) {
 	return Recipe{Name: recipeName, Config: config}, nil
 }
 
-// EnsureExists checks whether a recipe has been cloned locally or not.
+// EnsureExists ensures that a recipe is locally cloned
 func EnsureExists(recipe string) error {
 	recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(recipe))
-	url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe)
+
-	if err := gitPkg.Clone(recipeDir, url); err != nil {
+	if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
+		logrus.Debugf("%s does not exist, attemmpting to clone", recipeDir)
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe)
+		if err := gitPkg.Clone(recipeDir, url); err != nil {
+			return err
+		}
+	}
+
+	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -118,6 +128,10 @@ func EnsureVersion(recipeName, version string) error {
 		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
 	}
 
+	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
+		return err
+	}
+
 	repo, err := git.PlainOpen(recipeDir)
 	if err != nil {
 		return err
@@ -143,7 +157,8 @@ func EnsureVersion(recipeName, version string) error {
 	logrus.Debugf("read '%s' as tags for recipe '%s'", strings.Join(parsedTags, ", "), recipeName)
 
 	if tagRef.String() == "" {
-		return fmt.Errorf("%s is not available?", version)
+		logrus.Warnf("%s recipe has no local tag: %s? this recipe version is not released?", recipeName, version)
+		return nil
 	}
 
 	worktree, err := repo.Worktree()
@@ -154,7 +169,7 @@ func EnsureVersion(recipeName, version string) error {
 	opts := &git.CheckoutOptions{
 		Branch: tagRef,
 		Create: false,
-		Force:  true,
+		Keep:   true,
 	}
 	if err := worktree.Checkout(opts); err != nil {
 		return err
@@ -178,6 +193,10 @@ func EnsureLatest(recipeName string) error {
 		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
 	}
 
+	if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
+		return err
+	}
+
 	logrus.Debugf("attempting to open git repository in '%s'", recipeDir)
 
 	repo, err := git.PlainOpen(recipeDir)
@@ -202,7 +221,7 @@ func EnsureLatest(recipeName string) error {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 

pkg/ssh/ssh.go

@@ -330,9 +330,9 @@ func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.Publ
 
 		fmt.Printf(fmt.Sprintf(`
 You are attempting to make an SSH connection to a server but there is no entry
-in your ~/.ssh/known_hosts file which confirms that this is indeed the server
+in your ~/.ssh/known_hosts file which confirms that you have already validated
-you want to connect to. Please take a moment to validate the following SSH host
+that this is indeed the server you want to connect to. Please take a moment to
-key, it is important.
+validate the following SSH host key, it is important.
 
     Host:        %s
     Fingerprint: %s
@@ -409,12 +409,31 @@ func connect(username, host, port string, authMethod ssh.AuthMethod, timeout tim
 }
 
 func connectWithAgentTimeout(host, username, port string, timeout time.Duration) (*Client, error) {
+	logrus.Debugf("using ssh-agent to make an SSH connection for %s", host)
+
 	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
 	if err != nil {
 		return nil, err
 	}
 
-	authMethod := ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
+	agentCl := agent.NewClient(sshAgent)
+	authMethod := ssh.PublicKeysCallback(agentCl.Signers)
+
+	loadedKeys, err := agentCl.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var convertedKeys []string
+	for _, key := range loadedKeys {
+		convertedKeys = append(convertedKeys, key.String())
+	}
+
+	if len(convertedKeys) > 0 {
+		logrus.Debugf("ssh-agent has these keys loaded: %s", strings.Join(convertedKeys, ","))
+	} else {
+		logrus.Debug("ssh-agent has no keys loaded")
+	}
 
 	return connect(username, host, port, authMethod, timeout)
 }
@@ -544,11 +563,16 @@ func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 	}
 
 	idf = ssh_config.Get(hostname, "IdentityFile")
-
-	hostConfig.Host = host
 	if idf != "" {
+		var err error
+		idf, err = identityFileAbsPath(idf)
+		if err != nil {
+			return hostConfig, err
+		}
 		hostConfig.IdentityFile = idf
 	}
+
+	hostConfig.Host = host
 	hostConfig.Port = port
 	hostConfig.User = username
 
@@ -556,3 +580,25 @@ func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 
 	return hostConfig, nil
 }
+
+func identityFileAbsPath(relPath string) (string, error) {
+	var err error
+	var absPath string
+
+	if strings.HasPrefix(relPath, "~/") {
+		systemUser, err := user.Current()
+		if err != nil {
+			return absPath, err
+		}
+		absPath = filepath.Join(systemUser.HomeDir, relPath[2:])
+	} else {
+		absPath, err = filepath.Abs(relPath)
+		if err != nil {
+			return absPath, err
+		}
+	}
+
+	logrus.Debugf("resolved %s to %s to read the ssh identity file", relPath, absPath)
+
+	return absPath, nil
+}

pkg/upstream/commandconn/connection.go

@@ -2,6 +2,7 @@ package commandconn
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/url"
 
@@ -34,9 +35,25 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 		if err != nil {
 			return nil, errors.Wrap(err, "ssh host connection is not valid")
 		}
+
 		if err := sshPkg.EnsureHostKey(ctxConnDetails.Host); err != nil {
 			return nil, err
 		}
+
+		hostConfig, err := sshPkg.GetHostConfig(
+			ctxConnDetails.Host,
+			ctxConnDetails.User,
+			ctxConnDetails.Port,
+		)
+		if err != nil {
+			return nil, err
+		}
+		if hostConfig.IdentityFile != "" {
+			msg := "discovered %s as identity file for %s, using for ssh connection"
+			logrus.Debugf(msg, hostConfig.IdentityFile, ctxConnDetails.Host)
+			sshFlags = append(sshFlags, fmt.Sprintf("-o IdentityFile=%s", hostConfig.IdentityFile))
+		}
+
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)

pkg/upstream/stack/loader.go

@@ -13,6 +13,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// DontSkipValidation ensures validation is done for compose file loading
+func DontSkipValidation(opts *loader.Options) {
+	opts.SkipValidation = false
+}
+
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
 func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
@@ -21,13 +26,12 @@ func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Confi
 	}
 
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails)
+	config, err := loader.Load(configDetails, DontSkipValidation)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
 			return nil, fmt.Errorf("compose file contains unsupported options:\n\n%s",
 				propertyWarnings(fpe.Properties))
 		}
-
 		return nil, err
 	}
 

pkg/upstream/stack/stack.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"strings"
+	"time"
 
 	abraClient "coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/upstream/convert"
@@ -349,7 +350,7 @@ func deployServices(
 		existingServiceMap[service.Spec.Name] = service
 	}
 
-	var serviceIDs []string
+	serviceIDs := make(map[string]string)
 	for internalName, serviceSpec := range services {
 		var (
 			name        = namespace.Scope(internalName)
@@ -409,7 +410,7 @@ func deployServices(
 				return errors.Wrapf(err, "failed to update service %s", name)
 			}
 
-			serviceIDs = append(serviceIDs, service.ID)
+			serviceIDs[service.ID] = name
 
 			for _, warning := range response.Warnings {
 				logrus.Warn(warning)
@@ -429,15 +430,19 @@ func deployServices(
 				return errors.Wrapf(err, "failed to create service %s", name)
 			}
 
-			serviceIDs = append(serviceIDs, serviceCreateResponse.ID)
+			serviceIDs[serviceCreateResponse.ID] = name
 		}
 	}
 
-	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceIDs, ", "))
+	var serviceNames []string
+	for _, serviceName := range serviceIDs {
+		serviceNames = append(serviceNames, serviceName)
+	}
+	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
 
 	ch := make(chan error, len(serviceIDs))
-	for _, serviceID := range serviceIDs {
+	for serviceID, serviceName := range serviceIDs {
-		logrus.Debugf("waiting on %s to converge", serviceID)
+		logrus.Debugf("waiting on %s to converge", serviceName)
 		go func(s string) {
 			ch <- waitOnService(ctx, cl, s)
 		}(serviceID)
@@ -479,5 +484,13 @@ func waitOnService(ctx context.Context, cl *dockerclient.Client, serviceID strin
 	}()
 
 	go io.Copy(ioutil.Discard, pipeReader)
-	return <-errChan
+
+	timeout := 60 * time.Second
+
+	select {
+	case err := <-errChan:
+		return err
+	case <-time.After(timeout):
+		return fmt.Errorf("%s has still not converged (%s second timeout)?", serviceID, timeout)
+	}
 }

pkg/web/client.go

@@ -0,0 +1,23 @@
+package web
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/sirupsen/logrus"
+)
+
+type customLeveledLogger struct {
+	retryablehttp.Logger
+}
+
+func (l customLeveledLogger) Printf(msg string, args ...interface{}) {
+	logrus.Debugf(fmt.Sprintf(msg, args...))
+}
+
+// NewHTTPRetryClient instantiates a new http client with retries baked in
+func NewHTTPRetryClient() *retryablehttp.Client {
+	retryClient := retryablehttp.NewClient()
+	retryClient.Logger = customLeveledLogger{}
+	return retryClient
+}

pkg/web/web.go

@@ -3,7 +3,6 @@ package web
 
 import (
 	"encoding/json"
-	"net/http"
 	"time"
 )
 
@@ -13,7 +12,7 @@ const Timeout = 10 * time.Second
 
 // ReadJSON reads JSON and parses it into your chosen interface pointer
 func ReadJSON(url string, target interface{}) error {
-	httpClient := &http.Client{Timeout: Timeout}
+	httpClient := NewHTTPRetryClient()
 	res, err := httpClient.Get(url)
 	if err != nil {
 		return err

renovate.json

@@ -0,0 +1,3 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json"
+}

scripts/installer/installer

@@ -2,7 +2,7 @@
 
 ABRA_VERSION="0.3.0-alpha"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.3.1-alpha-rc1"
+RC_VERSION="0.3.1-alpha-rc2"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do
@@ -76,13 +76,13 @@ function install_abra_release {
       p=$HOME/.local/bin
       com="echo PATH=\$PATH:$p"
       if [[ $SHELL =~ "bash" ]]; then
-          echo "echo $com >> $HOME/.bashrc"
+          echo "$com >> $HOME/.bashrc"
       elif [[ $SHELL =~ "fizsh" ]]; then
-          echo "echo $com >> $HOME/.fizsh/.fizshrc"
+          echo "$com >> $HOME/.fizsh/.fizshrc"
       elif [[ $SHELL =~ "zsh" ]]; then
-          echo "echo $com >> $HOME/.zshrc"
+          echo "$com >> $HOME/.zshrc"
       else
-          echo "echo $com >> $HOME/.profile"
+          echo "$com >> $HOME/.profile"
       fi
   fi