NOT WORKING: WIP on adding server-wide logs functionality

Closes coop-cloud/organising#266.

cli/app/app.go

@@ -31,7 +31,6 @@ to scaling apps up and spinning them down.
 		appLogsCommand,
 		appCpCommand,
 		appRunCommand,
-		appEditCommand,
 		appRollbackCommand,
 		appSecretCommand,
 		appVolumeCommand,

cli/app/edit.go

@@ -1,95 +0,0 @@
-package app
-
-import (
-	"fmt"
-	"os"
-	"strings"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/config"
-	"coopcloud.tech/abra/pkg/upstream/container"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-var appEditCommand = &cli.Command{
-	Name:    "edit-file",
-	Aliases: []string{"e", "edit"},
-	Usage:   "Edit a file in the container",
-	Description: `
-This command allows you to edit files inside a runnning container. This is 
-usually discouraged but sometimes necessary. Syntax:
-	
-	abra app edit-file <app> <service> <file>
-
-	i.e.
-	abra app edit-file traefik_example_com app /etc/passwd
-
-It will automatically get the ownership and access rights of the file using
-stat inside the container and then run chmod and chown after sending the file.
-	`,
-	Action: func(c *cli.Context) error {
-		app := internal.ValidateApp(c)
-
-		service := c.Args().Get(1)
-		file := c.Args().Get(2)
-		if file == "" {
-			logrus.Fatal("missing <file> argument")
-		} else if service == "" {
-			logrus.Fatal("missing <service> argument")
-		}
-		splitpath := strings.Split(file, "/")
-		filename := splitpath[len(splitpath)-1]
-		editDir := fmt.Sprintf("%s/tmp/edits/%s_%s", config.ABRA_DIR, app.Name, service)
-		if err := os.MkdirAll(editDir, 0755); err != nil {
-			logrus.Fatal(err)
-		}
-		fmt.Println("Success!!")
-		fmt.Println(editDir)
-		err := internal.ConfigureAndCp(c, app, file, editDir, service, false)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		// pull stuff from stat
-		cmd := []string{"stat", "-c", "%a", file}
-		execCreateOpts := types.ExecConfig{
-			AttachStderr: true,
-			AttachStdin:  true,
-			AttachStdout: true,
-			Cmd:          cmd,
-			Detach:       false,
-			Tty:          true,
-		}
-
-		// FIXME: an absolutely monumental hack to instantiate another command-line
-		// client withing our command-line client so that we pass something down
-		// the tubes that satisfies the necessary interface requirements. We should
-		// refactor our vendored container code to not require all this cruft.  For
-		// now, It Works.
-		dcli, err := command.NewDockerCli()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if err := container.RunExec(dcli, cl, containers[0].ID, &execCreateOpts); err != nil {
-			logrus.Fatal(err)
-		}
-		return nil
-
-	},
-	BashComplete: func(c *cli.Context) {
-		appNames, err := config.GetAppNames()
-		if err != nil {
-			logrus.Warn(err)
-		}
-		if c.NArg() > 0 {
-			return
-		}
-		for _, a := range appNames {
-			fmt.Println(a)
-		}
-	},
-}

cli/app/list.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/catalogue"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/ssh"
@@ -70,14 +71,18 @@ can take some time.
 		}
 		sort.Sort(config.ByServerAndType(apps))
 
+		alreadySeen := make(map[string]bool)
 		for _, app := range apps {
-			if err := ssh.EnsureHostKey(app.Server); err != nil {
-				logrus.Fatal(err)
+			if _, ok := alreadySeen[app.Server]; !ok {
+				if err := ssh.EnsureHostKey(app.Server); err != nil {
+					logrus.Fatal(fmt.Sprintf(internal.SSHFailMsg, app.Server))
+				}
+				alreadySeen[app.Server] = true
 			}
 		}
 
 		statuses := make(map[string]map[string]string)
-		tableCol := []string{"Server", "Type", "Domain"}
+		tableCol := []string{"Server", "Type", "App Name", "Domain"}
 		if status {
 			tableCol = append(tableCol, "Status", "Version", "Updates")
 			statuses, err = config.GetAppStatuses(appFiles)
@@ -96,11 +101,19 @@ can take some time.
 			canUpgradeCount      int
 		)
 
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var appsCount int
 		for _, app := range apps {
 			var tableRow []string
 			if app.Type == appType || appType == "" {
+				appsCount++
+
 				// If type flag is set, check for it, if not, Type == ""
-				tableRow = []string{app.Server, app.Type, app.Domain}
+				tableRow = []string{app.Server, app.Type, app.StackName(), app.Domain}
 				if status {
 					stackName := app.StackName()
 					status := "unknown"
@@ -121,7 +134,8 @@ can take some time.
 
 					var newUpdates []string
 					if version != "unknown" {
-						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+
+						updates, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 						if err != nil {
 							logrus.Fatal(err)
 						}
@@ -163,14 +177,19 @@ can take some time.
 			table.Append(tableRow)
 		}
 
-		stats := fmt.Sprintf(
-			"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
-			len(apps),
-			versionedAppsCount,
-			unversionedAppsCount,
-			onLatestCount,
-			canUpgradeCount,
-		)
+		var stats string
+		if status {
+			stats = fmt.Sprintf(
+				"Total apps: %v | Versioned: %v | Unversioned: %v | On latest: %v | Can upgrade: %v",
+				appsCount,
+				versionedAppsCount,
+				unversionedAppsCount,
+				onLatestCount,
+				canUpgradeCount,
+			)
+		} else {
+			stats = fmt.Sprintf("Total apps: %v", appsCount)
+		}
 
 		table.SetCaption(true, stats)
 		table.Render()

cli/app/logs.go

@@ -4,74 +4,54 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"sync"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	dockerClient "github.com/docker/docker/client"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
-// stackLogs lists logs for all stack services
-func stackLogs(c *cli.Context, stackName string, client *dockerClient.Client) {
-	filters := filters.NewArgs()
-	filters.Add("name", stackName)
-	serviceOpts := types.ServiceListOptions{Filters: filters}
-	services, err := client.ServiceList(c.Context, serviceOpts)
-	if err != nil {
-		logrus.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	for _, service := range services {
-		wg.Add(1)
-		go func(s string) {
-			logOpts := types.ContainerLogsOptions{
-				Details:    true,
-				Follow:     true,
-				ShowStderr: true,
-				ShowStdout: true,
-				Tail:       "20",
-				Timestamps: true,
-			}
-			logs, err := client.ServiceLogs(c.Context, s, logOpts)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-			// defer after err check as any err returns a nil io.ReadCloser
-			defer logs.Close()
-
-			_, err = io.Copy(os.Stdout, logs)
-			if err != nil && err != io.EOF {
-				logrus.Fatal(err)
-			}
-		}(service.ID)
-	}
-	wg.Wait()
-	os.Exit(0)
-}
-
 var appLogsCommand = &cli.Command{
 	Name:      "logs",
 	Aliases:   []string{"l"},
 	ArgsUsage: "[<service>]",
 	Usage:     "Tail app logs",
+	Flags: []cli.Flag{
+		internal.StderrFlag,
+		internal.StdoutFlag,
+		internal.HealthcheckFlag,
+	},
 	Action: func(c *cli.Context) error {
 		app := internal.ValidateApp(c)
 
+		if !internal.Stderr && !internal.Stdout && !internal.Healthcheck {
+			internal.Stderr = true
+			internal.Stdout = true
+			internal.Healthcheck = true
+		}
+
+		logrus.Debugf("flags parsed. --stderr: %t, --stdout: %t, --healthcheck: %t", internal.Stderr, internal.Stdout, internal.Healthcheck)
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
+		logOpts := types.ContainerLogsOptions{
+			Details:    false,
+			Follow:     true,
+			ShowStderr: internal.Stderr,
+			ShowStdout: internal.Stdout,
+			Tail:       "20",
+			Timestamps: true,
+		}
+
 		serviceName := c.Args().Get(1)
 		if serviceName == "" {
 			logrus.Debug("tailing logs for all app services")
-			stackLogs(c, app.StackName(), cl)
+			internal.StackLogs(c, app.StackName(), logOpts, cl)
 		}
 		logrus.Debugf("tailing logs for '%s'", serviceName)
 
@@ -87,14 +67,6 @@ var appLogsCommand = &cli.Command{
 			logrus.Fatalf("expected 1 service but got %v", len(services))
 		}
 
-		logOpts := types.ContainerLogsOptions{
-			Details:    true,
-			Follow:     true,
-			ShowStderr: true,
-			ShowStdout: true,
-			Tail:       "20",
-			Timestamps: true,
-		}
 		logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
 		if err != nil {
 			logrus.Fatal(err)

cli/app/ps.go

@@ -26,9 +26,10 @@ var watchFlag = &cli.BoolFlag{
 }
 
 var appPsCommand = &cli.Command{
-	Name:    "ps",
-	Usage:   "Check app status",
-	Aliases: []string{"p"},
+	Name:        "ps",
+	Usage:       "Check app status",
+	Description: "This command shows a more detailed status output of a specific deployed app.",
+	Aliases:     []string{"p"},
 	Flags: []cli.Flag{
 		watchFlag,
 	},
@@ -75,7 +76,7 @@ func showPSOutput(c *cli.Context) {
 		logrus.Fatal(err)
 	}
 
-	tableCol := []string{"image", "created", "status", "ports", "names"}
+	tableCol := []string{"image", "created", "status", "ports", "app name", "services"}
 	table := abraFormatter.CreateTable(tableCol)
 
 	for _, container := range containers {
@@ -90,6 +91,7 @@ func showPSOutput(c *cli.Context) {
 			abraFormatter.HumanDuration(container.Created),
 			container.Status,
 			formatter.DisplayablePorts(container.Ports),
+			app.StackName(),
 			strings.Join(containerNames, "\n"),
 		}
 		table.Append(tableRow)

cli/app/remove.go

@@ -7,6 +7,7 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
+	stack "coopcloud.tech/abra/pkg/upstream/stack"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -48,23 +49,18 @@ var appRemoveCommand = &cli.Command{
 			}
 		}
 
-		appFiles, err := config.LoadAppFiles("")
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			logrus.Fatal(err)
 		}
+
 		if !internal.Force {
-			// FIXME: only query for app we are interested in, not all of them!
-			statuses, err := config.GetAppStatuses(appFiles)
+			isDeployed, _, err := stack.IsDeployed(c.Context, cl, app.StackName())
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			if statuses[app.Name]["status"] == "deployed" {
-				logrus.Fatalf("'%s' is still deployed. Run \"abra app %s undeploy\" or pass --force", app.Name, app.Name)
+			if isDeployed {
+				logrus.Fatalf("'%s' is still deployed. Run \"abra app undeploy %s \" or pass --force", app.Name, app.Name)
 			}
 		}
 

cli/app/rollback.go

@@ -70,7 +70,12 @@ recipes.
 			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/app/upgrade.go

@@ -62,7 +62,12 @@ recipes.
 			logrus.Fatalf("'%s' is not deployed?", app.Name)
 		}
 
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}

cli/autocomplete.go

@@ -56,7 +56,7 @@ Example:
 
 Supported shells are as follows:
 
-		fish
+		fizsh
 		zsh
     bash
 `,
@@ -69,16 +69,16 @@ Supported shells are as follows:
 		}
 
 		supportedShells := map[string]bool{
-			"bash": true,
-			"zsh":  true,
-			"fish": true,
+			"bash":  true,
+			"zsh":   true,
+			"fizsh": true,
 		}
 
 		if _, ok := supportedShells[shellType]; !ok {
 			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
 		}
 
-		if shellType == "fish" {
+		if shellType == "fizsh" {
 			shellType = "zsh" // handled the same on the autocompletion side
 		}
 

cli/catalogue/generate.go

@@ -40,15 +40,18 @@ var CatalogueSkipList = map[string]bool{
 	"docker-cp-deploy":      true,
 	"docker-dind-bats-kcov": true,
 	"docs.coopcloud.tech":   true,
+	"drone-abra":            true,
 	"example":               true,
 	"gardening":             true,
 	"go-abra":               true,
 	"organising":            true,
 	"pyabra":                true,
 	"radicle-seed-node":     true,
+	"recipes":               true,
 	"stack-ssh-deploy":      true,
 	"swarm-cronjob":         true,
 	"tagcmp":                true,
+	"traefik-cert-dumper":   true,
 	"tyop":                  true,
 }
 
@@ -91,6 +94,9 @@ A new catalogue copy can be published to the recipes repository by passing the
 	ArgsUsage: "[<recipe>]",
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
+		if recipeName != "" {
+			internal.ValidateRecipe(c)
+		}
 
 		catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes")
@@ -130,6 +136,15 @@ A new catalogue copy can be published to the recipes repository by passing the
 					logrus.Fatal(err)
 				}
 
+				isClean, err := gitPkg.IsClean(rm.Name)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				if !isClean {
+					logrus.Fatalf("'%s' has locally unstaged changes", rm.Name)
+				}
+
 				if err := gitPkg.EnsureUpToDate(recipeDir); err != nil {
 					logrus.Fatal(err)
 				}
@@ -161,6 +176,11 @@ A new catalogue copy can be published to the recipes repository by passing the
 				logrus.Fatal(err)
 			}
 
+			features, category, err := catalogue.GetRecipeFeaturesAndCategory(recipeMeta.Name)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
 			catl[recipeMeta.Name] = catalogue.RecipeMeta{
 				Name:          recipeMeta.Name,
 				Repository:    recipeMeta.CloneURL,
@@ -169,8 +189,8 @@ A new catalogue copy can be published to the recipes repository by passing the
 				Description:   recipeMeta.Description,
 				Website:       recipeMeta.Website,
 				Versions:      versions,
-				// Category:      ..., // FIXME: parse & load
-				// Features:      ..., // FIXME: parse & load
+				Category:      category,
+				Features:      features,
 			}
 			catlBar.Add(1)
 		}

cli/cli.go

@@ -30,18 +30,6 @@ var VerboseFlag = &cli.BoolFlag{
 	Usage:       "Show INFO messages",
 }
 
-// Debug stores the variable from DebugFlag.
-var Debug bool
-
-// DebugFlag turns on/off verbose logging down to the DEBUG level.
-var DebugFlag = &cli.BoolFlag{
-	Name:        "debug",
-	Aliases:     []string{"d"},
-	Value:       false,
-	Destination: &Debug,
-	Usage:       "Show DEBUG messages",
-}
-
 func newAbraApp(version, commit string) *cli.App {
 	app := &cli.App{
 		Name: "abra",
@@ -66,7 +54,7 @@ func newAbraApp(version, commit string) *cli.App {
 		},
 		Flags: []cli.Flag{
 			VerboseFlag,
-			DebugFlag,
+			internal.DebugFlag,
 			internal.NoInputFlag,
 		},
 		Authors: []*cli.Author{
@@ -80,7 +68,7 @@ func newAbraApp(version, commit string) *cli.App {
 	app.EnableBashCompletion = true
 
 	app.Before = func(c *cli.Context) error {
-		if Debug {
+		if internal.Debug {
 			logrus.SetLevel(logrus.DebugLevel)
 			logrus.SetFormatter(&logrus.TextFormatter{})
 			logrus.SetOutput(os.Stderr)

cli/internal/common.go

@@ -259,3 +259,60 @@ var HetznerCloudAPITokenFlag = &cli.StringFlag{
 	EnvVars:     []string{"HCLOUD_TOKEN"},
 	Destination: &HetznerCloudAPIToken,
 }
+
+// Debug stores the variable from DebugFlag.
+var Debug bool
+
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
+var DebugFlag = &cli.BoolFlag{
+	Name:        "debug",
+	Aliases:     []string{"d"},
+	Value:       false,
+	Destination: &Debug,
+	Usage:       "Show DEBUG messages",
+}
+
+// SSHFailMsg is a hopefully helpful SSH failure message
+var SSHFailMsg = `
+Woops, Abra is unable to connect to connect to %s.
+
+Here are a few tips for debugging your local SSH config. Abra uses plain 'ol
+SSH to make connections to servers, so if your SSH config is working, Abra is
+working.
+
+In the first place, Abra will always try to read your Docker context connection
+string for SSH connection details. You can view your server context configs
+with the following command. Are they correct?
+
+    abra server ls
+
+Is your ssh-agent running? You can start it by running the following command:
+
+    eval "$(ssh-agent)"
+
+If your SSH private key loaded? You can check by running the following command:
+
+    ssh-add -L
+
+If you are using a non-default public/private key, you can configure this in
+your ~/.ssh/config file which Abra will read in order to figure out connection
+details:
+
+Host foo.coopcloud.tech
+  Hostname foo.coopcloud.tech
+  User bar
+  Port 12345
+  IdentityFile ~/.ssh/bar@foo.coopcloud.tech
+
+If you're only using password authentication, you can use the following config:
+
+Host foo.coopcloud.tech
+  Hostname foo.coopcloud.tech
+  User bar
+  Port 12345
+  PreferredAuthentications=password
+  PubkeyAuthentication=no
+
+Good luck!
+
+`

cli/internal/deploy.go

@@ -26,7 +26,7 @@ func DeployAction(c *cli.Context) error {
 		logrus.Fatal(err)
 	}
 
-	logrus.Debugf("checking whether '%s' is already deployed", stackName)
+	logrus.Debugf("checking whether %s is already deployed", stackName)
 
 	isDeployed, deployedVersion, err := stack.IsDeployed(c.Context, cl, stackName)
 	if err != nil {
@@ -34,24 +34,26 @@ func DeployAction(c *cli.Context) error {
 	}
 
 	if isDeployed {
-		if Force {
-			logrus.Warnf("'%s' already deployed but continuing (--force)", stackName)
-		} else if Chaos {
-			logrus.Warnf("'%s' already deployed but continuing (--chaos)", stackName)
+		if Force || Chaos {
+			logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", stackName)
 		} else {
-			logrus.Fatalf("'%s' is already deployed", stackName)
+			logrus.Fatalf("%s is already deployed", stackName)
 		}
 	}
 
 	version := deployedVersion
 	if version == "" && !Chaos {
-		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type)
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		if len(versions) > 0 {
 			version = versions[len(versions)-1]
-			logrus.Debugf("choosing '%s' as version to deploy", version)
+			logrus.Debugf("choosing %s as version to deploy", version)
 			if err := recipe.EnsureVersion(app.Type, version); err != nil {
 				logrus.Fatal(err)
 			}
@@ -65,7 +67,13 @@ func DeployAction(c *cli.Context) error {
 	}
 
 	if version == "" && !Chaos {
-		logrus.Debugf("choosing '%s' as version to deploy", version)
+		logrus.Debugf("choosing %s as version to deploy", version)
+		if err := recipe.EnsureVersion(app.Type, version); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	if version != "" && !Chaos {
 		if err := recipe.EnsureVersion(app.Type, version); err != nil {
 			logrus.Fatal(err)
 		}

cli/internal/logs.go

@@ -0,0 +1,68 @@
+package internal
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Stderr bool
+var StderrFlag = &cli.BoolFlag{
+	Name:        "stderr",
+	Aliases:     []string{"e"},
+	Value:       false,
+	Destination: &Stderr,
+}
+
+var Stdout bool
+var StdoutFlag = &cli.BoolFlag{
+	Name:        "stdout",
+	Aliases:     []string{"o"},
+	Value:       false,
+	Destination: &Stdout,
+}
+
+var Healthcheck bool
+var HealthcheckFlag = &cli.BoolFlag{
+	Name:        "healthcheck",
+	Aliases:     []string{"c"},
+	Value:       false,
+	Destination: &Healthcheck,
+}
+
+// StackLogs lists logs for all stack services
+func StackLogs(c *cli.Context, stackName string, logOpts types.ContainerLogsOptions, client *dockerClient.Client) {
+	filters := filters.NewArgs()
+	filters.Add("name", stackName)
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(c.Context, serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			logs, err := client.ServiceLogs(c.Context, s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			// defer after err check as any err returns a nil io.ReadCloser
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+	wg.Wait()
+	os.Exit(0)
+}

cli/internal/validate.go

@@ -27,7 +27,11 @@ func ValidateRecipe(c *cli.Context) recipe.Recipe {
 
 	recipe, err := recipe.Get(recipeName)
 	if err != nil {
-		logrus.Fatal(err)
+		if c.Command.Name == "generate" {
+			logrus.Warn(err)
+		} else {
+			logrus.Fatal(err)
+		}
 	}
 
 	logrus.Debugf("validated '%s' as recipe argument", recipeName)
@@ -108,6 +112,35 @@ func ValidateApp(c *cli.Context) config.App {
 	return app
 }
 
+// ValidateAppByName ensures the app is valid and takes an app name as an argument, not context.
+func ValidateAppByName(c *cli.Context, appName string) config.App {
+	if AppName != "" {
+		appName = AppName
+		logrus.Debugf("programmatically setting app name to %s", appName)
+	}
+
+	if appName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+	}
+
+	app, err := app.Get(appName)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := recipe.EnsureExists(app.Type); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ssh.EnsureHostKey(app.Server); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debugf("validated '%s' as app argument", appName)
+
+	return app
+}
+
 // ValidateDomain ensures the domain name arg is valid.
 func ValidateDomain(c *cli.Context) (string, error) {
 	domainName := c.Args().First()

cli/server/add.go

@@ -236,7 +236,7 @@ func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, d
 		}
 	}
 
-	logrus.Infof("docker is installed on %s", domainName)
+	logrus.Infof("docker is already installed on %s", domainName)
 
 	return nil
 }
@@ -276,7 +276,8 @@ func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error
 		AdvertiseAddr: ipv4,
 	}
 	if _, err := cl.SwarmInit(c.Context, initReq); err != nil {
-		if !strings.Contains(err.Error(), "is already part of a swarm") {
+		if !strings.Contains(err.Error(), "is already part of a swarm") ||
+			!strings.Contains(err.Error(), "must specify a listening address") {
 			return err
 		}
 		logrus.Infof("swarm mode already initialised on %s", domainName)

cli/server/logs.go

@@ -0,0 +1,101 @@
+package server
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var Taillen string
+var TaillenFlag = &cli.StringFlag{
+	Name:        "tail",
+	Aliases:     []string{"t"},
+	Value:       "5",
+	Destination: &Taillen,
+	Usage:       "change how many lines are shown",
+}
+
+var serverLogsCommand = &cli.Command{
+	Name:      "logs",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<server>",
+	Usage:     "show logs from all apps from server",
+	Flags: []cli.Flag{
+		TaillenFlag,
+		internal.StderrFlag,
+		internal.StdoutFlag,
+		internal.HealthcheckFlag,
+	},
+	Action: func(c *cli.Context) error {
+		serverName, err := internal.ValidateServer(c)
+		serviceName := ""
+		if !internal.Stderr && !internal.Stdout && !internal.Healthcheck {
+			internal.Stderr = true
+			internal.Stdout = true
+			internal.Healthcheck = true
+		}
+
+		logrus.Debugf("flags parsed. --stderr: %t, --stdout: %t, --healthcheck: %t", internal.Stderr, internal.Stdout, internal.Healthcheck)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		appMap, err := config.LoadAppFiles(serverName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logOpts := types.ContainerLogsOptions{
+			Details:    false,
+			Follow:     false,
+			ShowStderr: internal.Stderr,
+			ShowStdout: internal.Stdout,
+			Tail:       Taillen,
+			Timestamps: true,
+		}
+
+		var appFiles []config.App
+		for appname, _ := range appMap {
+			app := internal.ValidateAppByName(c, appname)
+			appFiles = append(appFiles, app)
+		}
+		for _, app := range appFiles {
+			fmt.Println(app)
+			logrus.Debugf("checking logs for: %s", app.Name)
+			cl, err := client.New(app.Server)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Debugf("tailing logs for all services")
+			filters := filters.NewArgs()
+			filters.Add("name", service)
+			serviceOpts := types.ServiceListOptions{Filters: filters}
+			services, err := cl.ServiceList(c.Context, serviceOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logs, err := cl.ServiceLogs(c.Context, services[0].ID, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			logrus.Info(app.StackName())
+			for {
+				_, err = io.Copy(os.Stdout, logs)
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+			logs.Close()
+		}
+		return nil
+	},
+}

cli/server/server.go

@@ -23,5 +23,6 @@ apps, see available flags on "server add" for more.
 		serverAddCommand,
 		serverListCommand,
 		serverRemoveCommand,
+		serverLogsCommand,
 	},
 }

go.mod

@@ -7,12 +7,12 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.2
 	github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731170023-c37c0920d1a4
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/docker/cli v20.10.10+incompatible
+	github.com/docker/cli v20.10.11+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v20.10.10+incompatible
+	github.com/docker/docker v20.10.11+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/go-git/go-git/v5 v5.4.2
-	github.com/hetznercloud/hcloud-go v1.33.0
+	github.com/hetznercloud/hcloud-go v1.33.1
 	github.com/moby/sys/signal v0.6.0
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/olekukonko/tablewriter v0.0.5
@@ -34,6 +34,7 @@ require (
 	github.com/gliderlabs/ssh v0.3.3
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/kevinburke/ssh_config v1.1.0
 	github.com/libdns/gandi v1.0.2
 	github.com/libdns/libdns v0.2.1

go.sum

@@ -260,14 +260,14 @@ github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/cli v20.10.10+incompatible h1:kcbwdgWbrBOH8QwQzaJmyriHwF7XIl4HT1qh0HTRys4=
-github.com/docker/cli v20.10.10+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc=
+github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.10+incompatible h1:GKkP0T7U4ks6X3lmmHKC2QDprnpRJor2Z5a8m62R9ZM=
-github.com/docker/docker v20.10.10+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
+github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
@@ -444,14 +444,20 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
+github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hetznercloud/hcloud-go v1.33.0 h1:cHsRgZv5JUX+I9g69KNTSUBRoPEHKgMHV38u4QKhnjQ=
-github.com/hetznercloud/hcloud-go v1.33.0/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
+github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ=
+github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
 github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ=
 github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

pkg/catalogue/catalogue.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -46,6 +45,7 @@ type features struct {
 	Image       image  `json:"image"`
 	Status      int    `json:"status"`
 	Tests       string `json:"tests"`
+	SSO         string `json:"sso"`
 }
 
 // tag represents a git tag.
@@ -122,7 +122,7 @@ func (r ByRecipeName) Less(i, j int) bool {
 // recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
 // is up to date.
 func recipeCatalogueFSIsLatest() (bool, error) {
-	httpClient := &http.Client{Timeout: web.Timeout}
+	httpClient := web.NewHTTPRetryClient()
 	res, err := httpClient.Head(RecipeCatalogueURL)
 	if err != nil {
 		return false, err
@@ -374,6 +374,127 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 	return reposMeta, nil
 }
 
+func GetStringInBetween(str, start, end string) (result string, err error) {
+	// GetStringInBetween returns empty string if no start or end string found
+	s := strings.Index(str, start)
+	if s == -1 {
+		return "", fmt.Errorf("marker string '%s' not found", start)
+	}
+	s += len(start)
+	e := strings.Index(str[s:], end)
+	if e == -1 {
+		return "", fmt.Errorf("end marker '%s' not found", end)
+	}
+	return str[s : s+e], nil
+}
+
+func GetImageMetadata(imageRowString string) (image, error) {
+	img := image{}
+
+	imgFields := strings.Split(imageRowString, ",")
+
+	for i, elem := range imgFields {
+		imgFields[i] = strings.TrimSpace(elem)
+	}
+
+	if len(imgFields) < 3 {
+		logrus.Warnf("image string has incorrect format: %s", imageRowString)
+		return img, nil
+	}
+
+	img.Rating = imgFields[1]
+	img.Source = imgFields[2]
+
+	imgString := imgFields[0]
+
+	imageName, err := GetStringInBetween(imgString, "[", "]")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	img.Image = strings.ReplaceAll(imageName, "`", "")
+
+	imageURL, err := GetStringInBetween(imgString, "(", ")")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	img.URL = imageURL
+
+	return img, nil
+}
+
+func GetRecipeFeaturesAndCategory(recipeName string) (features, string, error) {
+	feat := features{}
+
+	var category string
+
+	readmePath := path.Join(config.ABRA_DIR, "apps", recipeName, "README.md")
+
+	logrus.Debugf("attempting to open '%s'", readmePath)
+
+	readmeFS, err := ioutil.ReadFile(readmePath)
+	if err != nil {
+		return feat, category, err
+	}
+
+	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
+		string(readmeFS),
+		"<!-- metadata -->", "<!-- endmetadata -->",
+	)
+	if err != nil {
+		return feat, category, err
+	}
+
+	readmeLines := strings.Split( // Array item from lines
+		strings.ReplaceAll( // Remove \t tabs
+			readmeMetadata, "\t", "",
+		),
+		"\n")
+
+	for _, val := range readmeLines {
+		if strings.Contains(val, "**Category**") {
+			category = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Category**:"),
+			)
+		}
+		if strings.Contains(val, "**Backups**") {
+			feat.Backups = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Backups**:"),
+			)
+		}
+		if strings.Contains(val, "**Email**") {
+			feat.Email = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Email**:"),
+			)
+		}
+		if strings.Contains(val, "**SSO**") {
+			feat.SSO = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **SSO**:"),
+			)
+		}
+		if strings.Contains(val, "**Healthcheck**") {
+			feat.Healthcheck = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Healthcheck**:"),
+			)
+		}
+		if strings.Contains(val, "**Tests**") {
+			feat.Tests = strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Tests**:"),
+			)
+		}
+		if strings.Contains(val, "**Image**") {
+			imageMetadata, err := GetImageMetadata(strings.TrimSpace(
+				strings.TrimPrefix(val, "* **Image**:"),
+			))
+			if err != nil {
+				continue
+			}
+			feat.Image = imageMetadata
+		}
+	}
+
+	return feat, category, nil
+}
+
 // GetRecipeVersions retrieves all recipe versions.
 func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 	versions := RecipeVersions{}
@@ -404,7 +525,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 
 		checkOutOpts := &git.CheckoutOptions{
 			Create: false,
-			Force:  true,
+			Keep:   true,
 			Branch: plumbing.ReferenceName(ref.Name()),
 		}
 		if err := worktree.Checkout(checkOutOpts); err != nil {
@@ -432,9 +553,21 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 				path = strings.Split(path, "/")[1]
 			}
 
+			var tag string
+			switch img.(type) {
+			case reference.NamedTagged:
+				tag = img.(reference.NamedTagged).Tag()
+			case reference.Named:
+				logrus.Warnf("%s service is missing image tag?", path)
+				continue
+			}
+
+			logrus.Debugf("looking up image: '%s' from '%s'", img, path)
+
 			digest, err := client.GetTagDigest(img)
 			if err != nil {
-				return err
+				logrus.Warn(err)
+				continue
 			}
 
 			versionMeta[service.Name] = ServiceMeta{
@@ -465,7 +598,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {
@@ -480,14 +613,9 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 }
 
 // GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
-func GetRecipeCatalogueVersions(recipeName string) ([]string, error) {
+func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
 	var versions []string
 
-	catl, err := ReadRecipeCatalogue()
-	if err != nil {
-		return versions, err
-	}
-
 	if recipeMeta, exists := catl[recipeName]; exists {
 		for _, versionMeta := range recipeMeta.Versions {
 			for tag := range versionMeta {

pkg/client/registry.go

@@ -9,6 +9,7 @@ import (
 
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/docker/distribution/reference"
+	"github.com/hashicorp/go-retryablehttp"
 )
 
 type RawTag struct {
@@ -35,12 +36,12 @@ func GetRegistryTags(image string) (RawTags, error) {
 func getRegv2Token(image reference.Named) (string, error) {
 	img := reference.Path(image)
 	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
-	req, err := http.NewRequest("GET", authTokenURL, nil)
+	req, err := retryablehttp.NewRequest("GET", authTokenURL, nil)
 	if err != nil {
 		return "", err
 	}
 
-	client := &http.Client{Timeout: web.Timeout}
+	client := web.NewHTTPRetryClient()
 	res, err := client.Do(req)
 	if err != nil {
 		return "", err
@@ -78,7 +79,7 @@ func GetTagDigest(image reference.Named) (string, error) {
 	tag := image.(reference.NamedTagged).Tag()
 	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
 
-	req, err := http.NewRequest("GET", manifestURL, nil)
+	req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
 	if err != nil {
 		return "", err
 	}
@@ -96,7 +97,7 @@ func GetTagDigest(image reference.Named) (string, error) {
 		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
 	}
 
-	client := &http.Client{Timeout: web.Timeout}
+	client := web.NewHTTPRetryClient()
 	res, err := client.Do(req)
 	if err != nil {
 		return "", err

pkg/git/clone.go

@@ -47,6 +47,16 @@ func EnsureUpToDate(dir string) error {
 		return err
 	}
 
+	recipeName := filepath.Base(dir)
+	isClean, err := IsClean(recipeName)
+	if err != nil {
+		return err
+	}
+
+	if !isClean {
+		return fmt.Errorf("'%s' has locally unstaged changes", recipeName)
+	}
+
 	branch := "master"
 	if _, err := repo.Branch("master"); err != nil {
 		if _, err := repo.Branch("main"); err != nil {
@@ -66,7 +76,7 @@ func EnsureUpToDate(dir string) error {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {

pkg/git/read.go

@@ -1,11 +1,17 @@
 package git
 
 import (
+	"io/ioutil"
+	"os/user"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/go-git/go-git/v5"
+	gitConfigPkg "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,6 +46,12 @@ func IsClean(recipeName string) (bool, error) {
 		return false, err
 	}
 
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return false, err
+	}
+	worktree.Excludes = append(patterns, worktree.Excludes...)
+
 	status, err := worktree.Status()
 	if err != nil {
 		return false, err
@@ -53,3 +65,95 @@ func IsClean(recipeName string) (bool, error) {
 
 	return status.IsClean(), nil
 }
+
+// GetExcludesFiles reads the exlude files from a global git ignore
+func GetExcludesFiles() ([]gitignore.Pattern, error) {
+	var err error
+	var patterns []gitignore.Pattern
+
+	cfg, err := parseGitConfig()
+	if err != nil {
+		return patterns, err
+	}
+
+	excludesfile := getExcludesFile(cfg)
+	patterns, err = parseExcludesFile(excludesfile)
+	if err != nil {
+		return patterns, err
+	}
+
+	return patterns, nil
+}
+
+func parseGitConfig() (*gitConfigPkg.Config, error) {
+	cfg := gitConfigPkg.NewConfig()
+
+	usr, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := ioutil.ReadFile(usr.HomeDir + "/.gitconfig")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cfg.Unmarshal(b); err != nil {
+		return nil, err
+	}
+
+	return cfg, err
+}
+
+func getExcludesFile(cfg *gitConfigPkg.Config) string {
+	for _, sec := range cfg.Raw.Sections {
+		if sec.Name == "core" {
+			for _, opt := range sec.Options {
+				if opt.Key == "excludesfile" {
+					return opt.Value
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func parseExcludesFile(excludesfile string) ([]gitignore.Pattern, error) {
+	excludesfile, err := expandTilde(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadFile(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	var ps []gitignore.Pattern
+	for _, s := range strings.Split(string(data), "\n") {
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			ps = append(ps, gitignore.ParsePattern(s, nil))
+		}
+	}
+
+	return ps, nil
+}
+
+func expandTilde(path string) (string, error) {
+	if !strings.HasPrefix(path, "~") {
+		return path, nil
+	}
+	var paths []string
+	u, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+	for _, p := range strings.Split(path, string(filepath.Separator)) {
+		if p == "~" {
+			paths = append(paths, u.HomeDir)
+		} else {
+			paths = append(paths, p)
+		}
+	}
+	return "/" + filepath.Join(paths...), nil
+}

pkg/recipe/recipe.go

@@ -84,7 +84,7 @@ func Get(recipeName string) (Recipe, error) {
 	envSamplePath := path.Join(config.ABRA_DIR, "apps", recipeName, ".env.sample")
 	sampleEnv, err := config.ReadEnv(envSamplePath)
 	if err != nil {
-		logrus.Fatal(err)
+		return Recipe{}, err
 	}
 
 	opts := stack.Deploy{Composefiles: composeFiles}
@@ -157,7 +157,8 @@ func EnsureVersion(recipeName, version string) error {
 	logrus.Debugf("read '%s' as tags for recipe '%s'", strings.Join(parsedTags, ", "), recipeName)
 
 	if tagRef.String() == "" {
-		return fmt.Errorf("%s is not available?", version)
+		logrus.Warnf("%s recipe has no local tag: %s? this recipe version is not released?", recipeName, version)
+		return nil
 	}
 
 	worktree, err := repo.Worktree()
@@ -168,7 +169,7 @@ func EnsureVersion(recipeName, version string) error {
 	opts := &git.CheckoutOptions{
 		Branch: tagRef,
 		Create: false,
-		Force:  true,
+		Keep:   true,
 	}
 	if err := worktree.Checkout(opts); err != nil {
 		return err
@@ -220,7 +221,7 @@ func EnsureLatest(recipeName string) error {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 

pkg/ssh/ssh.go

@@ -330,9 +330,9 @@ func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.Publ
 
 		fmt.Printf(fmt.Sprintf(`
 You are attempting to make an SSH connection to a server but there is no entry
-in your ~/.ssh/known_hosts file which confirms that this is indeed the server
-you want to connect to. Please take a moment to validate the following SSH host
-key, it is important.
+in your ~/.ssh/known_hosts file which confirms that you have already validated
+that this is indeed the server you want to connect to. Please take a moment to
+validate the following SSH host key, it is important.
 
     Host:        %s
     Fingerprint: %s
@@ -409,12 +409,31 @@ func connect(username, host, port string, authMethod ssh.AuthMethod, timeout tim
 }
 
 func connectWithAgentTimeout(host, username, port string, timeout time.Duration) (*Client, error) {
+	logrus.Debugf("using ssh-agent to make an SSH connection for %s", host)
+
 	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
 	if err != nil {
 		return nil, err
 	}
 
-	authMethod := ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
+	agentCl := agent.NewClient(sshAgent)
+	authMethod := ssh.PublicKeysCallback(agentCl.Signers)
+
+	loadedKeys, err := agentCl.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var convertedKeys []string
+	for _, key := range loadedKeys {
+		convertedKeys = append(convertedKeys, key.String())
+	}
+
+	if len(convertedKeys) > 0 {
+		logrus.Debugf("ssh-agent has these keys loaded: %s", strings.Join(convertedKeys, ","))
+	} else {
+		logrus.Debug("ssh-agent has no keys loaded")
+	}
 
 	return connect(username, host, port, authMethod, timeout)
 }
@@ -544,11 +563,16 @@ func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 	}
 
 	idf = ssh_config.Get(hostname, "IdentityFile")
-
-	hostConfig.Host = host
 	if idf != "" {
+		var err error
+		idf, err = identityFileAbsPath(idf)
+		if err != nil {
+			return hostConfig, err
+		}
 		hostConfig.IdentityFile = idf
 	}
+
+	hostConfig.Host = host
 	hostConfig.Port = port
 	hostConfig.User = username
 
@@ -556,3 +580,25 @@ func GetHostConfig(hostname, username, port string) (HostConfig, error) {
 
 	return hostConfig, nil
 }
+
+func identityFileAbsPath(relPath string) (string, error) {
+	var err error
+	var absPath string
+
+	if strings.HasPrefix(relPath, "~/") {
+		systemUser, err := user.Current()
+		if err != nil {
+			return absPath, err
+		}
+		absPath = filepath.Join(systemUser.HomeDir, relPath[2:])
+	} else {
+		absPath, err = filepath.Abs(relPath)
+		if err != nil {
+			return absPath, err
+		}
+	}
+
+	logrus.Debugf("resolved %s to %s to read the ssh identity file", relPath, absPath)
+
+	return absPath, nil
+}

pkg/upstream/commandconn/connection.go

@@ -2,6 +2,7 @@ package commandconn
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/url"
 
@@ -34,9 +35,25 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
 		if err != nil {
 			return nil, errors.Wrap(err, "ssh host connection is not valid")
 		}
+
 		if err := sshPkg.EnsureHostKey(ctxConnDetails.Host); err != nil {
 			return nil, err
 		}
+
+		hostConfig, err := sshPkg.GetHostConfig(
+			ctxConnDetails.Host,
+			ctxConnDetails.User,
+			ctxConnDetails.Port,
+		)
+		if err != nil {
+			return nil, err
+		}
+		if hostConfig.IdentityFile != "" {
+			msg := "discovered %s as identity file for %s, using for ssh connection"
+			logrus.Debugf(msg, hostConfig.IdentityFile, ctxConnDetails.Host)
+			sshFlags = append(sshFlags, fmt.Sprintf("-o IdentityFile=%s", hostConfig.IdentityFile))
+		}
+
 		return &connhelper.ConnectionHelper{
 			Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)

pkg/upstream/stack/loader.go

@@ -13,6 +13,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// DontSkipValidation ensures validation is done for compose file loading
+func DontSkipValidation(opts *loader.Options) {
+	opts.SkipValidation = false
+}
+
 // LoadComposefile parse the composefile specified in the cli and returns its Config and version.
 func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
 	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
@@ -21,13 +26,12 @@ func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Confi
 	}
 
 	dicts := getDictsFrom(configDetails.ConfigFiles)
-	config, err := loader.Load(configDetails)
+	config, err := loader.Load(configDetails, DontSkipValidation)
 	if err != nil {
 		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
 			return nil, fmt.Errorf("compose file contains unsupported options:\n\n%s",
 				propertyWarnings(fpe.Properties))
 		}
-
 		return nil, err
 	}
 

pkg/upstream/stack/stack.go

@@ -350,7 +350,7 @@ func deployServices(
 		existingServiceMap[service.Spec.Name] = service
 	}
 
-	var serviceIDs []string
+	serviceIDs := make(map[string]string)
 	for internalName, serviceSpec := range services {
 		var (
 			name        = namespace.Scope(internalName)
@@ -410,7 +410,7 @@ func deployServices(
 				return errors.Wrapf(err, "failed to update service %s", name)
 			}
 
-			serviceIDs = append(serviceIDs, service.ID)
+			serviceIDs[service.ID] = name
 
 			for _, warning := range response.Warnings {
 				logrus.Warn(warning)
@@ -430,15 +430,19 @@ func deployServices(
 				return errors.Wrapf(err, "failed to create service %s", name)
 			}
 
-			serviceIDs = append(serviceIDs, serviceCreateResponse.ID)
+			serviceIDs[serviceCreateResponse.ID] = name
 		}
 	}
 
-	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceIDs, ", "))
+	var serviceNames []string
+	for _, serviceName := range serviceIDs {
+		serviceNames = append(serviceNames, serviceName)
+	}
+	logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
 
 	ch := make(chan error, len(serviceIDs))
-	for _, serviceID := range serviceIDs {
-		logrus.Debugf("waiting on %s to converge", serviceID)
+	for serviceID, serviceName := range serviceIDs {
+		logrus.Debugf("waiting on %s to converge", serviceName)
 		go func(s string) {
 			ch <- waitOnService(ctx, cl, s)
 		}(serviceID)

pkg/web/client.go

@@ -0,0 +1,23 @@
+package web
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/sirupsen/logrus"
+)
+
+type customLeveledLogger struct {
+	retryablehttp.Logger
+}
+
+func (l customLeveledLogger) Printf(msg string, args ...interface{}) {
+	logrus.Debugf(fmt.Sprintf(msg, args...))
+}
+
+// NewHTTPRetryClient instantiates a new http client with retries baked in
+func NewHTTPRetryClient() *retryablehttp.Client {
+	retryClient := retryablehttp.NewClient()
+	retryClient.Logger = customLeveledLogger{}
+	return retryClient
+}

pkg/web/web.go

@@ -3,7 +3,6 @@ package web
 
 import (
 	"encoding/json"
-	"net/http"
 	"time"
 )
 
@@ -13,7 +12,7 @@ const Timeout = 10 * time.Second
 
 // ReadJSON reads JSON and parses it into your chosen interface pointer
 func ReadJSON(url string, target interface{}) error {
-	httpClient := &http.Client{Timeout: Timeout}
+	httpClient := NewHTTPRetryClient()
 	res, err := httpClient.Get(url)
 	if err != nil {
 		return err

scripts/installer/installer

@@ -2,7 +2,7 @@
 
 ABRA_VERSION="0.3.0-alpha"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.3.1-alpha-rc1"
+RC_VERSION="0.3.1-alpha-rc2"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do
@@ -76,13 +76,13 @@ function install_abra_release {
       p=$HOME/.local/bin
       com="echo PATH=\$PATH:$p"
       if [[ $SHELL =~ "bash" ]]; then
-          echo "echo $com >> $HOME/.bashrc"
+          echo "$com >> $HOME/.bashrc"
       elif [[ $SHELL =~ "fizsh" ]]; then
-          echo "echo $com >> $HOME/.fizsh/.fizshrc"
+          echo "$com >> $HOME/.fizsh/.fizshrc"
       elif [[ $SHELL =~ "zsh" ]]; then
-          echo "echo $com >> $HOME/.zshrc"
+          echo "$com >> $HOME/.zshrc"
       else
-          echo "echo $com >> $HOME/.profile"
+          echo "$com >> $HOME/.profile"
       fi
   fi