feat: kadabra, the app auto-updater (!268)

coop-cloud/organising#236

Autoupdater `kadabra` is ready for testing.
It should run on the server, check for available minor/patch updates and automatically upgrade the apps.

Co-authored-by: Moritz <moritz.m@local-it.org>
Reviewed-on: #268

.gitignore

@@ -3,6 +3,7 @@
 .envrc
 .vscode/
 abra
+/kadabra
 dist/
 tests/integration/.abra/catalogue
 vendor/

.goreleaser.yml

@@ -1,5 +1,4 @@
 ---
-project_name: abra
 gitea_urls:
   api: https://git.coopcloud.tech/api/v1
   download: https://git.coopcloud.tech/
@@ -11,6 +10,26 @@ builds:
   - env:
       - CGO_ENABLED=0
     dir: cmd/abra
+    id: abra
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 5
+      - 6
+      - 7
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+  - env:
+      - CGO_ENABLED=0
+    id: kadabra
+    dir: cmd/kadabra
     goos:
       - linux
       - darwin

Makefile

@@ -1,4 +1,5 @@
 ABRA    := ./cmd/abra
+KADABRA := ./cmd/kadabra
 COMMIT  := $(shell git rev-list -1 HEAD)
 GOPATH  := $(shell go env GOPATH)
 LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
@@ -18,9 +19,11 @@ build-dev:
 
 build:
 	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
+	@go build -ldflags=$(DIST_LDFLAGS) $(KADABRA)
 
 clean:
 	@rm '$(GOPATH)/bin/abra'
+	@rm '$(GOPATH)/bin/kadabra'
 
 format:
 	@gofmt -s -w .

cli/updater/updater.go

@@ -0,0 +1,435 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/lint"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/convert"
+	"coopcloud.tech/abra/pkg/upstream/stack"
+	"coopcloud.tech/tagcmp"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerclient "github.com/docker/docker/client"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+const SERVER = "localhost"
+
+var majorUpdate bool
+var majorFlag = &cli.BoolFlag{
+	Name:        "major, m",
+	Usage:       "Also check for major updates",
+	Destination: &majorUpdate,
+}
+
+var updateAll bool
+var allFlag = &cli.BoolFlag{
+	Name:        "all, a",
+	Usage:       "Update all deployed apps",
+	Destination: &updateAll,
+}
+
+// Check for available upgrades
+var Notify = cli.Command{
+	Name:    "notify",
+	Aliases: []string{"n"},
+	Usage:   "Check for available upgrades",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		majorFlag,
+	},
+	Before:      internal.SubCommandBefore,
+	Description: `It reads the deployed app versions and looks for new versions in the recipe catalogue. If a new patch/minor version is available, a notification is printed. To include major versions use the --major flag.`,
+	Action: func(c *cli.Context) error {
+
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if recipeName != "" {
+				_, err = getLatestUpgrade(cl, stackName, recipeName)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+		return nil
+	},
+}
+
+// Upgrade apps
+var UpgradeApp = cli.Command{
+	Name:      "upgrade",
+	Aliases:   []string{"u"},
+	Usage:     "Upgrade apps",
+	ArgsUsage: "<stack_name> <recipe>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+		internal.ChaosFlag,
+		majorFlag,
+		allFlag,
+	},
+	Before:      internal.SubCommandBefore,
+	Description: `Upgrade an app by specifying its stack name and recipe. By passing --all instead every deployed app is upgraded. For each apps with enabled auto updates the deployed version is compared with the current recipe catalogue version. If a new patch/minor version is available the app is upgraded. To include major versions use the --major flag. Don't do that, it will probably break things. Only apps that are not deployed with --chaos are upgraded, to update chaos deployments use the --chaos flag. Use it with care.`,
+	Action: func(c *cli.Context) error {
+		cl, err := client.New("default")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if !updateAll {
+			stackName := c.Args().Get(0)
+			recipeName := c.Args().Get(1)
+			err = tryUpgrade(cl, stackName, recipeName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			return nil
+		}
+
+		stacks, err := stack.GetStacks(cl)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for _, stackInfo := range stacks {
+			stackName := stackInfo.Name
+			recipeName, err := getLabel(cl, stackName, "recipe")
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			err = tryUpgrade(cl, stackName, recipeName)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+		return nil
+	},
+}
+
+// getLabel reads docker label in the format coop-cloud.${STACK_NAME}.${LABEL}
+func getLabel(cl *dockerclient.Client, stackName string, label string) (string, error) {
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return "", err
+	}
+	for _, service := range services {
+		labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label)
+		if labelValue, ok := service.Spec.Labels[labelKey]; ok {
+			return labelValue, nil
+		}
+	}
+	logrus.Debugf("no %s label found for %s", label, stackName)
+	return "", nil
+}
+
+// getBoolLabel reads a boolean docker label
+func getBoolLabel(cl *dockerclient.Client, stackName string, label string) (bool, error) {
+	lableValue, err := getLabel(cl, stackName, label)
+	if err != nil {
+		return false, err
+	}
+	if lableValue != "" {
+		value, err := strconv.ParseBool(lableValue)
+		if err != nil {
+			return false, err
+		}
+		return value, nil
+	}
+	logrus.Debugf("Boolean label %s could not be found for %s, set default to false.", label, stackName)
+	return false, nil
+}
+
+// getEnv reads Env variables from docker services
+func getEnv(cl *dockerclient.Client, stackName string) (config.AppEnv, error) {
+	envMap := make(map[string]string)
+	filter := filters.NewArgs()
+	filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
+
+	services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
+	if err != nil {
+		return nil, err
+	}
+	for _, service := range services {
+		envList := service.Spec.TaskTemplate.ContainerSpec.Env
+		for _, envString := range envList {
+			splitString := strings.SplitN(envString, "=", 2)
+			if len(splitString) != 2 {
+				logrus.Debugf("can't separate key from value: %s (this variable is probably unset)", envString)
+				continue
+			}
+			k := splitString[0]
+			v := splitString[1]
+			logrus.Debugf("For %s read env %s with value: %s from docker service", stackName, k, v)
+			envMap[k] = v
+		}
+	}
+	return envMap, nil
+}
+
+// getLatestUpgrade returns the latest available version for an app regarding to the --major flag
+// if it is newer than the currently deployed version
+func getLatestUpgrade(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	deployedVersion, err := getDeployedVersion(cl, stackName, recipeName)
+	if err != nil {
+		return "", err
+	}
+	availableUpgrades, err := getAvailableUpgrades(cl, stackName, recipeName, deployedVersion)
+	if err != nil {
+		return "", err
+	}
+	if len(availableUpgrades) == 0 {
+		logrus.Debugf("no available upgrades for %s", stackName)
+		return "", nil
+	}
+	// Uncomment to select the next version instead of the last version
+	// availableUpgrades = internal.ReverseStringList(availableUpgrades)
+	var chosenUpgrade string
+	if len(availableUpgrades) > 0 {
+		chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
+		logrus.Infof("%s (%s) can be upgraded from version %s to %s", stackName, recipeName, deployedVersion, chosenUpgrade)
+	}
+	return chosenUpgrade, nil
+
+}
+
+// getDeployedVersion returns the currently deployed version of an app
+func getDeployedVersion(cl *dockerclient.Client, stackName string, recipeName string) (string, error) {
+	logrus.Debugf("Retrieve deployed version whether %s is already deployed", stackName)
+	isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
+	if err != nil {
+		return "", err
+	}
+	if !isDeployed {
+		return "", fmt.Errorf("%s is not deployed?", stackName)
+	}
+	if deployedVersion == "unknown" {
+		return "", fmt.Errorf("failed to determine deployed version of %s", stackName)
+	}
+	return deployedVersion, nil
+}
+
+// getAvailableUpgrades returns all available versions of an app that are newer than
+// the deployed version. It only includes major steps if the --major flag is set.
+func getAvailableUpgrades(cl *dockerclient.Client, stackName string, recipeName string,
+	deployedVersion string) ([]string, error) {
+	catl, err := recipe.ReadRecipeCatalogue()
+	if err != nil {
+		return nil, err
+	}
+
+	versions, err := recipe.GetRecipeCatalogueVersions(recipeName, catl)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(versions) == 0 {
+		return nil, fmt.Errorf("no published releases for %s in the recipe catalogue?", recipeName)
+	}
+
+	var availableUpgrades []string
+
+	for _, version := range versions {
+		parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
+		if err != nil {
+			return nil, err
+		}
+		parsedVersion, err := tagcmp.Parse(version)
+		if err != nil {
+			return nil, err
+		}
+		versionDelta, err := parsedDeployedVersion.UpgradeDelta(parsedVersion)
+		if err != nil {
+			return nil, err
+		}
+		if 0 < versionDelta.UpgradeType() && (versionDelta.UpgradeType() < 4 || majorUpdate) {
+			availableUpgrades = append(availableUpgrades, version)
+		}
+	}
+	logrus.Debugf("Available updates for %s: %s", stackName, availableUpgrades)
+
+	return availableUpgrades, nil
+
+}
+
+// processRecipeRepoVersion clones, pulls, checks out the version and lints the recipe repository
+func processRecipeRepoVersion(recipeName string, version string) error {
+	if err := recipe.EnsureExists(recipeName); err != nil {
+		return err
+	}
+	if err := recipe.EnsureUpToDate(recipeName); err != nil {
+		return err
+	}
+	if err := recipe.EnsureVersion(recipeName, version); err != nil {
+		return err
+	}
+	if r, err := recipe.Get(recipeName); err != nil {
+		return err
+	} else if err := lint.LintForErrors(r); err != nil {
+		return err
+	}
+	return nil
+}
+
+// mergeAbraShEnv merges abra.sh env's into the app env's
+func mergeAbraShEnv(recipeName string, env config.AppEnv) error {
+	abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
+	abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+	if err != nil {
+		return err
+	}
+	for k, v := range abraShEnv {
+		logrus.Debugf("read v:%s k: %s", v, k)
+		env[k] = v
+	}
+	return nil
+}
+
+// createDeployConfig merges and enriches the compose config for the deployment
+func createDeployConfig(recipeName string, stackName string, env config.AppEnv) (*composetypes.Config, stack.Deploy, error) {
+	env["STACK_NAME"] = stackName
+
+	deployOpts := stack.Deploy{
+		Namespace:    stackName,
+		Prune:        false,
+		ResolveImage: stack.ResolveImageAlways,
+	}
+	composeFiles, err := config.GetAppComposeFiles(recipeName, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+	deployOpts.Composefiles = composeFiles
+	compose, err := config.GetAppComposeConfig(stackName, deployOpts, env)
+	if err != nil {
+		return nil, deployOpts, err
+	}
+	config.ExposeAllEnv(stackName, compose, env)
+	// after the upgrade the deployment won't be in chaos state anymore
+	config.SetChaosLabel(compose, stackName, false)
+	config.SetRecipeLabel(compose, stackName, recipeName)
+	config.SetUpdateLabel(compose, stackName, env)
+	return compose, deployOpts, nil
+}
+
+// tryUpgrade performs the upgrade if all the requirements are fulfilled
+func tryUpgrade(cl *dockerclient.Client, stackName string, recipeName string) error {
+	if recipeName == "" {
+		logrus.Debugf("Don't update %s due to missing recipe name", stackName)
+		return nil
+	}
+	chaos, err := getBoolLabel(cl, stackName, "chaos")
+	if err != nil {
+		return err
+	}
+	if chaos && !internal.Chaos {
+		logrus.Debugf("Don't update %s due to chaos deployment.", stackName)
+		return nil
+	}
+	updatesEnabled, err := getBoolLabel(cl, stackName, "autoupdate")
+	if err != nil {
+		return err
+	}
+	if !updatesEnabled {
+		logrus.Debugf("Don't update %s due to disabling auto updates or missing ENABLE_AUTOUPDATE env.", stackName)
+		return nil
+	}
+	upgradeVersion, err := getLatestUpgrade(cl, stackName, recipeName)
+	if err != nil {
+		return err
+	}
+	if upgradeVersion == "" {
+		logrus.Debugf("Don't update %s due to no new version.", stackName)
+		return nil
+	}
+	err = upgrade(cl, stackName, recipeName, upgradeVersion)
+	return err
+
+}
+
+// upgrade performs all necessary steps to upgrade an app
+func upgrade(cl *dockerclient.Client, stackName string, recipeName string, upgradeVersion string) error {
+	env, err := getEnv(cl, stackName)
+	if err != nil {
+		return err
+	}
+	app := config.App{
+		Name:   stackName,
+		Recipe: recipeName,
+		Server: SERVER,
+		Env:    env,
+	}
+
+	if err = processRecipeRepoVersion(recipeName, upgradeVersion); err != nil {
+		return err
+	}
+
+	if err = mergeAbraShEnv(recipeName, app.Env); err != nil {
+		return err
+	}
+
+	compose, deployOpts, err := createDeployConfig(recipeName, stackName, app.Env)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("Upgrade %s (%s) to version %s", stackName, recipeName, upgradeVersion)
+	err = stack.RunDeploy(cl, deployOpts, compose, stackName, true)
+	return err
+}
+
+func newAbraApp(version, commit string) *cli.App {
+	app := &cli.App{
+		Name: "kadabra",
+		Usage: `The Co-op Cloud autoupdater
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []cli.Command{
+			Notify,
+			UpgradeApp,
+		},
+	}
+
+	app.Before = func(c *cli.Context) error {
+		logrus.Debugf("abra version %s, commit %s", version, commit)
+		return nil
+	}
+	return app
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := newAbraApp(version, commit)
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cmd/kadabra/main.go

@@ -0,0 +1,23 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli/updater"
+)
+
+// Version is the current version of Abra
+var Version string
+
+// Commit is the current git commit of Abra
+var Commit string
+
+func main() {
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	updater.RunApp(Version, Commit)
+}

pkg/recipe/recipe.go

@@ -310,8 +310,7 @@ func EnsureVersion(recipeName, version string) error {
 	logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
 
 	if tagRef.String() == "" {
-		logrus.Warnf("no published release discovered for %s", recipeName)
-		return nil
+		return fmt.Errorf("no published release discovered for %s", recipeName)
 	}
 
 	worktree, err := repo.Worktree()

pkg/upstream/stack/stack.go

@@ -10,6 +10,7 @@ import (
 
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"github.com/docker/cli/cli/command/service/progress"
+	"github.com/docker/cli/cli/command/stack/formatter"
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -484,3 +485,37 @@ If a service is failing to even start, try smoke out the error with:
 `, appName, timeout, appName, appName, appName))
 	}
 }
+
+// Copypasta from https://github.com/docker/cli/blob/master/cli/command/stack/swarm/list.go
+// GetStacks lists the swarm stacks.
+func GetStacks(cl *dockerClient.Client) ([]*formatter.Stack, error) {
+	services, err := cl.ServiceList(
+		context.Background(),
+		types.ServiceListOptions{Filters: getAllStacksFilter()})
+	if err != nil {
+		return nil, err
+	}
+	m := make(map[string]*formatter.Stack)
+	for _, service := range services {
+		labels := service.Spec.Labels
+		name, ok := labels[convert.LabelNamespace]
+		if !ok {
+			return nil, errors.Errorf("cannot get label %s for service %s",
+				convert.LabelNamespace, service.ID)
+		}
+		ztack, ok := m[name]
+		if !ok {
+			m[name] = &formatter.Stack{
+				Name:     name,
+				Services: 1,
+			}
+		} else {
+			ztack.Services++
+		}
+	}
+	var stacks []*formatter.Stack
+	for _, stack := range m {
+		stacks = append(stacks, stack)
+	}
+	return stacks, nil
+}