kadabra, the app auto-updater #268

Merged
moritz merged 21 commits from moritz/abra:update_daemon into main 2023-02-08 18:54:06 +00:00
4 changed files with 316 additions and 0 deletions
Showing only changes of commit d894cb0ee7 - Show all commits

1
.gitignore vendored
View File

@ -3,6 +3,7 @@
.envrc
.vscode/
abra
kadabra
dist/
tests/integration/.abra/catalogue
vendor/

View File

@ -1,4 +1,5 @@
ABRA := ./cmd/abra
KADABRA := ./cmd/kadabra
moritz marked this conversation as resolved Outdated

Indentation

Indentation
COMMIT := $(shell git rev-list -1 HEAD)
GOPATH := $(shell go env GOPATH)
LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
@ -18,9 +19,11 @@ build-dev:
build:
@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
@go build -ldflags=$(DIST_LDFLAGS) $(KADABRA)
clean:
@rm '$(GOPATH)/bin/abra'
@rm '$(GOPATH)/bin/kadabra'
format:
@gofmt -s -w .

277
cli/updater/updater.go Normal file
View File

@ -0,0 +1,277 @@
package updater
import (
"context"
"fmt"
"os"
"strings"
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/upstream/convert"
"coopcloud.tech/abra/pkg/upstream/stack"
"coopcloud.tech/tagcmp"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerclient "github.com/docker/docker/client"
// "github.com/docker/cli/cli/command/stack/swarm"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
moritz marked this conversation as resolved Outdated

To remove?

To remove?
// Upgrade a specific app
var UpgradeApp = cli.Command{
Name: "appupgrade",
Aliases: []string{"a"},
Usage: "Upgrade an app",
ArgsUsage: "<stack_name> <recipe>",
Flags: []cli.Flag{
internal.DebugFlag,
internal.ForceFlag,
internal.DontWaitConvergeFlag,
},
Before: internal.SubCommandBefore,
Description: `Upgrade an app`,
Action: func(c *cli.Context) error {
stackName := c.Args().Get(0)
recipeName := c.Args().Get(1)
cl, err := client.New("default")
if err != nil {
logrus.Fatal(err)
}
upgrade(cl, stackName, recipeName)
moritz marked this conversation as resolved Outdated

If you don't add more info on top of the Usage, then maybe we can skip it?

If you don't add more info on top of the `Usage`, then maybe we can skip it?
return nil
},
}
// Upgrade all appS
var UpgradeAll = cli.Command{
Name: "upgrade",
Aliases: []string{"u"},
moritz marked this conversation as resolved Outdated

To remove? Copy/pasta is part of the Go experience 🙃

To remove? Copy/pasta is part of the Go experience 🙃
Usage: "Upgrade all apps",
Flags: []cli.Flag{
internal.DebugFlag,
internal.ForceFlag,
internal.DontWaitConvergeFlag,
},
Before: internal.SubCommandBefore,
Description: `Upgrade all deployed apps`,
Action: func(c *cli.Context) error {
cl, err := client.New("default")
if err != nil {
logrus.Fatal(err)
}
// stacks := swarm.GetStacks(cl)
stacks, err := stack.GetStacks(cl)
if err != nil {
logrus.Fatal(err)
}
for _, stackInfo := range stacks {
stackName := stackInfo.Name
recipeName := getRecipe(cl, stackName)
// TODO: read chaos from docker label
if recipeName != "" {
logrus.Debugf("RecipeName: %s", recipeName)
upgrade(cl, stackName, recipeName)
}
}
return nil
},
}
// Read recipe from docker label
func getRecipe(cl *dockerclient.Client, stackName string) string {
filter := filters.NewArgs()
filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
if err != nil {
logrus.Fatal(err)
}
for _, service := range services {
labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName)
if recipeName, ok := service.Spec.Labels[labelKey]; ok {
return recipeName
moritz marked this conversation as resolved Outdated

Could we merge this code with the appupgrade command, rename to upgrade and use a -a/--all flag to trigger upgrade for all? Then you can do some bookkeeping logic for the flag / args to make a decision on what to do? Smaller command surface seems good.

Could we merge this code with the `appupgrade` command, rename to `upgrade` and use a -a/--all flag to trigger upgrade for all? Then you can do some bookkeeping logic for the flag / args to make a decision on what to do? Smaller command surface seems good.
}
}
logrus.Debugf("no recipe name found for %s", stackName)
moritz marked this conversation as resolved Outdated

Nitpick, you could if !updateAll { ... return nil } and then avoid the else indentation on the next block.

Nitpick, you could `if !updateAll { ... return nil }` and then avoid the `else` indentation on the next block.
return ""
}
// Read Env variables from docker services
func getEnv(cl *dockerclient.Client, stackName string) config.AppEnv {
envMap := make(map[string]string)
filter := filters.NewArgs()
moritz marked this conversation as resolved Outdated

I think it may make sense to document here that we're accepting a <stack_name> and not an <app_name> because we don't have access to the ~/.abra/... files? Further context on when and where this is supposed to be run?

I think it may make sense to document here that we're accepting a `<stack_name>` and not an `<app_name>` because we don't have access to the `~/.abra/...` files? Further context on when and where this is supposed to be run?
filter.Add("label", fmt.Sprintf("%s=%s", convert.LabelNamespace, stackName))
services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filter})
if err != nil {
logrus.Fatal(err)
}
for _, service := range services {
envList := service.Spec.TaskTemplate.ContainerSpec.Env
for _, envString := range envList {
splitString := strings.SplitN(envString, "=", 2)
if len(splitString) != 2 {
logrus.Debugf("can't separate key from value: %s (this variable is probably unset)", envString)
continue
}
k := splitString[0]
v := splitString[1]
logrus.Debugf("Env Key: %s Value: %s", k, v)
envMap[k] = v
}
}
return envMap
}
func upgrade(cl *dockerclient.Client, stackName string, recipeName string) {
moritz marked this conversation as resolved Outdated

If possible it'd be good to make the difference clear in the logging, which case is it exactly?

If possible it'd be good to make the difference clear in the logging, which case is it exactly?
logrus.Debugf("Upgrade StackName: %s \n Recipe: %s", stackName, recipeName)
app := config.App{
Name: stackName,
Recipe: recipeName,
Server: "localhost",
Env: getEnv(cl, stackName),
}
// Workaround, is there a better way?
moritz marked this conversation as resolved Outdated
// getLabel ...

Same for other functions below.

``` // getLabel ... ``` Same for other functions below.
app.Env["STACK_NAME"] = stackName
// TODO: read COMPOSE_FILE from docker label
// TODO: evaluate ENABLE_AUTO_UPDATE env var
logrus.Debugf("Retrieve deployed version whether %s is already deployed", stackName)
isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)
if err != nil {
logrus.Fatal(err)
}
if !isDeployed {
logrus.Fatalf("%s is not deployed?", stackName)
}
catl, err := recipe.ReadRecipeCatalogue()
if err != nil {
logrus.Fatal(err)
}
versions, err := recipe.GetRecipeCatalogueVersions(recipeName, catl)
if err != nil {
logrus.Fatal(err)
}
moritz marked this conversation as resolved Outdated

Usually we would return "", err here and the caller handles the logrus.Fatal(err) in the command code? It easier to reason about where the err gets handled later on when we're maintaining the code. Same for the other functions.

Usually we would `return "", err` here and the caller handles the `logrus.Fatal(err)` in the command code? It easier to reason about where the `err` gets handled later on when we're maintaining the code. Same for the other functions.

Ok I applied this, so all errors are passed to the Command function. Is this the way in go? Because it's quite blowing up the code and for debeggung it seems harder for me to trace where the error originates from.

Ok I applied this, so all errors are passed to the Command function. Is this the way in go? Because it's quite blowing up the code and for debeggung it seems harder for me to trace where the error originates from.

@moritz thanks! Did you push the code? I'm unsure where the changes are.

Yeh it's pretty verbose but that's how we do it so far and Go is kinda verbose as it is... we started without this approach and then it was the reverse, it was hard to understand where the program was exiting with Fatal(...) calls... in the functions or further down? Now everything ends back in the Command code which is easier to manage?

You may want to use %w to unwrap errors? See https://stackoverflow.com/a/61287626 for more details. Then you can add a note to each error string which gets unwrapped further up the stack so that you can see exactly where the error came from. I don't really use this elsewhere in the code as I didn't even know about it at the time but will probably start doing it from now on.

@moritz thanks! Did you push the code? I'm unsure where the changes are. Yeh it's pretty verbose but that's how we do it so far and Go is kinda verbose as it is... we started without this approach and then it was the reverse, it was hard to understand where the program was exiting with `Fatal(...)` calls... in the functions or further down? Now everything ends back in the Command code which is easier to manage? You may want to use `%w` to unwrap errors? See https://stackoverflow.com/a/61287626 for more details. Then you can add a note to each error string which gets unwrapped further up the stack so that you can see exactly where the error came from. I don't really use this elsewhere in the code as I didn't even know about it at the time but will probably start doing it from now on.

Now the code is pushed.

Now the code is pushed.
if len(versions) == 0 {
logrus.Fatalf("no published releases for %s in the recipe catalogue?", recipeName)
}
var availableUpgrades []string
if deployedVersion == "unknown" {
availableUpgrades = versions
logrus.Warnf("failed to determine version of deployed %s", stackName)
}
if deployedVersion != "unknown" {
for _, version := range versions {
parsedDeployedVersion, err := tagcmp.Parse(deployedVersion)
if err != nil {
logrus.Fatal(err)
}
parsedVersion, err := tagcmp.Parse(version)
if err != nil {
logrus.Fatal(err)
}
versionDelta, err := parsedDeployedVersion.UpgradeDelta(parsedVersion)
if err != nil {
logrus.Fatal(err)
}
// Only update Patch/Minor updates
if 0 < versionDelta.UpgradeType() && versionDelta.UpgradeType() < 4 {
availableUpgrades = append(availableUpgrades, version)
}
}
if len(availableUpgrades) == 0 {
logrus.Fatalf("no available upgrades, you're on latest (%s) ✌️", deployedVersion)
}
}
moritz marked this conversation as resolved Outdated

Missing doc string?

Missing doc string?
availableUpgrades = internal.ReverseStringList(availableUpgrades)
var chosenUpgrade string
if len(availableUpgrades) > 0 {
chosenUpgrade = availableUpgrades[len(availableUpgrades)-1]
logrus.Debugf("choosing %s as version to upgrade to", chosenUpgrade)
}

To remove or still useful to keep?

To remove or still useful to keep?

Maybe there should be a way like --next to choose which version to upgrade to?
I am not sure if it's better to always perform upgrades to the next higher version or to choose the latest. Probably for minor/patch updates using the latest version should be fine.

Maybe there should be a way like `--next` to choose which version to upgrade to? I am not sure if it's better to always perform upgrades to the next higher version or to choose the latest. Probably for minor/patch updates using the latest version should be fine.
if err := recipe.EnsureExists(recipeName); err != nil {
logrus.Fatal(err)
}
if err := recipe.EnsureVersion(recipeName, chosenUpgrade); err != nil {
logrus.Fatal(err)
}
abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipeName, "abra.sh")
abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
if err != nil {
logrus.Fatal(err)
}
for k, v := range abraShEnv {
logrus.Debugf("read v:%s k: %s", v, k)
app.Env[k] = v
}
composeFiles, err := config.GetAppComposeFiles(recipeName, app.Env)
if err != nil {
logrus.Fatal(err)
}
deployOpts := stack.Deploy{
Composefiles: composeFiles,
Namespace: stackName,
Prune: false,
ResolveImage: stack.ResolveImageAlways,
}
compose, err := config.GetAppComposeConfig(stackName, deployOpts, app.Env)
if err != nil {
logrus.Fatal(err)
}
if err := stack.RunDeploy(cl, deployOpts, compose, stackName, true); err != nil {

So e.g. you can do return nil, fmt.Errorf("getAvailableUpgrades: %s", err) and then in the Command level, use the %w to unwrap the error message so you get e.g. func1: func2: func3: msg with something like https://pkg.go.dev/fmt#Errorf passed to logrus? Not sure on the exact details.

So e.g. you can do `return nil, fmt.Errorf("getAvailableUpgrades: %s", err)` and then in the Command level, use the `%w` to unwrap the error message so you get e.g. `func1: func2: func3: msg` with something like https://pkg.go.dev/fmt#Errorf passed to logrus? Not sure on the exact details.
logrus.Fatal(err)
}
}
func newAbraApp(version, commit string) *cli.App {
app := &cli.App{
Name: "kadabra",
Usage: `The Co-op Cloud update daemon
____ ____ _ _
/ ___|___ ___ _ __ / ___| | ___ _ _ __| |
| | / _ \ _____ / _ \| '_ \ | | | |/ _ \| | | |/ _' |
| |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
\____\___/ \___/| .__/ \____|_|\___/ \__,_|\__,_|
|_|
`,
Version: fmt.Sprintf("%s-%s", version, commit[:7]),
Commands: []cli.Command{
UpgradeApp,
UpgradeAll,
},
}
app.Before = func(c *cli.Context) error {
logrus.Debugf("abra version %s, commit %s", version, commit)
return nil
}
return app
}
// RunApp runs CLI abra app.
func RunApp(version, commit string) {
app := newAbraApp(version, commit)
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}
}

View File

@ -10,6 +10,7 @@ import (
"coopcloud.tech/abra/pkg/upstream/convert"
"github.com/docker/cli/cli/command/service/progress"
"github.com/docker/cli/cli/command/stack/formatter"
composetypes "github.com/docker/cli/cli/compose/types"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
@ -484,3 +485,37 @@ If a service is failing to even start, try smoke out the error with:
`, appName, timeout, appName, appName, appName))
}
}
// FIXME: Copypasta from https://github.com/docker/cli/blob/master/cli/command/stack/swarm/list.go because I could't import "github.com/docker/cli/cli/command/stack/swarm"
moritz marked this conversation as resolved Outdated

Fine to remove this imho, perhaps keep reference to where it came from but remove the FIXME, I mean?

Fine to remove this imho, perhaps keep reference to where it came from but remove the `FIXME`, I mean?
// GetStacks lists the swarm stacks.
func GetStacks(cl *dockerclient.Client) ([]*formatter.Stack, error) {
services, err := cl.ServiceList(
context.Background(),
types.ServiceListOptions{Filters: getAllStacksFilter()})
if err != nil {
return nil, err
}
m := make(map[string]*formatter.Stack)
for _, service := range services {
labels := service.Spec.Labels
name, ok := labels[convert.LabelNamespace]
if !ok {
return nil, errors.Errorf("cannot get label %s for service %s",
convert.LabelNamespace, service.ID)
}
ztack, ok := m[name]
if !ok {
m[name] = &formatter.Stack{
Name: name,
Services: 1,
}
} else {
ztack.Services++
}
}
var stacks []*formatter.Stack
for _, stack := range m {
stacks = append(stacks, stack)
}
return stacks, nil
}