diff --git a/.dockerignore b/.dockerignore index d55c5bc6..e4ddfced 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,8 @@ -Dockerfile -.dockerignore -*.swp *.swo +*.swp +.dockerignore +Dockerfile +abra +dist +kadabra +tags diff --git a/.drone.yml b/.drone.yml index 26207d49..08b78ccb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,18 +7,13 @@ steps: commands: - make check - - name: make build - image: golang:1.21 - commands: - - make build - depends_on: - - make check - - name: make test image: golang:1.21 environment: ABRA_DIR: "/root/.abra" commands: + - make build-abra + - ./abra help # show version, initialise $ABRA_DIR - make test depends_on: - make check @@ -29,13 +24,12 @@ steps: - git fetch --tags depends_on: - make check - - make build - make test when: event: tag - name: release - image: goreleaser/goreleaser:v1.18.2 + image: goreleaser/goreleaser:v1.24.0 environment: GITEA_TOKEN: from_secret: goreleaser_gitea_token @@ -49,22 +43,22 @@ steps: when: event: tag - # - name: publish image - # image: plugins/docker - # settings: - # auto_tag: true - # username: 3wordchant - # password: - # from_secret: git_coopcloud_tech_token_3wc - # repo: git.coopcloud.tech/coop-cloud/abra - # tags: dev - # registry: git.coopcloud.tech - # when: - # event: - # exclude: - # - pull_request - # depends_on: - # - make check + - name: publish image + image: plugins/docker + settings: + auto_tag: true + username: 3wordchant + password: + from_secret: git_coopcloud_tech_token_3wc + repo: git.coopcloud.tech/coop-cloud/abra + tags: dev + registry: git.coopcloud.tech + when: + event: + exclude: + - pull_request + depends_on: + - make check volumes: - name: deps diff --git a/.goreleaser.yml b/.goreleaser.yml index 307505e5..24f6a009 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -51,12 +51,6 @@ builds: - "-X 'main.Commit={{ .Commit }}'" - "-X 'main.Version={{ .Version }}'" -archives: - - replacements: - 386: i386 - amd64: x86_64 - format: binary - checksum: name_template: "checksums.txt" diff --git a/AUTHORS.md b/AUTHORS.md index 6698c3cb..9e9c7620 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -11,6 +11,7 @@ - kawaiipunk - knoflook - moritz +- p4u1 - rix - roxxers - vera diff --git a/Dockerfile b/Dockerfile index 3b2db743..ad13b47a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,13 @@ +# Build image FROM golang:1.21-alpine AS build ENV GOPRIVATE coopcloud.tech -RUN apk add --no-cache make git gcc musl-dev +RUN apk add --no-cache \ + gcc \ + git \ + make \ + musl-dev COPY . /app @@ -10,7 +15,15 @@ WORKDIR /app RUN CGO_ENABLED=0 make build -FROM scratch +# Release image ("slim") +FROM alpine:3.19.1 + +RUN apk add --no-cache \ + ca-certificates \ + git \ + openssh + +RUN update-ca-certificates COPY --from=build /app/abra /abra diff --git a/Makefile b/Makefile index a1266111..7657868b 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ ABRA := ./cmd/abra KADABRA := ./cmd/kadabra COMMIT := $(shell git rev-list -1 HEAD) GOPATH := $(shell go env GOPATH) +GOVERSION := 1.21 LDFLAGS := "-X 'main.Commit=$(COMMIT)'" DIST_LDFLAGS := $(LDFLAGS)" -s -w" export GOPRIVATE=coopcloud.tech -all: format check build test +# NOTE(d1): default `make` optimised for Abra hacking +all: format check build-abra test run-abra: @go run -ldflags=$(LDFLAGS) $(ABRA) @@ -29,6 +31,12 @@ build-kadabra: build: build-abra build-kadabra +build-docker-abra: + @docker run -it -v $(PWD):/abra golang:$(GOVERSION) \ + bash -c 'cd /abra; ./scripts/docker/build.sh' + +build-docker: build-docker-abra + clean: @rm '$(GOPATH)/bin/abra' @rm '$(GOPATH)/bin/kadabra' diff --git a/cli/app/backup.go b/cli/app/backup.go index f9a239b1..f6c8d2ae 100644 --- a/cli/app/backup.go +++ b/cli/app/backup.go @@ -1,414 +1,296 @@ package app import ( - "archive/tar" - "context" "fmt" - "io" - "os" - "path/filepath" - "strings" - "time" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" - containerPkg "coopcloud.tech/abra/pkg/container" - recipePkg "coopcloud.tech/abra/pkg/recipe" - "coopcloud.tech/abra/pkg/upstream/container" - "github.com/docker/cli/cli/command" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - dockerClient "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" - "github.com/klauspost/pgzip" + "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -type backupConfig struct { - preHookCmd string - postHookCmd string - backupPaths []string +var snapshot string +var snapshotFlag = &cli.StringFlag{ + Name: "snapshot, s", + Usage: "Lists specific snapshot", + Destination: &snapshot, } -var appBackupCommand = cli.Command{ - Name: "backup", - Aliases: []string{"bk"}, - Usage: "Run app backup", - ArgsUsage: " []", +var includePath string +var includePathFlag = &cli.StringFlag{ + Name: "path, p", + Usage: "Include path", + Destination: &includePath, +} + +var resticRepo string +var resticRepoFlag = &cli.StringFlag{ + Name: "repo, r", + Usage: "Restic repository", + Destination: &resticRepo, +} + +var appBackupListCommand = cli.Command{ + Name: "list", + Aliases: []string{"ls"}, Flags: []cli.Flag{ internal.DebugFlag, internal.OfflineFlag, - internal.ChaosFlag, + snapshotFlag, + includePathFlag, }, Before: internal.SubCommandBefore, + Usage: "List all backups", BashComplete: autocomplete.AppNameComplete, - Description: ` -Run an app backup. - -A backup command and pre/post hook commands are defined in the recipe -configuration. Abra reads this configuration and run the comands in the context -of the deployed services. Pass if you only want to back up a single -service. All backups are placed in the ~/.abra/backups directory. - -A single backup file is produced for all backup paths specified for a service. -If we have the following backup configuration: - - - "backupbot.backup.path=/var/lib/foo,/var/lib/bar" - -And we run "abra app backup example.com app", Abra will produce a file that -looks like: - - ~/.abra/backups/example_com_app_609341138.tar.gz - -This file is a compressed archive which contains all backup paths. To see paths, run: - - tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz - -(Make sure to change the name of the backup file) - -This single file can be used to restore your app. See "abra app restore" for more. -`, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) - recipe, err := recipePkg.Get(app.Recipe, internal.Offline) - if err != nil { + if err := recipe.EnsureExists(app.Recipe); err != nil { logrus.Fatal(err) } if !internal.Chaos { - if err := recipePkg.EnsureIsClean(app.Recipe); err != nil { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { logrus.Fatal(err) } if !internal.Offline { - if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { logrus.Fatal(err) } } - if err := recipePkg.EnsureLatest(app.Recipe); err != nil { + if err := recipe.EnsureLatest(app.Recipe); err != nil { logrus.Fatal(err) } } - backupConfigs := make(map[string]backupConfig) - for _, service := range recipe.Config.Services { - if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok { - if backupsEnabled == "true" { - fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) - bkConfig := backupConfig{} - - logrus.Debugf("backup config detected for %s", fullServiceName) - - if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok { - logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths) - bkConfig.backupPaths = strings.Split(paths, ",") - } - - if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok { - logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) - bkConfig.preHookCmd = preHookCmd - } - - if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok { - logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) - bkConfig.postHookCmd = postHookCmd - } - - backupConfigs[service.Name] = bkConfig - } - } - } - cl, err := client.New(app.Server) if err != nil { logrus.Fatal(err) } - serviceName := c.Args().Get(1) - if serviceName != "" { - backupConfig, ok := backupConfigs[serviceName] - if !ok { - logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName) - } + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + logrus.Fatal(err) + } - logrus.Infof("running backup for the %s service", serviceName) + execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} + if snapshot != "" { + logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + if includePath != "" { + logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath) + execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath)) + } - if err := runBackup(cl, app, serviceName, backupConfig); err != nil { - logrus.Fatal(err) - } - } else { - if len(backupConfigs) == 0 { - logrus.Fatalf("no backup configs discovered for %s?", app.Name) - } - - for serviceName, backupConfig := range backupConfigs { - logrus.Infof("running backup for the %s service", serviceName) - - if err := runBackup(cl, app, serviceName, backupConfig); err != nil { - logrus.Fatal(err) - } - } + if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil { + logrus.Fatal(err) } return nil }, } -// TimeStamp generates a file name friendly timestamp. -func TimeStamp() string { - ts := time.Now().UTC().Format(time.RFC3339) - return strings.Replace(ts, ":", "-", -1) -} +var appBackupDownloadCommand = cli.Command{ + Name: "download", + Aliases: []string{"d"}, + Flags: []cli.Flag{ + internal.DebugFlag, + internal.OfflineFlag, + snapshotFlag, + includePathFlag, + }, + Before: internal.SubCommandBefore, + Usage: "Download a backup", + BashComplete: autocomplete.AppNameComplete, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) -// runBackup does the actual backup logic. -func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error { - if len(bkConfig.backupPaths) == 0 { - return fmt.Errorf("backup paths are empty for %s?", serviceName) - } - - // FIXME: avoid instantiating a new CLI - dcli, err := command.NewDockerCli() - if err != nil { - return err - } - - filters := filters.NewArgs() - filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) - - targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) - if err != nil { - return err - } - - fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) - if bkConfig.preHookCmd != "" { - splitCmd := internal.SafeSplit(bkConfig.preHookCmd) - - logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) - - preHookExecOpts := types.ExecConfig{ - AttachStderr: true, - AttachStdin: true, - AttachStdout: true, - Cmd: splitCmd, - Detach: false, - Tty: true, + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) } - if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { - return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error()) + if !internal.Chaos { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Offline { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + if err := recipe.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } } - logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd) - } - - var tempBackupPaths []string - for _, remoteBackupPath := range bkConfig.backupPaths { - sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_") - localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp())) - logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) - - logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath) - - content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath) + cl, err := client.New(app.Server) if err != nil { - logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) - if err := cleanupTempArchives(tempBackupPaths); err != nil { - return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + logrus.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + logrus.Fatal(err) + } + + execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} + if snapshot != "" { + logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + if includePath != "" { + logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath) + execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath)) + } + + if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil { + logrus.Fatal(err) + } + + remoteBackupDir := "/tmp/backup.tar.gz" + currentWorkingDir := "." + if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil { + logrus.Fatal(err) + } + + fmt.Println("backup successfully downloaded to current working directory") + + return nil + }, +} + +var appBackupCreateCommand = cli.Command{ + Name: "create", + Aliases: []string{"c"}, + Flags: []cli.Flag{ + internal.DebugFlag, + internal.OfflineFlag, + resticRepoFlag, + }, + Before: internal.SubCommandBefore, + Usage: "Create a new backup", + BashComplete: autocomplete.AppNameComplete, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) + + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Chaos { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) } - return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) - } - defer content.Close() - _, srcBase := archive.SplitPathDirEntry(remoteBackupPath) - preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath) - if err := copyToFile(localBackupPath, preArchive); err != nil { - logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) - if err := cleanupTempArchives(tempBackupPaths); err != nil { - return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + if !internal.Offline { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } } - return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) - } - tempBackupPaths = append(tempBackupPaths, localBackupPath) - } - - logrus.Infof("compressing and merging archives...") - - if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil { - logrus.Debugf("failed to merge archive files: %s", err.Error()) - if err := cleanupTempArchives(tempBackupPaths); err != nil { - return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) - } - return fmt.Errorf("failed to merge archive files: %s", err.Error()) - } - - if err := cleanupTempArchives(tempBackupPaths); err != nil { - return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) - } - - if bkConfig.postHookCmd != "" { - splitCmd := internal.SafeSplit(bkConfig.postHookCmd) - - logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) - - postHookExecOpts := types.ExecConfig{ - AttachStderr: true, - AttachStdin: true, - AttachStdout: true, - Cmd: splitCmd, - Detach: false, - Tty: true, - } - - if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { - return err - } - - logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd) - } - - return nil -} - -func copyToFile(outfile string, r io.Reader) error { - tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp") - if err != nil { - return err - } - - tmpPath := tmpFile.Name() - - _, err = io.Copy(tmpFile, r) - tmpFile.Close() - - if err != nil { - os.Remove(tmpPath) - return err - } - - if err = os.Rename(tmpPath, outfile); err != nil { - os.Remove(tmpPath) - return err - } - - return nil -} - -func cleanupTempArchives(tarPaths []string) error { - for _, tarPath := range tarPaths { - if err := os.RemoveAll(tarPath); err != nil { - return err - } - - logrus.Debugf("remove temporary archive file %s", tarPath) - } - - return nil -} - -func mergeArchives(tarPaths []string, serviceName string) error { - var out io.Writer - var cout *pgzip.Writer - - localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp())) - - fout, err := os.Create(localBackupPath) - if err != nil { - return fmt.Errorf("Failed to open %s: %s", localBackupPath, err) - } - - defer fout.Close() - out = fout - - cout = pgzip.NewWriter(out) - out = cout - - tw := tar.NewWriter(out) - - for _, tarPath := range tarPaths { - if err := addTar(tw, tarPath); err != nil { - return fmt.Errorf("failed to merge %s: %v", tarPath, err) - } - } - - if err := tw.Close(); err != nil { - return fmt.Errorf("failed to close tar writer %v", err) - } - - if cout != nil { - if err := cout.Flush(); err != nil { - return fmt.Errorf("failed to flush: %s", err) - } else if err = cout.Close(); err != nil { - return fmt.Errorf("failed to close compressed writer: %s", err) - } - } - - logrus.Infof("backed up %s to %s", serviceName, localBackupPath) - - return nil -} - -func addTar(tw *tar.Writer, pth string) (err error) { - var tr *tar.Reader - var rc io.ReadCloser - var hdr *tar.Header - - if tr, rc, err = openTarFile(pth); err != nil { - return - } - - for { - if hdr, err = tr.Next(); err != nil { - if err == io.EOF { - err = nil + if err := recipe.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) } - break } - if err = tw.WriteHeader(hdr); err != nil { - break - } else if _, err = io.Copy(tw, tr); err != nil { - break + + cl, err := client.New(app.Server) + if err != nil { + logrus.Fatal(err) } - } - if err == nil { - err = rc.Close() - } else { - rc.Close() - } - return + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + logrus.Fatal(err) + } + + execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} + if resticRepo != "" { + logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo) + execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo)) + } + + if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil { + logrus.Fatal(err) + } + + return nil + }, } -func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) { - var fin *os.File - var n int - buff := make([]byte, 1024) +var appBackupSnapshotsCommand = cli.Command{ + Name: "snapshots", + Aliases: []string{"s"}, + Flags: []cli.Flag{ + internal.DebugFlag, + internal.OfflineFlag, + snapshotFlag, + }, + Before: internal.SubCommandBefore, + Usage: "List backup snapshots", + BashComplete: autocomplete.AppNameComplete, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) - if fin, err = os.Open(pth); err != nil { - return - } + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) + } - if n, err = fin.Read(buff); err != nil { - fin.Close() - return - } else if n == 0 { - fin.Close() - err = fmt.Errorf("%s is empty", pth) - return - } + if !internal.Chaos { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } - if _, err = fin.Seek(0, 0); err != nil { - fin.Close() - return - } + if !internal.Offline { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } - rc = fin - tr = tar.NewReader(rc) + if err := recipe.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } + } - return tr, rc, nil + cl, err := client.New(app.Server) + if err != nil { + logrus.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + logrus.Fatal(err) + } + + execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} + if snapshot != "" { + logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + + if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil { + logrus.Fatal(err) + } + + return nil + }, +} + +var appBackupCommand = cli.Command{ + Name: "backup", + Aliases: []string{"b"}, + Usage: "Manage app backups", + ArgsUsage: "", + Subcommands: []cli.Command{ + appBackupListCommand, + appBackupSnapshotsCommand, + appBackupDownloadCommand, + appBackupCreateCommand, + }, } diff --git a/cli/app/check.go b/cli/app/check.go index d950fd9f..6b0e8455 100644 --- a/cli/app/check.go +++ b/cli/app/check.go @@ -1,13 +1,10 @@ package app import ( - "os" - "path" - "strings" - "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/recipe" recipePkg "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" @@ -15,9 +12,21 @@ import ( ) var appCheckCommand = cli.Command{ - Name: "check", - Aliases: []string{"chk"}, - Usage: "Check if an app is configured correctly", + Name: "check", + Aliases: []string{"chk"}, + Usage: "Ensure an app is well configured", + Description: ` +This command compares env vars in both the app ".env" and recipe ".env.sample" +file. + +The goal is to ensure that recipe ".env.sample" env vars are defined in your +app ".env" file. Only env var definitions in the ".env.sample" which are +uncommented, e.g. "FOO=bar" are checked. If an app ".env" file does not include +these env vars, then "check" will complain. + +Recipe maintainers may or may not provide defaults for env vars within their +recipes regardless of commenting or not (e.g. through the use of +${FOO:} syntax). "check" does not confirm or deny this for you.`, ArgsUsage: "", Flags: []cli.Flag{ internal.DebugFlag, @@ -49,32 +58,23 @@ var appCheckCommand = cli.Command{ } } - envSamplePath := path.Join(config.RECIPES_DIR, app.Recipe, ".env.sample") - if _, err := os.Stat(envSamplePath); err != nil { - if os.IsNotExist(err) { - logrus.Fatalf("%s does not exist?", envSamplePath) - } - logrus.Fatal(err) - } + tableCol := []string{"recipe env sample", "app env"} + table := formatter.CreateTable(tableCol) - envSample, err := config.ReadEnv(envSamplePath) + envVars, err := config.CheckEnv(app) if err != nil { logrus.Fatal(err) } - var missing []string - for k := range envSample { - if _, ok := app.Env[k]; !ok { - missing = append(missing, k) + for _, envVar := range envVars { + if envVar.Present { + table.Append([]string{envVar.Name, "✅"}) + } else { + table.Append([]string{envVar.Name, "❌"}) } } - if len(missing) > 0 { - missingEnvVars := strings.Join(missing, ", ") - logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars) - } - - logrus.Infof("all necessary environment variables defined for %s", app.Name) + table.Render() return nil }, diff --git a/cli/app/cmd.go b/cli/app/cmd.go index 47c4f54d..002eecfa 100644 --- a/cli/app/cmd.go +++ b/cli/app/cmd.go @@ -6,9 +6,11 @@ import ( "os" "os/exec" "path" + "sort" "strings" "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/app" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" @@ -22,8 +24,7 @@ var appCmdCommand = cli.Command{ Name: "command", Aliases: []string{"cmd"}, Usage: "Run app commands", - Description: ` -Run an app specific command. + Description: `Run an app specific command. These commands are bash functions, defined in the abra.sh of the recipe itself. They can be run within the context of a service (e.g. app) or locally on your @@ -43,8 +44,19 @@ Example: internal.OfflineFlag, internal.ChaosFlag, }, - BashComplete: autocomplete.AppNameComplete, - Before: internal.SubCommandBefore, + Before: internal.SubCommandBefore, + Subcommands: []cli.Command{appCmdListCommand}, + BashComplete: func(ctx *cli.Context) { + args := ctx.Args() + switch len(args) { + case 0: + autocomplete.AppNameComplete(ctx) + case 1: + autocomplete.ServiceNameComplete(args.Get(0)) + case 2: + cmdNameComplete(args.Get(0)) + } + }, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) @@ -186,3 +198,76 @@ func parseCmdArgs(args []string, isLocal bool) (bool, string) { return hasCmdArgs, parsedCmdArgs } + +func cmdNameComplete(appName string) { + app, err := app.Get(appName) + if err != nil { + return + } + cmdNames, _ := getShCmdNames(app) + if err != nil { + return + } + for _, n := range cmdNames { + fmt.Println(n) + } +} + +var appCmdListCommand = cli.Command{ + Name: "list", + Aliases: []string{"ls"}, + Usage: "List all available commands", + ArgsUsage: "", + Flags: []cli.Flag{ + internal.DebugFlag, + internal.OfflineFlag, + internal.ChaosFlag, + }, + BashComplete: autocomplete.AppNameComplete, + Before: internal.SubCommandBefore, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) + + if err := recipe.EnsureExists(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Chaos { + if err := recipePkg.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Offline { + if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + if err := recipePkg.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + cmdNames, err := getShCmdNames(app) + if err != nil { + logrus.Fatal(err) + } + + for _, cmdName := range cmdNames { + fmt.Println(cmdName) + } + + return nil + }, +} + +func getShCmdNames(app config.App) ([]string, error) { + abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, app.Recipe, "abra.sh") + cmdNames, err := config.ReadAbraShCmdNames(abraShPath) + if err != nil { + return nil, err + } + + sort.Strings(cmdNames) + return cmdNames, nil +} diff --git a/cli/app/cp.go b/cli/app/cp.go index 937c7975..f27b42ce 100644 --- a/cli/app/cp.go +++ b/cli/app/cp.go @@ -2,19 +2,24 @@ package app import ( "context" + "errors" "fmt" + "io" "os" + "path" + "path/filepath" "strings" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" - "coopcloud.tech/abra/pkg/container" + containerPkg "coopcloud.tech/abra/pkg/container" "coopcloud.tech/abra/pkg/formatter" + "coopcloud.tech/abra/pkg/upstream/container" + "github.com/docker/cli/cli/command" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" dockerClient "github.com/docker/docker/client" + "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/archive" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -49,46 +54,14 @@ And if you want to copy that file back to your current working directory locally dst := c.Args().Get(2) if src == "" { logrus.Fatal("missing argument") - } else if dst == "" { + } + if dst == "" { logrus.Fatal("missing argument") } - parsedSrc := strings.SplitN(src, ":", 2) - parsedDst := strings.SplitN(dst, ":", 2) - errorMsg := "one of / arguments must take $SERVICE:$PATH form" - if len(parsedSrc) == 2 && len(parsedDst) == 2 { - logrus.Fatal(errorMsg) - } else if len(parsedSrc) != 2 { - if len(parsedDst) != 2 { - logrus.Fatal(errorMsg) - } - } else if len(parsedDst) != 2 { - if len(parsedSrc) != 2 { - logrus.Fatal(errorMsg) - } - } - - var service string - var srcPath string - var dstPath string - isToContainer := false // - if len(parsedSrc) == 2 { - service = parsedSrc[0] - srcPath = parsedSrc[1] - dstPath = dst - logrus.Debugf("assuming transfer is coming FROM the container") - } else if len(parsedDst) == 2 { - service = parsedDst[0] - dstPath = parsedDst[1] - srcPath = src - isToContainer = true // - logrus.Debugf("assuming transfer is going TO the container") - } - - if isToContainer { - if _, err := os.Stat(srcPath); os.IsNotExist(err) { - logrus.Fatalf("%s does not exist locally?", srcPath) - } + srcPath, dstPath, service, toContainer, err := parseSrcAndDst(src, dst) + if err != nil { + logrus.Fatal(err) } cl, err := client.New(app.Server) @@ -96,7 +69,18 @@ And if you want to copy that file back to your current working directory locally logrus.Fatal(err) } - if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil { + container, err := containerPkg.GetContainerFromStackAndService(cl, app.StackName(), service) + if err != nil { + logrus.Fatal(err) + } + logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server) + + if toContainer { + err = CopyToContainer(cl, container.ID, srcPath, dstPath) + } else { + err = CopyFromContainer(cl, container.ID, srcPath, dstPath) + } + if err != nil { logrus.Fatal(err) } @@ -104,46 +88,292 @@ And if you want to copy that file back to your current working directory locally }, } -func configureAndCp( - c *cli.Context, - cl *dockerClient.Client, - app config.App, - srcPath string, - dstPath string, - service string, - isToContainer bool) error { - filters := filters.NewArgs() - filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service)) +var errServiceMissing = errors.New("one of / arguments must take $SERVICE:$PATH form") - container, err := container.GetContainer(context.Background(), cl, filters, internal.NoInput) +// parseSrcAndDst parses src and dest string. One of src or dst must be of the form $SERVICE:$PATH +func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service string, toContainer bool, err error) { + parsedSrc := strings.SplitN(src, ":", 2) + parsedDst := strings.SplitN(dst, ":", 2) + if len(parsedSrc)+len(parsedDst) != 3 { + return "", "", "", false, errServiceMissing + } + if len(parsedSrc) == 2 { + return parsedSrc[1], dst, parsedSrc[0], false, nil + } + if len(parsedDst) == 2 { + return src, parsedDst[1], parsedDst[0], true, nil + } + return "", "", "", false, errServiceMissing +} + +// CopyToContainer copies a file or directory from the local file system to the container. +// See the possible copy modes and their documentation. +func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error { + srcStat, err := os.Stat(srcPath) if err != nil { - logrus.Fatal(err) + return fmt.Errorf("local %s ", err) } - logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server) + dstStat, err := cl.ContainerStatPath(context.Background(), containerID, dstPath) + dstExists := true + if err != nil { + if errdefs.IsNotFound(err) { + dstExists = false + } else { + return fmt.Errorf("remote path: %s", err) + } + } - if isToContainer { - toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip} - content, err := archive.TarWithOptions(srcPath, toTarOpts) - if err != nil { - logrus.Fatal(err) - } + mode, err := copyMode(srcPath, dstPath, srcStat.Mode(), dstStat.Mode, dstExists) + if err != nil { + return err + } + movePath := "" + switch mode { + case CopyModeDirToDir: + // Add the src directory to the destination path + _, srcDir := path.Split(srcPath) + dstPath = path.Join(dstPath, srcDir) - copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false} - if err := cl.CopyToContainer(context.Background(), container.ID, dstPath, content, copyOpts); err != nil { - logrus.Fatal(err) - } - } else { - content, _, err := cl.CopyFromContainer(context.Background(), container.ID, srcPath) + // Make sure the dst directory exits. + dcli, err := command.NewDockerCli() if err != nil { - logrus.Fatal(err) + return err } - defer content.Close() - fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip} - if err := archive.Untar(content, dstPath, fromTarOpts); err != nil { - logrus.Fatal(err) + if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: []string{"mkdir", "-p", dstPath}, + Detach: false, + Tty: true, + }); err != nil { + return fmt.Errorf("create remote directory: %s", err) + } + case CopyModeFileToFile: + // Remove the file component from the path, since docker can only copy + // to a directory. + dstPath, _ = path.Split(dstPath) + case CopyModeFileToFileRename: + // Copy the file to the temp directory and move it to its dstPath + // afterwards. + movePath = dstPath + dstPath = "/tmp" + } + + toTarOpts := &archive.TarOptions{IncludeSourceDir: true, NoOverwriteDirNonDir: true, Compression: archive.Gzip} + content, err := archive.TarWithOptions(srcPath, toTarOpts) + if err != nil { + return err + } + + logrus.Debugf("copy %s from local to %s on container", srcPath, dstPath) + copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false} + if err := cl.CopyToContainer(context.Background(), containerID, dstPath, content, copyOpts); err != nil { + return err + } + + if movePath != "" { + _, srcFile := path.Split(srcPath) + dcli, err := command.NewDockerCli() + if err != nil { + return err + } + if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: []string{"mv", path.Join("/tmp", srcFile), movePath}, + Detach: false, + Tty: true, + }); err != nil { + return fmt.Errorf("create remote directory: %s", err) } } return nil } + +// CopyFromContainer copies a file or directory from the given container to the local file system. +// See the possible copy modes and their documentation. +func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error { + srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath) + if err != nil { + if errdefs.IsNotFound(err) { + return fmt.Errorf("remote: %s does not exist", srcPath) + } else { + return fmt.Errorf("remote path: %s", err) + } + } + + dstStat, err := os.Stat(dstPath) + dstExists := true + var dstMode os.FileMode + if err != nil { + if os.IsNotExist(err) { + dstExists = false + } else { + return fmt.Errorf("remote path: %s", err) + } + } else { + dstMode = dstStat.Mode() + } + + mode, err := copyMode(srcPath, dstPath, srcStat.Mode, dstMode, dstExists) + if err != nil { + return err + } + + moveDstDir := "" + moveDstFile := "" + switch mode { + case CopyModeFileToFile: + // Remove the file component from the path, since docker can only copy + // to a directory. + dstPath, _ = path.Split(dstPath) + case CopyModeFileToFileRename: + // Copy the file to the temp directory and move it to its dstPath + // afterwards. + moveDstFile = dstPath + dstPath = "/tmp" + case CopyModeFilesToDir: + // Copy the directory to the temp directory and move it to its + // dstPath afterwards. + moveDstDir = path.Join(dstPath, "/") + dstPath = "/tmp" + + // Make sure the temp directory always gets removed + defer os.Remove(path.Join("/tmp")) + } + + content, _, err := cl.CopyFromContainer(context.Background(), containerID, srcPath) + if err != nil { + return fmt.Errorf("copy: %s", err) + } + defer content.Close() + if err := archive.Untar(content, dstPath, &archive.TarOptions{ + NoOverwriteDirNonDir: true, + Compression: archive.Gzip, + NoLchown: true, + }); err != nil { + return fmt.Errorf("untar: %s", err) + } + + if moveDstFile != "" { + _, srcFile := path.Split(strings.TrimSuffix(srcPath, "/")) + if err := moveFile(path.Join("/tmp", srcFile), moveDstFile); err != nil { + return err + } + } + if moveDstDir != "" { + _, srcDir := path.Split(strings.TrimSuffix(srcPath, "/")) + if err := moveDir(path.Join("/tmp", srcDir), moveDstDir); err != nil { + return err + } + } + return nil +} + +var ( + ErrCopyDirToFile = fmt.Errorf("can't copy dir to file") + ErrDstDirNotExist = fmt.Errorf("destination directory does not exist") +) + +type CopyMode int + +const ( + // Copy a src file to a dest file. The src and dest file names are the same. + // / + / -> / + CopyModeFileToFile = CopyMode(iota) + // Copy a src file to a dest file. The src and dest file names are not the same. + // / + / -> / + CopyModeFileToFileRename + // Copy a src file to dest directory. The dest file gets created in the dest + // folder with the src filename. + // / + -> / + CopyModeFileToDir + // Copy a src directory to dest directory. + // + -> / + CopyModeDirToDir + // Copy all files in the src directory to the dest directory. This works recursively. + // / + -> / + CopyModeFilesToDir +) + +// copyMode takes a src and dest path and file mode to determine the copy mode. +// See the possible copy modes and their documentation. +func copyMode(srcPath, dstPath string, srcMode os.FileMode, dstMode os.FileMode, dstExists bool) (CopyMode, error) { + _, srcFile := path.Split(srcPath) + _, dstFile := path.Split(dstPath) + if srcMode.IsDir() { + if !dstExists { + return -1, ErrDstDirNotExist + } + if dstMode.IsDir() { + if strings.HasSuffix(srcPath, "/") { + return CopyModeFilesToDir, nil + } + return CopyModeDirToDir, nil + } + return -1, ErrCopyDirToFile + } + + if dstMode.IsDir() { + return CopyModeFileToDir, nil + } + + if srcFile != dstFile { + return CopyModeFileToFileRename, nil + } + + return CopyModeFileToFile, nil +} + +// moveDir moves all files from a source path to the destination path recursively. +func moveDir(sourcePath, destPath string) error { + return filepath.Walk(sourcePath, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + newPath := path.Join(destPath, strings.TrimPrefix(p, sourcePath)) + if info.IsDir() { + err := os.Mkdir(newPath, info.Mode()) + if err != nil { + if os.IsExist(err) { + return nil + } + return err + } + } + if info.Mode().IsRegular() { + return moveFile(p, newPath) + } + return nil + }) +} + +// moveFile moves a file from a source path to a destination path. +func moveFile(sourcePath, destPath string) error { + inputFile, err := os.Open(sourcePath) + if err != nil { + return err + } + outputFile, err := os.Create(destPath) + if err != nil { + inputFile.Close() + return err + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + inputFile.Close() + if err != nil { + return err + } + + // Remove file after succesfull copy. + err = os.Remove(sourcePath) + if err != nil { + return err + } + return nil +} diff --git a/cli/app/cp_test.go b/cli/app/cp_test.go new file mode 100644 index 00000000..e1fef30f --- /dev/null +++ b/cli/app/cp_test.go @@ -0,0 +1,113 @@ +package app + +import ( + "os" + "testing" +) + +func TestParse(t *testing.T) { + tests := []struct { + src string + dst string + srcPath string + dstPath string + service string + toContainer bool + err error + }{ + {src: "foo", dst: "bar", err: errServiceMissing}, + {src: "app:foo", dst: "app:bar", err: errServiceMissing}, + {src: "app:foo", dst: "bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: false}, + {src: "foo", dst: "app:bar", srcPath: "foo", dstPath: "bar", service: "app", toContainer: true}, + } + + for i, tc := range tests { + srcPath, dstPath, service, toContainer, err := parseSrcAndDst(tc.src, tc.dst) + if srcPath != tc.srcPath { + t.Errorf("[%d] srcPath: want (%s), got(%s)", i, tc.srcPath, srcPath) + } + if dstPath != tc.dstPath { + t.Errorf("[%d] dstPath: want (%s), got(%s)", i, tc.dstPath, dstPath) + } + if service != tc.service { + t.Errorf("[%d] service: want (%s), got(%s)", i, tc.service, service) + } + if toContainer != tc.toContainer { + t.Errorf("[%d] toConainer: want (%t), got(%t)", i, tc.toContainer, toContainer) + } + if err == nil && tc.err != nil && err.Error() != tc.err.Error() { + t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err) + } + } +} + +func TestCopyMode(t *testing.T) { + tests := []struct { + srcPath string + dstPath string + srcMode os.FileMode + dstMode os.FileMode + dstExists bool + mode CopyMode + err error + }{ + { + srcPath: "foo.txt", + dstPath: "foo.txt", + srcMode: os.ModePerm, + dstMode: os.ModePerm, + dstExists: true, + mode: CopyModeFileToFile, + }, + { + srcPath: "foo.txt", + dstPath: "bar.txt", + srcMode: os.ModePerm, + dstExists: true, + mode: CopyModeFileToFileRename, + }, + { + srcPath: "foo", + dstPath: "foo", + srcMode: os.ModeDir, + dstMode: os.ModeDir, + dstExists: true, + mode: CopyModeDirToDir, + }, + { + srcPath: "foo/", + dstPath: "foo", + srcMode: os.ModeDir, + dstMode: os.ModeDir, + dstExists: true, + mode: CopyModeFilesToDir, + }, + { + srcPath: "foo", + dstPath: "foo", + srcMode: os.ModeDir, + dstExists: false, + mode: -1, + err: ErrDstDirNotExist, + }, + { + srcPath: "foo", + dstPath: "foo", + srcMode: os.ModeDir, + dstMode: os.ModePerm, + dstExists: true, + mode: -1, + err: ErrCopyDirToFile, + }, + } + + for i, tc := range tests { + mode, err := copyMode(tc.srcPath, tc.dstPath, tc.srcMode, tc.dstMode, tc.dstExists) + if mode != tc.mode { + t.Errorf("[%d] mode: want (%d), got(%d)", i, tc.mode, mode) + } + if err != tc.err { + t.Errorf("[%d] err: want (%s), got(%s)", i, tc.err, err) + } + } +} diff --git a/cli/app/deploy.go b/cli/app/deploy.go index 259fba73..edf8532f 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -6,6 +6,7 @@ import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/secret" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" @@ -50,6 +51,11 @@ recipes. app := internal.ValidateApp(c) stackName := app.StackName() + specificVersion := c.Args().Get(1) + if specificVersion != "" && internal.Chaos { + logrus.Fatal("cannot use and --chaos together") + } + if err := recipe.EnsureExists(app.Recipe); err != nil { logrus.Fatal(err) } @@ -91,6 +97,17 @@ recipes. logrus.Fatal(err) } + secStats, err := secret.PollSecretsStatus(cl, app) + if err != nil { + logrus.Fatal(err) + } + + for _, secStat := range secStats { + if !secStat.CreatedOnRemote { + logrus.Fatalf("unable to deploy, secrets not generated (%s)?", secStat.LocalName) + } + } + if isDeployed { if internal.Force || internal.Chaos { logrus.Warnf("%s is already deployed but continuing (--force/--chaos)", app.Name) @@ -100,7 +117,6 @@ recipes. } version := deployedVersion - specificVersion := c.Args().Get(1) if specificVersion != "" { version = specificVersion logrus.Debugf("choosing %s as version to deploy", version) @@ -188,6 +204,17 @@ recipes. config.SetChaosVersionLabel(compose, stackName, version) config.SetUpdateLabel(compose, stackName, app.Env) + envVars, err := config.CheckEnv(app) + if err != nil { + logrus.Fatal(err) + } + + for _, envVar := range envVars { + if !envVar.Present { + logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain) + } + } + if err := internal.DeployOverview(app, version, "continue with deployment?"); err != nil { logrus.Fatal(err) } diff --git a/cli/app/logs.go b/cli/app/logs.go index b1566f3f..b06634a2 100644 --- a/cli/app/logs.go +++ b/cli/app/logs.go @@ -2,75 +2,26 @@ package app import ( "context" - "fmt" "io" "os" + "slices" "sync" + "time" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" - "coopcloud.tech/abra/pkg/service" "coopcloud.tech/abra/pkg/upstream/stack" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/swarm" dockerClient "github.com/docker/docker/client" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -var logOpts = types.ContainerLogsOptions{ - ShowStderr: true, - ShowStdout: true, - Since: "", - Until: "", - Timestamps: true, - Follow: true, - Tail: "20", - Details: false, -} - -// stackLogs lists logs for all stack services -func stackLogs(c *cli.Context, app config.App, client *dockerClient.Client) { - filters, err := app.Filters(true, false) - if err != nil { - logrus.Fatal(err) - } - - serviceOpts := types.ServiceListOptions{Filters: filters} - services, err := client.ServiceList(context.Background(), serviceOpts) - if err != nil { - logrus.Fatal(err) - } - - var wg sync.WaitGroup - for _, service := range services { - wg.Add(1) - go func(s string) { - if internal.StdErrOnly { - logOpts.ShowStdout = false - } - - logs, err := client.ServiceLogs(context.Background(), s, logOpts) - if err != nil { - logrus.Fatal(err) - } - defer logs.Close() - - _, err = io.Copy(os.Stdout, logs) - if err != nil && err != io.EOF { - logrus.Fatal(err) - } - }(service.ID) - } - - wg.Wait() - - os.Exit(0) -} - var appLogsCommand = cli.Command{ Name: "logs", Aliases: []string{"l"}, @@ -105,46 +56,84 @@ var appLogsCommand = cli.Command{ logrus.Fatalf("%s is not deployed?", app.Name) } - logOpts.Since = internal.SinceLogs - serviceName := c.Args().Get(1) - if serviceName == "" { - logrus.Debugf("tailing logs for all %s services", app.Recipe) - stackLogs(c, app, cl) - } else { - logrus.Debugf("tailing logs for %s", serviceName) - if err := tailServiceLogs(c, cl, app, serviceName); err != nil { - logrus.Fatal(err) - } + serviceNames := []string{} + if serviceName != "" { + serviceNames = []string{serviceName} + } + err = tailLogs(cl, app, serviceNames) + if err != nil { + logrus.Fatal(err) } return nil }, } -func tailServiceLogs(c *cli.Context, cl *dockerClient.Client, app config.App, serviceName string) error { - filters := filters.NewArgs() - filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName)) - - chosenService, err := service.GetService(context.Background(), cl, filters, internal.NoInput) +// tailLogs prints logs for the given app with optional service names to be +// filtered on. It also checks if the latest task is not runnning and then +// prints the past tasks. +func tailLogs(cl *dockerClient.Client, app config.App, serviceNames []string) error { + f, err := app.Filters(true, false, serviceNames...) if err != nil { - logrus.Fatal(err) + return err } - if internal.StdErrOnly { - logOpts.ShowStdout = false - } - - logs, err := cl.ServiceLogs(context.Background(), chosenService.ID, logOpts) + services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: f}) if err != nil { - logrus.Fatal(err) + return err } - defer logs.Close() - _, err = io.Copy(os.Stdout, logs) - if err != nil && err != io.EOF { - logrus.Fatal(err) + var wg sync.WaitGroup + for _, service := range services { + filters := filters.NewArgs() + filters.Add("name", service.Spec.Name) + tasks, err := cl.TaskList(context.Background(), types.TaskListOptions{Filters: f}) + if err != nil { + return err + } + if len(tasks) > 0 { + // Need to sort the tasks by the CreatedAt field in the inverse order. + // Otherwise they are in the reversed order and not sorted properly. + slices.SortFunc[[]swarm.Task](tasks, func(t1, t2 swarm.Task) int { + return int(t2.Meta.CreatedAt.Unix() - t1.Meta.CreatedAt.Unix()) + }) + lastTask := tasks[0].Status + if lastTask.State != swarm.TaskStateRunning { + for _, task := range tasks { + logrus.Errorf("[%s] %s State %s: %s", service.Spec.Name, task.Meta.CreatedAt.Format(time.RFC3339), task.Status.State, task.Status.Err) + } + } + } + + // Collect the logs in a go routine, so the logs from all services are + // collected in parallel. + wg.Add(1) + go func(serviceID string) { + logs, err := cl.ServiceLogs(context.Background(), serviceID, types.ContainerLogsOptions{ + ShowStderr: true, + ShowStdout: !internal.StdErrOnly, + Since: internal.SinceLogs, + Until: "", + Timestamps: true, + Follow: true, + Tail: "20", + Details: false, + }) + if err != nil { + logrus.Fatal(err) + } + defer logs.Close() + + _, err = io.Copy(os.Stdout, logs) + if err != nil && err != io.EOF { + logrus.Fatal(err) + } + }(service.ID) } + // Wait for all log streams to be closed. + wg.Wait() + return nil } diff --git a/cli/app/new.go b/cli/app/new.go index cfa40b65..5511ef67 100644 --- a/cli/app/new.go +++ b/cli/app/new.go @@ -2,6 +2,7 @@ package app import ( "fmt" + "path" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" @@ -53,9 +54,17 @@ var appNewCommand = cli.Command{ internal.OfflineFlag, internal.ChaosFlag, }, - Before: internal.SubCommandBefore, - ArgsUsage: "[]", - BashComplete: autocomplete.RecipeNameComplete, + Before: internal.SubCommandBefore, + ArgsUsage: "[] []", + BashComplete: func(ctx *cli.Context) { + args := ctx.Args() + switch len(args) { + case 0: + autocomplete.RecipeNameComplete(ctx) + case 1: + autocomplete.RecipeVersionComplete(ctx.Args().Get(0)) + } + }, Action: func(c *cli.Context) error { recipe := internal.ValidateRecipe(c) @@ -68,8 +77,14 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } } - if err := recipePkg.EnsureLatest(recipe.Name); err != nil { - logrus.Fatal(err) + if c.Args().Get(1) == "" { + if err := recipePkg.EnsureLatest(recipe.Name); err != nil { + logrus.Fatal(err) + } + } else { + if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil { + logrus.Fatal(err) + } } } @@ -106,7 +121,8 @@ var appNewCommand = cli.Command{ logrus.Fatal(err) } - secretsConfig, err := secret.ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name) + envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample") + secretsConfig, err := secret.ReadSecretsConfig(envSamplePath, composeFiles, config.StackName(internal.Domain)) if err != nil { return err } @@ -170,7 +186,7 @@ var appNewCommand = cli.Command{ type AppSecrets map[string]string // createSecrets creates all secrets for a new app. -func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) { +func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.Secret, sanitisedAppName string) (AppSecrets, error) { // NOTE(d1): trim to match app.StackName() implementation if len(sanitisedAppName) > config.MAX_SANITISED_APP_NAME_LENGTH { logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:config.MAX_SANITISED_APP_NAME_LENGTH]) @@ -219,7 +235,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error { } // promptForSecrets asks if we should generate secrets for a new app. -func promptForSecrets(recipeName string, secretsConfig map[string]string) error { +func promptForSecrets(recipeName string, secretsConfig map[string]secret.Secret) error { if len(secretsConfig) == 0 { logrus.Debugf("%s has no secrets to generate, skipping...", recipeName) return nil diff --git a/cli/app/remove.go b/cli/app/remove.go index ea4efedf..f9167d82 100644 --- a/cli/app/remove.go +++ b/cli/app/remove.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "log" "os" "coopcloud.tech/abra/cli/internal" @@ -11,7 +12,6 @@ import ( stack "coopcloud.tech/abra/pkg/upstream/stack" "github.com/AlecAivazis/survey/v2" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/volume" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -110,26 +110,19 @@ flag. logrus.Fatal(err) } - volumeListOptions := volume.ListOptions{fs} - volumeListOKBody, err := cl.VolumeList(context.Background(), volumeListOptions) - volumeList := volumeListOKBody.Volumes + volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, fs) if err != nil { logrus.Fatal(err) } + volumeNames := client.GetVolumeNames(volumeList) - var vols []string - for _, vol := range volumeList { - vols = append(vols, vol.Name) - } - - if len(vols) > 0 { - for _, vol := range vols { - err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing - if err != nil { - logrus.Fatal(err) - } - logrus.Info(fmt.Sprintf("volume %s removed", vol)) + if len(volumeNames) > 0 { + err := client.RemoveVolumes(cl, context.Background(), volumeNames, internal.Force, 5) + if err != nil { + log.Fatalf("removing volumes failed: %s", err) } + + logrus.Infof("%d volumes removed successfully", len(volumeNames)) } else { logrus.Info("no volumes to remove") } diff --git a/cli/app/restore.go b/cli/app/restore.go index 1bf9c840..c80347f5 100644 --- a/cli/app/restore.go +++ b/cli/app/restore.go @@ -1,223 +1,82 @@ package app import ( - "context" - "errors" "fmt" - "os" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" - containerPkg "coopcloud.tech/abra/pkg/container" "coopcloud.tech/abra/pkg/recipe" - recipePkg "coopcloud.tech/abra/pkg/recipe" - "coopcloud.tech/abra/pkg/upstream/container" - "github.com/docker/cli/cli/command" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - dockerClient "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -type restoreConfig struct { - preHookCmd string - postHookCmd string +var targetPath string +var targetPathFlag = &cli.StringFlag{ + Name: "target, t", + Usage: "Target path", + Destination: &targetPath, } var appRestoreCommand = cli.Command{ Name: "restore", Aliases: []string{"rs"}, - Usage: "Run app restore", - ArgsUsage: " ", + Usage: "Restore an app backup", + ArgsUsage: " ", Flags: []cli.Flag{ internal.DebugFlag, internal.OfflineFlag, - internal.ChaosFlag, + targetPathFlag, }, Before: internal.SubCommandBefore, BashComplete: autocomplete.AppNameComplete, - Description: ` -Run an app restore. - -Pre/post hook commands are defined in the recipe configuration. Abra reads this -configuration and run the comands in the context of the service before -restoring the backup. - -Unlike "abra app backup", restore must be run on a per-service basis. You can -not restore all services in one go. Backup files produced by Abra are -compressed archives which use absolute paths. This allows Abra to restore -according to standard tar command logic, i.e. the backup will be restored to -the path it was originally backed up from. - -Example: - - abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz -`, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) - recipe, err := recipe.Get(app.Recipe, internal.Offline) - if err != nil { + if err := recipe.EnsureExists(app.Recipe); err != nil { logrus.Fatal(err) } if !internal.Chaos { - if err := recipePkg.EnsureIsClean(app.Recipe); err != nil { + if err := recipe.EnsureIsClean(app.Recipe); err != nil { logrus.Fatal(err) } if !internal.Offline { - if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil { + if err := recipe.EnsureUpToDate(app.Recipe); err != nil { logrus.Fatal(err) } } - if err := recipePkg.EnsureLatest(app.Recipe); err != nil { + if err := recipe.EnsureLatest(app.Recipe); err != nil { logrus.Fatal(err) } } - serviceName := c.Args().Get(1) - if serviceName == "" { - internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) - } - - backupPath := c.Args().Get(2) - if backupPath == "" { - internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) - } - - if _, err := os.Stat(backupPath); err != nil { - if os.IsNotExist(err) { - logrus.Fatalf("%s doesn't exist?", backupPath) - } - } - - restoreConfigs := make(map[string]restoreConfig) - for _, service := range recipe.Config.Services { - if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok { - if restoreEnabled == "true" { - fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) - rsConfig := restoreConfig{} - - logrus.Debugf("restore config detected for %s", fullServiceName) - - if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok { - logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) - rsConfig.preHookCmd = preHookCmd - } - - if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok { - logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) - rsConfig.postHookCmd = postHookCmd - } - - restoreConfigs[service.Name] = rsConfig - } - } - } - - rsConfig, ok := restoreConfigs[serviceName] - if !ok { - rsConfig = restoreConfig{} - } - cl, err := client.New(app.Server) if err != nil { logrus.Fatal(err) } - if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil { + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + logrus.Fatal(err) + } + + execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} + if snapshot != "" { + logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + if targetPath != "" { + logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath) + execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath)) + } + + if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil { logrus.Fatal(err) } return nil }, } - -// runRestore does the actual restore logic. -func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error { - // FIXME: avoid instantiating a new CLI - dcli, err := command.NewDockerCli() - if err != nil { - return err - } - - filters := filters.NewArgs() - filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) - - targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) - if err != nil { - return err - } - - fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) - if rsConfig.preHookCmd != "" { - splitCmd := internal.SafeSplit(rsConfig.preHookCmd) - - logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) - - preHookExecOpts := types.ExecConfig{ - AttachStderr: true, - AttachStdin: true, - AttachStdout: true, - Cmd: splitCmd, - Detach: false, - Tty: true, - } - - if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { - return err - } - - logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd) - } - - backupReader, err := os.Open(backupPath) - if err != nil { - return err - } - - content, err := archive.DecompressStream(backupReader) - if err != nil { - return err - } - - // NOTE(d1): we use absolute paths so tar knows what to do. it will restore - // files according to the paths set in the compressed archive - restorePath := "/" - - copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false} - if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil { - return err - } - - logrus.Infof("restored %s to %s", backupPath, fullServiceName) - - if rsConfig.postHookCmd != "" { - splitCmd := internal.SafeSplit(rsConfig.postHookCmd) - - logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) - - postHookExecOpts := types.ExecConfig{ - AttachStderr: true, - AttachStdin: true, - AttachStdout: true, - Cmd: splitCmd, - Detach: false, - Tty: true, - } - - if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { - return err - } - - logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd) - } - - return nil -} diff --git a/cli/app/rollback.go b/cli/app/rollback.go index c8104877..f39d114b 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -51,6 +51,11 @@ recipes. app := internal.ValidateApp(c) stackName := app.StackName() + specificVersion := c.Args().Get(1) + if specificVersion != "" && internal.Chaos { + logrus.Fatal("cannot use and --chaos together") + } + if err := recipe.EnsureExists(app.Recipe); err != nil { logrus.Fatal(err) } @@ -125,7 +130,6 @@ recipes. logrus.Warnf("failed to determine deployed version of %s", app.Name) } - specificVersion := c.Args().Get(1) if specificVersion != "" { parsedDeployedVersion, err := tagcmp.Parse(deployedVersion) if err != nil { diff --git a/cli/app/run.go b/cli/app/run.go index 4ae68c1b..b5e0a9ce 100644 --- a/cli/app/run.go +++ b/cli/app/run.go @@ -91,7 +91,7 @@ var appRunCommand = cli.Command{ logrus.Fatal(err) } - if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { + if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { logrus.Fatal(err) } diff --git a/cli/app/secret.go b/cli/app/secret.go index 67643937..5f782f46 100644 --- a/cli/app/secret.go +++ b/cli/app/secret.go @@ -20,19 +20,23 @@ import ( "github.com/urfave/cli" ) -var allSecrets bool -var allSecretsFlag = &cli.BoolFlag{ - Name: "all, a", - Destination: &allSecrets, - Usage: "Generate all secrets", -} +var ( + allSecrets bool + allSecretsFlag = &cli.BoolFlag{ + Name: "all, a", + Destination: &allSecrets, + Usage: "Generate all secrets", + } +) -var rmAllSecrets bool -var rmAllSecretsFlag = &cli.BoolFlag{ - Name: "all, a", - Destination: &rmAllSecrets, - Usage: "Remove all secrets", -} +var ( + rmAllSecrets bool + rmAllSecretsFlag = &cli.BoolFlag{ + Name: "all, a", + Destination: &rmAllSecrets, + Usage: "Remove all secrets", + } +) var appSecretGenerateCommand = cli.Command{ Name: "generate", @@ -87,7 +91,7 @@ var appSecretGenerateCommand = cli.Command{ logrus.Fatal(err) } - secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe) + secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName()) if err != nil { logrus.Fatal(err) } @@ -102,17 +106,14 @@ var appSecretGenerateCommand = cli.Command{ } else { secretName := c.Args().Get(1) secretVersion := c.Args().Get(2) - matches := false - for name := range secretsConfig { - if secretName == name { - secretsToCreate[name] = secretVersion - matches = true - } - } - - if !matches { + s, ok := secrets[secretName] + if !ok { logrus.Fatalf("%s doesn't exist in the env config?", secretName) } + s.Version = secretVersion + secrets = map[string]secret.Secret{ + secretName: s, + } } cl, err := client.New(app.Server) @@ -120,7 +121,7 @@ var appSecretGenerateCommand = cli.Command{ logrus.Fatal(err) } - secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server) + secretVals, err := secret.GenerateSecrets(cl, secrets, app.Server) if err != nil { logrus.Fatal(err) } @@ -280,7 +281,7 @@ Example: logrus.Fatal(err) } - secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe) + secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.StackName()) if err != nil { logrus.Fatal(err) } @@ -315,12 +316,7 @@ Example: match := false secretToRm := c.Args().Get(1) - for secretName, secretValue := range secretsConfig { - val, err := secret.ParseSecretValue(secretValue) - if err != nil { - logrus.Fatal(err) - } - + for secretName, val := range secrets { secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) if _, ok := remoteSecretNames[secretRemoteName]; ok { if secretToRm != "" { @@ -360,6 +356,7 @@ var appSecretLsCommand = cli.Command{ internal.DebugFlag, internal.OfflineFlag, internal.ChaosFlag, + internal.MachineReadableFlag, }, Before: internal.SubCommandBefore, Usage: "List all secrets", @@ -387,12 +384,7 @@ var appSecretLsCommand = cli.Command{ } } - composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) - if err != nil { - logrus.Fatal(err) - } - - secretsConfig, err := secret.ReadSecretsConfig(app.Env, composeFiles, app.Recipe) + cl, err := client.New(app.Server) if err != nil { logrus.Fatal(err) } @@ -400,42 +392,27 @@ var appSecretLsCommand = cli.Command{ tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"} table := formatter.CreateTable(tableCol) - cl, err := client.New(app.Server) + secStats, err := secret.PollSecretsStatus(cl, app) if err != nil { logrus.Fatal(err) } - filters, err := app.Filters(false, false) - if err != nil { - logrus.Fatal(err) - } - - secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters}) - if err != nil { - logrus.Fatal(err) - } - - remoteSecretNames := make(map[string]bool) - for _, cont := range secretList { - remoteSecretNames[cont.Spec.Annotations.Name] = true - } - - for secretName, secretValue := range secretsConfig { - createdRemote := false - val, err := secret.ParseSecretValue(secretValue) - if err != nil { - logrus.Fatal(err) + for _, secStat := range secStats { + tableRow := []string{ + secStat.LocalName, + secStat.Version, + secStat.RemoteName, + strconv.FormatBool(secStat.CreatedOnRemote), } - secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) - if _, ok := remoteSecretNames[secretRemoteName]; ok { - createdRemote = true - } - tableRow := []string{secretName, val.Version, secretRemoteName, strconv.FormatBool(createdRemote)} table.Append(tableRow) } if table.NumLines() > 0 { - table.Render() + if internal.MachineReadable { + table.JSONRender() + } else { + table.Render() + } } else { logrus.Warnf("no secrets stored for %s", app.Name) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index 845c15fd..8c10337b 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -56,6 +56,11 @@ recipes. app := internal.ValidateApp(c) stackName := app.StackName() + specificVersion := c.Args().Get(1) + if specificVersion != "" && internal.Chaos { + logrus.Fatal("cannot use and --chaos together") + } + if !internal.Chaos { if err := recipe.EnsureIsClean(app.Recipe); err != nil { logrus.Fatal(err) @@ -126,7 +131,6 @@ recipes. logrus.Warnf("failed to determine deployed version of %s", app.Name) } - specificVersion := c.Args().Get(1) if specificVersion != "" { parsedDeployedVersion, err := tagcmp.Parse(deployedVersion) if err != nil { @@ -254,6 +258,17 @@ recipes. config.SetChaosVersionLabel(compose, stackName, chosenUpgrade) config.SetUpdateLabel(compose, stackName, app.Env) + envVars, err := config.CheckEnv(app) + if err != nil { + logrus.Fatal(err) + } + + for _, envVar := range envVars { + if !envVar.Present { + logrus.Warnf("env var %s missing from %s.env, present in recipe .env.sample", envVar.Name, app.Domain) + } + } + if err := internal.NewVersionOverview(app, deployedVersion, chosenUpgrade, releaseNotes); err != nil { logrus.Fatal(err) } diff --git a/cli/app/volume.go b/cli/app/volume.go index b64ce69c..4fb72c6b 100644 --- a/cli/app/volume.go +++ b/cli/app/volume.go @@ -2,6 +2,7 @@ package app import ( "context" + "log" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" @@ -131,12 +132,12 @@ Passing "--force/-f" will select all volumes for removal. Be careful. } if len(volumesToRemove) > 0 { - err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force) + err := client.RemoveVolumes(cl, context.Background(), volumesToRemove, internal.Force, 5) if err != nil { - logrus.Fatal(err) + log.Fatalf("removing volumes failed: %s", err) } - logrus.Info("volumes removed successfully") + logrus.Infof("%d volumes removed successfully", len(volumesToRemove)) } else { logrus.Info("no volumes removed") } diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go index 7c97372f..b7d09ccc 100644 --- a/cli/catalogue/catalogue.go +++ b/cli/catalogue/catalogue.go @@ -98,11 +98,6 @@ keys configured on your account. continue } - if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists { - catlBar.Add(1) - continue - } - versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline) if err != nil { logrus.Warn(err) @@ -173,7 +168,7 @@ keys configured on your account. } msg := "chore: publish new catalogue release changes" - if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil { + if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil { logrus.Fatal(err) } diff --git a/cli/internal/backup.go b/cli/internal/backup.go index 79951810..530735c9 100644 --- a/cli/internal/backup.go +++ b/cli/internal/backup.go @@ -1,35 +1,67 @@ package internal import ( - "strings" + "context" + + "coopcloud.tech/abra/pkg/config" + containerPkg "coopcloud.tech/abra/pkg/container" + "coopcloud.tech/abra/pkg/service" + "coopcloud.tech/abra/pkg/upstream/container" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + dockerClient "github.com/docker/docker/client" + "github.com/sirupsen/logrus" ) -// SafeSplit splits up a string into a list of commands safely. -func SafeSplit(s string) []string { - split := strings.Split(s, " ") - - var result []string - var inquote string - var block string - for _, i := range split { - if inquote == "" { - if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") { - inquote = string(i[0]) - block = strings.TrimPrefix(i, inquote) + " " - } else { - result = append(result, i) - } - } else { - if !strings.HasSuffix(i, inquote) { - block += i + " " - } else { - block += strings.TrimSuffix(i, inquote) - inquote = "" - result = append(result, block) - block = "" - } - } +// RetrieveBackupBotContainer gets the deployed backupbot container. +func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) { + ctx := context.Background() + chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput) + if err != nil { + return types.Container{}, err } - return result + logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name) + + filters := filters.NewArgs() + filters.Add("name", chosenService.Spec.Name) + targetContainer, err := containerPkg.GetContainer( + ctx, + cl, + filters, + NoInput, + ) + if err != nil { + return types.Container{}, err + } + + return targetContainer, nil +} + +// RunBackupCmdRemote runs a backup related command on a remote backupbot container. +func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error { + execBackupListOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: []string{"/usr/bin/backup", "--", backupCmd}, + Detach: false, + Env: execEnv, + Tty: true, + } + + logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts) + + // FIXME: avoid instantiating a new CLI + dcli, err := command.NewDockerCli() + if err != nil { + return err + } + + if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil { + return err + } + + return nil } diff --git a/cli/internal/cli.go b/cli/internal/cli.go index 07e16ac5..b7555bdc 100644 --- a/cli/internal/cli.go +++ b/cli/internal/cli.go @@ -54,7 +54,7 @@ var Chaos bool // ChaosFlag turns on/off chaos functionality. var ChaosFlag = &cli.BoolFlag{ Name: "chaos, C", - Usage: "Deploy uncommitted recipes changes. Use with care!", + Usage: "Proceed with uncommitted recipes changes. Use with care!", Destination: &Chaos, } diff --git a/cli/internal/command.go b/cli/internal/command.go index 6f02ae1c..13c007be 100644 --- a/cli/internal/command.go +++ b/cli/internal/command.go @@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, Tty: false, } - if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { + if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name) shell = "/bin/sh" } @@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, execCreateOpts.Tty = false } - if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { + if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { return err } diff --git a/cli/recipe/diff.go b/cli/recipe/diff.go new file mode 100644 index 00000000..f505bbe6 --- /dev/null +++ b/cli/recipe/diff.go @@ -0,0 +1,40 @@ +package recipe + +import ( + "path" + + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/config" + gitPkg "coopcloud.tech/abra/pkg/git" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var recipeDiffCommand = cli.Command{ + Name: "diff", + Usage: "Show unstaged changes in recipe config", + Description: "Due to limitations in our underlying Git dependency, this command requires /usr/bin/git.", + Aliases: []string{"d"}, + ArgsUsage: "", + Flags: []cli.Flag{ + internal.DebugFlag, + internal.NoInputFlag, + }, + Before: internal.SubCommandBefore, + BashComplete: autocomplete.RecipeNameComplete, + Action: func(c *cli.Context) error { + recipeName := c.Args().First() + + if recipeName != "" { + internal.ValidateRecipe(c) + } + + recipeDir := path.Join(config.RECIPES_DIR, recipeName) + if err := gitPkg.DiffUnstaged(recipeDir); err != nil { + logrus.Fatal(err) + } + + return nil + }, +} diff --git a/cli/recipe/fetch.go b/cli/recipe/fetch.go index acfc1708..de81cfec 100644 --- a/cli/recipe/fetch.go +++ b/cli/recipe/fetch.go @@ -3,6 +3,7 @@ package recipe import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -17,26 +18,31 @@ var recipeFetchCommand = cli.Command{ Flags: []cli.Flag{ internal.DebugFlag, internal.NoInputFlag, + internal.OfflineFlag, }, Before: internal.SubCommandBefore, BashComplete: autocomplete.RecipeNameComplete, Action: func(c *cli.Context) error { recipeName := c.Args().First() - if recipeName != "" { internal.ValidateRecipe(c) + if err := recipe.Ensure(recipeName); err != nil { + logrus.Fatal(err) + } + return nil } - if err := recipe.EnsureExists(recipeName); err != nil { + catalogue, err := recipe.ReadRecipeCatalogue(internal.Offline) + if err != nil { logrus.Fatal(err) } - if err := recipe.EnsureUpToDate(recipeName); err != nil { - logrus.Fatal(err) - } - - if err := recipe.EnsureLatest(recipeName); err != nil { - logrus.Fatal(err) + catlBar := formatter.CreateProgressbar(len(catalogue), "fetching latest recipes...") + for recipeName := range catalogue { + if err := recipe.Ensure(recipeName); err != nil { + logrus.Error(err) + } + catlBar.Add(1) } return nil diff --git a/cli/recipe/lint.go b/cli/recipe/lint.go index a01d3ecc..542164f9 100644 --- a/cli/recipe/lint.go +++ b/cli/recipe/lint.go @@ -68,7 +68,7 @@ var recipeLintCommand = cli.Command{ skippedOutput := "-" if skipped { - skippedOutput = "yes" + skippedOutput = "✅" } satisfied := false @@ -87,9 +87,9 @@ var recipeLintCommand = cli.Command{ } } - satisfiedOutput := "yes" + satisfiedOutput := "✅" if !satisfied { - satisfiedOutput = "NO" + satisfiedOutput = "❌" if skipped { satisfiedOutput = "-" } diff --git a/cli/recipe/recipe.go b/cli/recipe/recipe.go index 8333f0f0..ad3b6365 100644 --- a/cli/recipe/recipe.go +++ b/cli/recipe/recipe.go @@ -30,5 +30,7 @@ manner. Abra supports convenient automation for recipe maintainenace, see the recipeSyncCommand, recipeUpgradeCommand, recipeVersionCommand, + recipeResetCommand, + recipeDiffCommand, }, } diff --git a/cli/recipe/release.go b/cli/recipe/release.go index 3ce13ac6..d2da8230 100644 --- a/cli/recipe/release.go +++ b/cli/recipe/release.go @@ -1,7 +1,9 @@ package recipe import ( + "errors" "fmt" + "os" "path" "strconv" "strings" @@ -106,6 +108,18 @@ your SSH keys configured on your account. } } + isClean, err := gitPkg.IsClean(recipe.Dir()) + if err != nil { + logrus.Fatal(err) + } + + if !isClean { + logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name) + if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil { + logrus.Fatal(err) + } + } + if len(tags) > 0 { logrus.Warnf("previous git tags detected, assuming this is a new semver release") if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil { @@ -128,7 +142,7 @@ your SSH keys configured on your account. // getImageVersions retrieves image versions for a recipe func getImageVersions(recipe recipe.Recipe) (map[string]string, error) { - var services = make(map[string]string) + services := make(map[string]string) missingTag := false for _, service := range recipe.Config.Services { @@ -195,6 +209,10 @@ func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string tagString = fmt.Sprintf("%s+%s", tag.String(), mainAppVersion) } + if err := addReleaseNotes(recipe, tagString); err != nil { + logrus.Fatal(err) + } + if err := commitRelease(recipe, tagString); err != nil { logrus.Fatal(err) } @@ -225,6 +243,82 @@ func getTagCreateOptions(tag string) (git.CreateTagOptions, error) { return git.CreateTagOptions{Message: msg}, nil } +// addReleaseNotes checks if the release/next release note exists and moves the +// file to release/. +func addReleaseNotes(recipe recipe.Recipe, tag string) error { + repoPath := path.Join(config.RECIPES_DIR, recipe.Name) + tagReleaseNotePath := path.Join(repoPath, "release", tag) + if _, err := os.Stat(tagReleaseNotePath); err == nil { + // Release note for current tag already exist exists. + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + nextReleaseNotePath := path.Join(repoPath, "release", "next") + if _, err := os.Stat(nextReleaseNotePath); err == nil { + // release/next note exists. Move it to release/ + if internal.Dry { + logrus.Debugf("dry run: move release note from 'next' to %s", tag) + return nil + } + if !internal.NoInput { + prompt := &survey.Input{ + Message: "Use release note in release/next?", + } + var addReleaseNote bool + if err := survey.AskOne(prompt, &addReleaseNote); err != nil { + return err + } + if !addReleaseNote { + return nil + } + } + err := os.Rename(nextReleaseNotePath, tagReleaseNotePath) + if err != nil { + return err + } + err = gitPkg.Add(repoPath, path.Join("release", "next"), internal.Dry) + if err != nil { + return err + } + err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry) + if err != nil { + return err + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + // No release note exists for the current release. + if internal.NoInput { + return nil + } + + prompt := &survey.Input{ + Message: "Release Note (leave empty for no release note)", + } + var releaseNote string + if err := survey.AskOne(prompt, &releaseNote); err != nil { + return err + } + + if releaseNote == "" { + return nil + } + + err := os.WriteFile(tagReleaseNotePath, []byte(releaseNote), 0o644) + if err != nil { + return err + } + err = gitPkg.Add(repoPath, path.Join("release", tag), internal.Dry) + if err != nil { + return err + } + + return nil +} + func commitRelease(recipe recipe.Recipe, tag string) error { if internal.Dry { logrus.Debugf("dry run: no changes committed") @@ -244,7 +338,7 @@ func commitRelease(recipe recipe.Recipe, tag string) error { msg := fmt.Sprintf("chore: publish %s release", tag) repoPath := path.Join(config.RECIPES_DIR, recipe.Name) - if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil { + if err := gitPkg.Commit(repoPath, msg, internal.Dry); err != nil { return err } @@ -392,6 +486,10 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip } } + if err := addReleaseNotes(recipe, tagString); err != nil { + logrus.Fatal(err) + } + if err := commitRelease(recipe, tagString); err != nil { logrus.Fatalf("failed to commit changes: %s", err.Error()) } diff --git a/cli/recipe/reset.go b/cli/recipe/reset.go new file mode 100644 index 00000000..67208cdc --- /dev/null +++ b/cli/recipe/reset.go @@ -0,0 +1,56 @@ +package recipe + +import ( + "path" + + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/config" + "github.com/go-git/go-git/v5" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var recipeResetCommand = cli.Command{ + Name: "reset", + Usage: "Remove all unstaged changes from recipe config", + Description: "WARNING, this will delete your changes. Be Careful.", + Aliases: []string{"rs"}, + ArgsUsage: "", + Flags: []cli.Flag{ + internal.DebugFlag, + internal.NoInputFlag, + }, + Before: internal.SubCommandBefore, + BashComplete: autocomplete.RecipeNameComplete, + Action: func(c *cli.Context) error { + recipeName := c.Args().First() + + if recipeName != "" { + internal.ValidateRecipe(c) + } + + repoPath := path.Join(config.RECIPES_DIR, recipeName) + repo, err := git.PlainOpen(repoPath) + if err != nil { + logrus.Fatal(err) + } + + ref, err := repo.Head() + if err != nil { + logrus.Fatal(err) + } + + worktree, err := repo.Worktree() + if err != nil { + logrus.Fatal(err) + } + + opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset} + if err := worktree.Reset(opts); err != nil { + logrus.Fatal(err) + } + + return nil + }, +} diff --git a/cli/recipe/sync.go b/cli/recipe/sync.go index 1085a193..d5b4ad00 100644 --- a/cli/recipe/sync.go +++ b/cli/recipe/sync.go @@ -8,6 +8,7 @@ import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/config" + gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/tagcmp" "github.com/AlecAivazis/survey/v2" "github.com/go-git/go-git/v5" @@ -198,6 +199,17 @@ likely to change. logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name) } + isClean, err := gitPkg.IsClean(recipe.Dir()) + if err != nil { + logrus.Fatal(err) + } + if !isClean { + logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name) + if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil { + logrus.Fatal(err) + } + } + return nil }, } diff --git a/cli/recipe/upgrade.go b/cli/recipe/upgrade.go index aa258f16..4a4a33e2 100644 --- a/cli/recipe/upgrade.go +++ b/cli/recipe/upgrade.go @@ -14,6 +14,7 @@ import ( "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" + gitPkg "coopcloud.tech/abra/pkg/git" recipePkg "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" "github.com/AlecAivazis/survey/v2" @@ -326,6 +327,7 @@ You may invoke this command in "wizard" mode and be prompted for input: } fmt.Println(string(jsonstring)) + return nil } @@ -336,6 +338,18 @@ You may invoke this command in "wizard" mode and be prompted for input: } } } + + isClean, err := gitPkg.IsClean(recipeDir) + if err != nil { + logrus.Fatal(err) + } + if !isClean { + logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name) + if err := gitPkg.DiffUnstaged(recipeDir); err != nil { + logrus.Fatal(err) + } + } + return nil }, } diff --git a/go.mod b/go.mod index 7f03c92d..950841f8 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,23 @@ module coopcloud.tech/abra -go 1.18 +go 1.21 require ( coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52 + git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355 github.com/AlecAivazis/survey/v2 v2.3.7 - github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4 - github.com/docker/cli v24.0.6+incompatible + github.com/docker/cli v24.0.7+incompatible github.com/docker/distribution v2.8.3+incompatible - github.com/docker/docker v24.0.6+incompatible + github.com/docker/docker v24.0.7+incompatible github.com/docker/go-units v0.5.0 - github.com/go-git/go-git/v5 v5.9.0 + github.com/go-git/go-git/v5 v5.10.0 + github.com/google/go-cmp v0.5.9 github.com/moby/sys/signal v0.7.0 github.com/moby/term v0.5.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/schollz/progressbar/v3 v3.13.1 + github.com/schollz/progressbar/v3 v3.14.1 github.com/sirupsen/logrus v1.9.3 gotest.tools/v3 v3.5.1 ) @@ -47,7 +48,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -56,7 +56,7 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.14.2 // indirect github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect @@ -71,18 +71,18 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/skeema/knownhosts v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect - golang.org/x/crypto v0.13.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.15.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/term v0.14.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect golang.org/x/tools v0.13.0 // indirect @@ -104,7 +104,7 @@ require ( github.com/fvbommel/sortorder v1.0.2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 + github.com/hashicorp/go-retryablehttp v0.7.5 github.com/klauspost/pgzip v1.2.6 github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect @@ -116,5 +116,5 @@ require ( github.com/theupdateframework/notary v0.7.0 // indirect github.com/urfave/cli v1.22.9 github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect - golang.org/x/sys v0.12.0 + golang.org/x/sys v0.14.0 ) diff --git a/go.sum b/go.sum index 59d700a8..d767d61e 100644 --- a/go.sum +++ b/go.sum @@ -51,12 +51,12 @@ coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355 h1:tCv2B4qoN6RMheKDnCzIafOkWS5BB1h7hwhmo+9bVeE= +git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg= -github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -124,6 +124,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -132,6 +133,7 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -313,6 +315,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= @@ -336,16 +339,16 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= +github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= @@ -371,6 +374,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -408,13 +412,15 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= -github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ= +github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -584,8 +590,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -665,6 +671,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -698,8 +705,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= @@ -784,6 +791,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -877,12 +885,14 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -891,8 +901,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= -github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= +github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= +github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= @@ -1060,8 +1070,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1159,8 +1169,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1301,21 +1311,20 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1575,6 +1584,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pkg/autocomplete/autocomplete.go b/pkg/autocomplete/autocomplete.go index 807422fb..7b28f134 100644 --- a/pkg/autocomplete/autocomplete.go +++ b/pkg/autocomplete/autocomplete.go @@ -25,6 +25,16 @@ func AppNameComplete(c *cli.Context) { } } +func ServiceNameComplete(appName string) { + serviceNames, err := config.GetAppServiceNames(appName) + if err != nil { + return + } + for _, s := range serviceNames { + fmt.Println(s) + } +} + // RecipeNameComplete completes recipe names. func RecipeNameComplete(c *cli.Context) { catl, err := recipe.ReadRecipeCatalogue(false) @@ -41,6 +51,20 @@ func RecipeNameComplete(c *cli.Context) { } } +// RecipeVersionComplete completes versions for the recipe. +func RecipeVersionComplete(recipeName string) { + catl, err := recipe.ReadRecipeCatalogue(false) + if err != nil { + logrus.Warn(err) + } + + for _, v := range catl[recipeName].Versions { + for v2 := range v { + fmt.Println(v2) + } + } +} + // ServerNameComplete completes server names. func ServerNameComplete(c *cli.Context) { files, err := config.LoadAppFiles("") diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go index 1b108670..80e54885 100644 --- a/pkg/catalogue/catalogue.go +++ b/pkg/catalogue/catalogue.go @@ -12,46 +12,6 @@ import ( "github.com/sirupsen/logrus" ) -// CatalogueSkipList is all the repos that are not recipes. -var CatalogueSkipList = map[string]bool{ - "abra": true, - "abra-apps": true, - "abra-aur": true, - "abra-bash": true, - "abra-capsul": true, - "abra-gandi": true, - "abra-hetzner": true, - "abra-integration-test-recipe": true, - "apps": true, - "aur-abra-git": true, - "auto-mirror": true, - "auto-recipes-catalogue-json": true, - "backup-bot": true, - "backup-bot-two": true, - "beta.coopcloud.tech": true, - "comrade-renovate-bot": true, - "coopcloud.tech": true, - "coturn": true, - "docker-cp-deploy": true, - "docker-dind-bats-kcov": true, - "docs.coopcloud.tech": true, - "drone-abra": true, - "example": true, - "gardening": true, - "go-abra": true, - "organising": true, - "pyabra": true, - "radicle-seed-node": true, - "recipes-catalogue-json": true, - "recipes-wishlist": true, - "recipes.coopcloud.tech": true, - "stack-ssh-deploy": true, - "swarm-cronjob": true, - "tagcmp": true, - "traefik-cert-dumper": true, - "tyop": true, -} - // EnsureCatalogue ensures that the catalogue is cloned locally & present. func EnsureCatalogue() error { catalogueDir := path.Join(config.ABRA_DIR, "catalogue") diff --git a/pkg/client/volumes.go b/pkg/client/volumes.go index 4afd60c2..f295b081 100644 --- a/pkg/client/volumes.go +++ b/pkg/client/volumes.go @@ -2,15 +2,17 @@ package client import ( "context" + "fmt" + "time" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" + "github.com/sirupsen/logrus" ) func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*volume.Volume, error) { - volumeListOptions := volume.ListOptions{fs} - volumeListOKBody, err := cl.VolumeList(ctx, volumeListOptions) + volumeListOKBody, err := cl.VolumeList(ctx, volume.ListOptions{Filters: fs}) volumeList := volumeListOKBody.Volumes if err != nil { return volumeList, err @@ -29,13 +31,32 @@ func GetVolumeNames(volumes []*volume.Volume) []string { return volumeNames } -func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error { +func RemoveVolumes(cl *client.Client, ctx context.Context, volumeNames []string, force bool, retries int) error { for _, volName := range volumeNames { - err := cl.VolumeRemove(ctx, volName, force) + err := retryFunc(5, func() error { + return cl.VolumeRemove(context.Background(), volName, force) + }) if err != nil { - return err + return fmt.Errorf("volume %s: %s", volName, err) } } - return nil } + +// retryFunc retries the given function for the given retries. After the nth +// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0). +// It returns an error if the function still failed after the last retry. +func retryFunc(retries int, fn func() error) error { + for i := 0; i < retries; i++ { + err := fn() + if err == nil { + return nil + } + if i+1 < retries { + sleep := time.Duration(i+1) * time.Duration(i+1) + logrus.Infof("%s: waiting %d seconds before next retry", err, sleep) + time.Sleep(sleep * time.Second) + } + } + return fmt.Errorf("%d retries failed", retries) +} diff --git a/pkg/client/volumes_test.go b/pkg/client/volumes_test.go new file mode 100644 index 00000000..792af002 --- /dev/null +++ b/pkg/client/volumes_test.go @@ -0,0 +1,26 @@ +package client + +import ( + "fmt" + "testing" +) + +func TestRetryFunc(t *testing.T) { + err := retryFunc(1, func() error { return nil }) + if err != nil { + t.Errorf("should not return an error: %s", err) + } + + i := 0 + fn := func() error { + i++ + return fmt.Errorf("oh no, something went wrong!") + } + err = retryFunc(2, fn) + if err == nil { + t.Error("should return an error") + } + if i != 2 { + t.Errorf("The function should have been called 1 times, got %d", i) + } +} diff --git a/pkg/config/app.go b/pkg/config/app.go index 15e7d972..601d0636 100644 --- a/pkg/config/app.go +++ b/pkg/config/app.go @@ -25,6 +25,9 @@ import ( // AppEnv is a map of the values in an apps env config type AppEnv = map[string]string +// AppModifiers is a map of modifiers in an apps env config +type AppModifiers = map[string]map[string]string + // AppName is AppName type AppName = string @@ -47,34 +50,61 @@ type App struct { Path string } -// StackName gets whatever the docker safe (uses the right delimiting -// character, e.g. "_") stack name is for the app. In general, you don't want -// to use this to show anything to end-users, you want use a.Name instead. +// See documentation of config.StackName func (a App) StackName() string { if _, exists := a.Env["STACK_NAME"]; exists { return a.Env["STACK_NAME"] } - stackName := SanitiseAppName(a.Name) - - if len(stackName) > MAX_SANITISED_APP_NAME_LENGTH { - logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:MAX_SANITISED_APP_NAME_LENGTH]) - stackName = stackName[:MAX_SANITISED_APP_NAME_LENGTH] - } + stackName := StackName(a.Name) a.Env["STACK_NAME"] = stackName return stackName } -// Filters retrieves exact app filters for querying the container runtime. Due -// to upstream issues, filtering works different depending on what you're +// StackName gets whatever the docker safe (uses the right delimiting +// character, e.g. "_") stack name is for the app. In general, you don't want +// to use this to show anything to end-users, you want use a.Name instead. +func StackName(appName string) string { + stackName := SanitiseAppName(appName) + + if len(stackName) > MAX_SANITISED_APP_NAME_LENGTH { + logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:MAX_SANITISED_APP_NAME_LENGTH]) + stackName = stackName[:MAX_SANITISED_APP_NAME_LENGTH] + } + + return stackName +} + +// Filters retrieves app filters for querying the container runtime. By default +// it filters on all services in the app. It is also possible to pass an +// otional list of service names, which get filtered instead. +// +// Due to upstream issues, filtering works different depending on what you're // querying. So, for example, secrets don't work with regex! The caller needs // to implement their own validation that the right secrets are matched. In // order to handle these cases, we provide the `appendServiceNames` / // `exactMatch` modifiers. -func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) { +func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) { filters := filters.NewArgs() + if len(services) > 0 { + for _, serviceName := range services { + filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch)) + } + return filters, nil + } + + // When not appending the service name, just add one filter for the whole + // stack. + if !appendServiceNames { + f := fmt.Sprintf("%s", a.StackName()) + if exactMatch { + f = fmt.Sprintf("^%s", f) + } + filters.Add("name", f) + return filters, nil + } composeFiles, err := GetComposeFiles(a.Recipe, a.Env) if err != nil { @@ -88,28 +118,23 @@ func (a App) Filters(appendServiceNames, exactMatch bool) (filters.Args, error) } for _, service := range compose.Services { - var filter string - - if appendServiceNames { - if exactMatch { - filter = fmt.Sprintf("^%s_%s", a.StackName(), service.Name) - } else { - filter = fmt.Sprintf("%s_%s", a.StackName(), service.Name) - } - } else { - if exactMatch { - filter = fmt.Sprintf("^%s", a.StackName()) - } else { - filter = fmt.Sprintf("%s", a.StackName()) - } - } - - filters.Add("name", filter) + f := ServiceFilter(a.StackName(), service.Name, exactMatch) + filters.Add("name", f) } return filters, nil } +// ServiceFilter creates a filter string for filtering a service in the docker +// container runtime. When exact match is true, it uses regex to match the +// string exactly. +func ServiceFilter(stack, service string, exact bool) string { + if exact { + return fmt.Sprintf("^%s_%s", stack, service) + } + return fmt.Sprintf("%s_%s", stack, service) +} + // ByServer sort a slice of Apps type ByServer []App @@ -149,7 +174,7 @@ func (a ByName) Less(i, j int) bool { return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name) } -func readAppEnvFile(appFile AppFile, name AppName) (App, error) { +func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) { env, err := ReadEnv(appFile.Path) if err != nil { return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error()) @@ -157,7 +182,7 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) { logrus.Debugf("read env %s from %s", env, appFile.Path) - app, err := newApp(env, name, appFile) + app, err := NewApp(env, name, appFile) if err != nil { return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error()) } @@ -165,8 +190,8 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) { return app, nil } -// newApp creates new App object -func newApp(env AppEnv, name string, appFile AppFile) (App, error) { +// NewApp creates new App object +func NewApp(env AppEnv, name string, appFile AppFile) (App, error) { domain := env["DOMAIN"] recipe, exists := env["RECIPE"] @@ -232,7 +257,7 @@ func GetApp(apps AppFiles, name AppName) (App, error) { return App{}, fmt.Errorf("cannot find app with name %s", name) } - app, err := readAppEnvFile(appFile, name) + app, err := ReadAppEnvFile(appFile, name) if err != nil { return App{}, err } @@ -330,7 +355,7 @@ func TemplateAppEnvSample(recipeName, appName, server, domain string) error { return fmt.Errorf("%s already exists?", appEnvPath) } - err = ioutil.WriteFile(appEnvPath, envSample, 0664) + err = ioutil.WriteFile(appEnvPath, envSample, 0o664) if err != nil { return err } @@ -437,27 +462,56 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str return statuses, nil } +// ensurePathExists ensures that a path exists. +func ensurePathExists(path string) error { + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { + return err + } + return nil +} + // GetComposeFiles gets the list of compose files for an app (or recipe if you // don't already have an app) which should be merged into a composetypes.Config // while respecting the COMPOSE_FILE env var. func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) { var composeFiles []string - if _, ok := appEnv["COMPOSE_FILE"]; !ok { - logrus.Debug("no COMPOSE_FILE detected, loading compose.yml") + composeFileEnvVar, ok := appEnv["COMPOSE_FILE"] + if !ok { path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe) + if err := ensurePathExists(path); err != nil { + return composeFiles, err + } + logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path) composeFiles = append(composeFiles, path) return composeFiles, nil } - composeFileEnvVar := appEnv["COMPOSE_FILE"] - envVars := strings.Split(composeFileEnvVar, ":") - logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", ")) - for _, file := range strings.Split(composeFileEnvVar, ":") { + if !strings.Contains(composeFileEnvVar, ":") { + path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar) + if err := ensurePathExists(path); err != nil { + return composeFiles, err + } + logrus.Debugf("COMPOSE_FILE detected, loading %s", path) + composeFiles = append(composeFiles, path) + return composeFiles, nil + } + + numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1 + envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles) + if len(envVars) != numComposeFiles { + return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar) + } + + for _, file := range envVars { path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file) + if err := ensurePathExists(path); err != nil { + return composeFiles, err + } composeFiles = append(composeFiles, path) } + logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", ")) logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe) return composeFiles, nil @@ -563,7 +617,7 @@ func GetLabel(compose *composetypes.Config, stackName string, label string) stri // GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) { - var timeout = 50 // Default Timeout + timeout := 50 // Default Timeout var err error = nil if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" { logrus.Debugf("timeout label: %s", timeoutLabel) diff --git a/pkg/config/app_test.go b/pkg/config/app_test.go index 113d130b..0823f4e2 100644 --- a/pkg/config/app_test.go +++ b/pkg/config/app_test.go @@ -1,36 +1,197 @@ -package config +package config_test import ( + "encoding/json" + "fmt" "reflect" "testing" + + "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" + "github.com/docker/docker/api/types/filters" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" ) func TestNewApp(t *testing.T) { - app, err := newApp(expectedAppEnv, appName, expectedAppFile) + app, err := config.NewApp(ExpectedAppEnv, AppName, ExpectedAppFile) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(app, expectedApp) { - t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp) + if !reflect.DeepEqual(app, ExpectedApp) { + t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp) } } func TestReadAppEnvFile(t *testing.T) { - app, err := readAppEnvFile(expectedAppFile, appName) + app, err := config.ReadAppEnvFile(ExpectedAppFile, AppName) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(app, expectedApp) { - t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp) + if !reflect.DeepEqual(app, ExpectedApp) { + t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp) } } func TestGetApp(t *testing.T) { - app, err := GetApp(expectedAppFiles, appName) + app, err := config.GetApp(ExpectedAppFiles, AppName) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(app, expectedApp) { - t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp) + if !reflect.DeepEqual(app, ExpectedApp) { + t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, ExpectedApp) + } +} + +func TestGetComposeFiles(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + appEnv map[string]string + composeFiles []string + }{ + { + map[string]string{}, + []string{ + fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name), + }, + }, + { + map[string]string{"COMPOSE_FILE": "compose.yml"}, + []string{ + fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name), + }, + }, + { + map[string]string{"COMPOSE_FILE": "compose.extra_secret.yml"}, + []string{ + fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name), + }, + }, + { + map[string]string{"COMPOSE_FILE": "compose.yml:compose.extra_secret.yml"}, + []string{ + fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, r.Name), + fmt.Sprintf("%s/%s/compose.extra_secret.yml", config.RECIPES_DIR, r.Name), + }, + }, + } + + for _, test := range tests { + composeFiles, err := config.GetComposeFiles(r.Name, test.appEnv) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, composeFiles, test.composeFiles) + } +} + +func TestGetComposeFilesError(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + tests := []struct{ appEnv map[string]string }{ + {map[string]string{"COMPOSE_FILE": "compose.yml::compose.foo.yml"}}, + {map[string]string{"COMPOSE_FILE": "doesnt.exist.yml"}}, + } + + for _, test := range tests { + _, err := config.GetComposeFiles(r.Name, test.appEnv) + if err == nil { + t.Fatalf("should have failed: %v", test.appEnv) + } + } +} + +func TestFilters(t *testing.T) { + oldDir := config.RECIPES_DIR + config.RECIPES_DIR = "./testdir" + defer func() { + config.RECIPES_DIR = oldDir + }() + + app, err := config.NewApp(config.AppEnv{ + "DOMAIN": "test.example.com", + "RECIPE": "test-recipe", + }, "test_example_com", config.AppFile{ + Path: "./testdir/filtertest.end", + Server: "local", + }) + if err != nil { + t.Fatal(err) + } + + f, err := app.Filters(false, false) + if err != nil { + t.Error(err) + } + compareFilter(t, f, map[string]map[string]bool{ + "name": { + "test_example_com": true, + }, + }) + + f2, err := app.Filters(false, true) + if err != nil { + t.Error(err) + } + compareFilter(t, f2, map[string]map[string]bool{ + "name": { + "^test_example_com": true, + }, + }) + + f3, err := app.Filters(true, false) + if err != nil { + t.Error(err) + } + compareFilter(t, f3, map[string]map[string]bool{ + "name": { + "test_example_com_bar": true, + "test_example_com_foo": true, + }, + }) + + f4, err := app.Filters(true, true) + if err != nil { + t.Error(err) + } + compareFilter(t, f4, map[string]map[string]bool{ + "name": { + "^test_example_com_bar": true, + "^test_example_com_foo": true, + }, + }) + + f5, err := app.Filters(false, false, "foo") + if err != nil { + t.Error(err) + } + compareFilter(t, f5, map[string]map[string]bool{ + "name": { + "test_example_com_foo": true, + }, + }) +} + +func compareFilter(t *testing.T, f1 filters.Args, f2 map[string]map[string]bool) { + t.Helper() + j1, err := f1.MarshalJSON() + if err != nil { + t.Error(err) + } + j2, err := json.Marshal(f2) + if err != nil { + t.Error(err) + } + if diff := cmp.Diff(string(j2), string(j1)); diff != "" { + t.Errorf("filters mismatch (-want +got):\n%s", diff) } } diff --git a/pkg/config/env.go b/pkg/config/env.go index c351b77f..45bd9757 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -8,9 +8,11 @@ import ( "os" "path" "path/filepath" + "regexp" + "sort" "strings" - "github.com/Autonomic-Cooperative/godotenv" + "git.coopcloud.tech/coop-cloud/godotenv" "github.com/sirupsen/logrus" ) @@ -37,6 +39,13 @@ var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git" const MAX_SANITISED_APP_NAME_LENGTH = 45 const MAX_DOCKER_SECRET_LENGTH = 64 +var BackupbotLabel = "coop-cloud.backupbot.enabled" + +// envVarModifiers is a list of env var modifier strings. These are added to +// env vars as comments and modify their processing by Abra, e.g. determining +// how long secrets should be. +var envVarModifiers = []string{"length"} + // GetServers retrieves all servers. func GetServers() ([]string, error) { var servers []string @@ -53,16 +62,30 @@ func GetServers() ([]string, error) { // ReadEnv loads an app envivornment into a map. func ReadEnv(filePath string) (AppEnv, error) { - var envFile AppEnv + var envVars AppEnv - envFile, err := godotenv.Read(filePath) + envVars, _, err := godotenv.Read(filePath) if err != nil { return nil, err } - logrus.Debugf("read %s from %s", envFile, filePath) + logrus.Debugf("read %s from %s", envVars, filePath) - return envFile, nil + return envVars, nil +} + +// ReadEnv loads an app envivornment and their modifiers in two different maps. +func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) { + var envVars AppEnv + + envVars, mods, err := godotenv.Read(filePath) + if err != nil { + return nil, mods, err + } + + logrus.Debugf("read %s from %s", envVars, filePath) + + return envVars, mods, nil } // ReadServerNames retrieves all server names. @@ -152,22 +175,107 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) { } return envVars, err } + defer file.Close() + + exportRegex, err := regexp.Compile(`^export\s+(\w+=\w+)`) + if err != nil { + return envVars, err + } scanner := bufio.NewScanner(file) for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "export") { - splitVals := strings.Split(line, "export ") + txt := scanner.Text() + if exportRegex.MatchString(txt) { + splitVals := strings.Split(txt, "export ") envVarDef := splitVals[len(splitVals)-1] keyVal := strings.Split(envVarDef, "=") if len(keyVal) != 2 { - return envVars, fmt.Errorf("couldn't parse %s", line) + return envVars, fmt.Errorf("couldn't parse %s", txt) } envVars[keyVal[0]] = keyVal[1] } } - logrus.Debugf("read %s from %s", envVars, abraSh) + if len(envVars) > 0 { + logrus.Debugf("read %s from %s", envVars, abraSh) + } else { + logrus.Debugf("read 0 env var exports from %s", abraSh) + } return envVars, nil } + +type EnvVar struct { + Name string + Present bool +} + +func CheckEnv(app App) ([]EnvVar, error) { + var envVars []EnvVar + + envSamplePath := path.Join(RECIPES_DIR, app.Recipe, ".env.sample") + if _, err := os.Stat(envSamplePath); err != nil { + if os.IsNotExist(err) { + return envVars, fmt.Errorf("%s does not exist?", envSamplePath) + } + return envVars, err + } + + envSample, err := ReadEnv(envSamplePath) + if err != nil { + return envVars, err + } + + var keys []string + for key := range envSample { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + if _, ok := app.Env[key]; ok { + envVars = append(envVars, EnvVar{Name: key, Present: true}) + } else { + envVars = append(envVars, EnvVar{Name: key, Present: false}) + } + } + + return envVars, nil +} + +// ReadAbraShCmdNames reads the names of commands. +func ReadAbraShCmdNames(abraSh string) ([]string, error) { + var cmdNames []string + + file, err := os.Open(abraSh) + if err != nil { + if os.IsNotExist(err) { + return cmdNames, nil + } + return cmdNames, err + } + defer file.Close() + + cmdNameRegex, err := regexp.Compile(`(\w+)(\(\).*\{)`) + if err != nil { + return cmdNames, err + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + matches := cmdNameRegex.FindStringSubmatch(line) + if len(matches) > 0 { + cmdNames = append(cmdNames, matches[1]) + } + } + + if len(cmdNames) > 0 { + logrus.Debugf("read %s from %s", strings.Join(cmdNames, " "), abraSh) + } else { + logrus.Debugf("read 0 command names from %s", abraSh) + } + + return cmdNames, nil +} diff --git a/pkg/config/env_test.go b/pkg/config/env_test.go index 87049289..f0626e6b 100644 --- a/pkg/config/env_test.go +++ b/pkg/config/env_test.go @@ -1,60 +1,69 @@ -package config +package config_test import ( + "fmt" "os" "path" "reflect" + "slices" "strings" "testing" + + "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" ) -var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder") -var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config") +var ( + TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder") + ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config") +) // make sure these are in alphabetical order -var tFolders = []string{"folder1", "folder2"} -var tFiles = []string{"bar.env", "foo.env"} +var ( + TFolders = []string{"folder1", "folder2"} + TFiles = []string{"bar.env", "foo.env"} +) -var appName = "ecloud" -var serverName = "evil.corp" +var ( + AppName = "ecloud" + ServerName = "evil.corp" +) -var expectedAppEnv = AppEnv{ +var ExpectedAppEnv = config.AppEnv{ "DOMAIN": "ecloud.evil.corp", "RECIPE": "ecloud", } -var expectedApp = App{ - Name: appName, - Recipe: expectedAppEnv["RECIPE"], - Domain: expectedAppEnv["DOMAIN"], - Env: expectedAppEnv, - Path: expectedAppFile.Path, - Server: expectedAppFile.Server, +var ExpectedApp = config.App{ + Name: AppName, + Recipe: ExpectedAppEnv["RECIPE"], + Domain: ExpectedAppEnv["DOMAIN"], + Env: ExpectedAppEnv, + Path: ExpectedAppFile.Path, + Server: ExpectedAppFile.Server, } -var expectedAppFile = AppFile{ - Path: path.Join(validAbraConf, "servers", serverName, appName+".env"), - Server: serverName, +var ExpectedAppFile = config.AppFile{ + Path: path.Join(ValidAbraConf, "servers", ServerName, AppName+".env"), + Server: ServerName, } -var expectedAppFiles = map[string]AppFile{ - appName: expectedAppFile, +var ExpectedAppFiles = map[string]config.AppFile{ + AppName: ExpectedAppFile, } -// var expectedServerNames = []string{"evil.corp"} - func TestGetAllFoldersInDirectory(t *testing.T) { - folders, err := GetAllFoldersInDirectory(testFolder) + folders, err := config.GetAllFoldersInDirectory(TestFolder) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(folders, tFolders) { - t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ",")) + if !reflect.DeepEqual(folders, TFolders) { + t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(TFolders, ","), strings.Join(folders, ",")) } } func TestGetAllFilesInDirectory(t *testing.T) { - files, err := GetAllFilesInDirectory(testFolder) + files, err := config.GetAllFilesInDirectory(TestFolder) if err != nil { t.Fatal(err) } @@ -62,23 +71,205 @@ func TestGetAllFilesInDirectory(t *testing.T) { for _, file := range files { fileNames = append(fileNames, file.Name()) } - if !reflect.DeepEqual(fileNames, tFiles) { - t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ",")) + if !reflect.DeepEqual(fileNames, TFiles) { + t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(TFiles, ","), strings.Join(fileNames, ",")) } } func TestReadEnv(t *testing.T) { - env, err := ReadEnv(expectedAppFile.Path) + env, err := config.ReadEnv(ExpectedAppFile.Path) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(env, expectedAppEnv) { + if !reflect.DeepEqual(env, ExpectedAppEnv) { t.Fatalf( "did not get expected application settings. Expected: DOMAIN=%s RECIPE=%s; Got: DOMAIN=%s RECIPE=%s", - expectedAppEnv["DOMAIN"], - expectedAppEnv["RECIPE"], + ExpectedAppEnv["DOMAIN"], + ExpectedAppEnv["RECIPE"], env["DOMAIN"], env["RECIPE"], ) } } + +func TestReadAbraShEnvVars(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh") + abraShEnv, err := config.ReadAbraShEnvVars(abraShPath) + if err != nil { + t.Fatal(err) + } + + if len(abraShEnv) == 0 { + t.Error("at least one env var should be exported") + } + + if _, ok := abraShEnv["INNER_FOO"]; ok { + t.Error("INNER_FOO should not be exported") + } + + if _, ok := abraShEnv["INNER_BAZ"]; ok { + t.Error("INNER_BAZ should not be exported") + } + + if _, ok := abraShEnv["OUTER_FOO"]; !ok { + t.Error("OUTER_FOO should be exported") + } +} + +func TestReadAbraShCmdNames(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + abraShPath := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, "abra.sh") + cmdNames, err := config.ReadAbraShCmdNames(abraShPath) + if err != nil { + t.Fatal(err) + } + + if len(cmdNames) == 0 { + t.Error("at least one command name should be found") + } + + expectedCmdNames := []string{"test_cmd", "test_cmd_args"} + for _, cmdName := range expectedCmdNames { + if !slices.Contains(cmdNames, cmdName) { + t.Fatalf("%s should have been found in %s", cmdName, abraShPath) + } + } +} + +func TestCheckEnv(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") + envSample, err := config.ReadEnv(envSamplePath) + if err != nil { + t.Fatal(err) + } + + app := config.App{ + Name: "test-app", + Recipe: r.Name, + Domain: "example.com", + Env: envSample, + Path: "example.com.env", + Server: "example.com", + } + + envVars, err := config.CheckEnv(app) + if err != nil { + t.Fatal(err) + } + + for _, envVar := range envVars { + if !envVar.Present { + t.Fatalf("%s should be present", envVar.Name) + } + } +} + +func TestCheckEnvError(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") + envSample, err := config.ReadEnv(envSamplePath) + if err != nil { + t.Fatal(err) + } + + delete(envSample, "DOMAIN") + + app := config.App{ + Name: "test-app", + Recipe: r.Name, + Domain: "example.com", + Env: envSample, + Path: "example.com.env", + Server: "example.com", + } + + envVars, err := config.CheckEnv(app) + if err != nil { + t.Fatal(err) + } + + for _, envVar := range envVars { + if envVar.Name == "DOMAIN" && envVar.Present { + t.Fatalf("%s should not be present", envVar.Name) + } + } +} + +func TestEnvVarCommentsRemoved(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") + envSample, err := config.ReadEnv(envSamplePath) + if err != nil { + t.Fatal(err) + } + + envVar, exists := envSample["WITH_COMMENT"] + if !exists { + t.Fatal("WITH_COMMENT env var should be present in .env.sample") + } + + if strings.Contains(envVar, "should be removed") { + t.Fatalf("comment from '%s' should be removed", envVar) + } + + envVar, exists = envSample["SECRET_TEST_PASS_TWO_VERSION"] + if !exists { + t.Fatal("WITH_COMMENT env var should be present in .env.sample") + } + + if strings.Contains(envVar, "length") { + t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should have been removed") + } +} + +func TestEnvVarModifiersIncluded(t *testing.T) { + offline := true + r, err := recipe.Get("abra-test-recipe", offline) + if err != nil { + t.Fatal(err) + } + + envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample") + envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath) + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "v1") { + t.Errorf("value should be 'v1', got: '%s'", envSample["SECRET_TEST_PASS_TWO_VERSION"]) + } + if modifiers == nil || modifiers["SECRET_TEST_PASS_TWO_VERSION"] == nil { + t.Errorf("no modifiers included") + } else { + if modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"] != "10" { + t.Errorf("length modifier should be '10', got: '%s'", modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"]) + } + } +} diff --git a/pkg/config/testdir/filtertest.env b/pkg/config/testdir/filtertest.env new file mode 100644 index 00000000..9250f3b4 --- /dev/null +++ b/pkg/config/testdir/filtertest.env @@ -0,0 +1,2 @@ +RECIPE=test-recipe +DOMAIN=test.example.com diff --git a/pkg/config/testdir/test-recipe/compose.yml b/pkg/config/testdir/test-recipe/compose.yml new file mode 100644 index 00000000..8232eca0 --- /dev/null +++ b/pkg/config/testdir/test-recipe/compose.yml @@ -0,0 +1,6 @@ +version: "3.8" +services: + foo: + image: debian + bar: + image: debian diff --git a/pkg/container/container.go b/pkg/container/container.go index 3bdf46f2..1354b0dd 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter) } - if len(containers) != 1 { + if len(containers) > 1 { var containersRaw []string for _, container := range containers { containerName := strings.Join(container.Names, " ") @@ -68,3 +68,15 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no return containers[0], nil } + +// GetContainerFromStackAndService retrieves the container for the given stack and service. +func GetContainerFromStackAndService(cl *client.Client, stack, service string) (types.Container, error) { + filters := filters.NewArgs() + filters.Add("name", fmt.Sprintf("^%s_%s", stack, service)) + + container, err := GetContainer(context.Background(), cl, filters, true) + if err != nil { + return types.Container{}, err + } + return container, nil +} diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index 4c888a3f..46408049 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -7,11 +7,16 @@ import ( // EnsureIPv4 ensures that an ipv4 address is set for a domain name func EnsureIPv4(domainName string) (string, error) { - ipv4, err := net.ResolveIPAddr("ip", domainName) + ipv4, err := net.ResolveIPAddr("ip4", domainName) if err != nil { return "", err } + // NOTE(d1): e.g. when there is only an ipv6 record available + if ipv4 == nil { + return "", fmt.Errorf("unable to resolve ipv4 address for %s", domainName) + } + return ipv4.String(), nil } diff --git a/pkg/dns/dns_test.go b/pkg/dns/dns_test.go index 49b414fd..99249a71 100644 --- a/pkg/dns/dns_test.go +++ b/pkg/dns/dns_test.go @@ -3,6 +3,8 @@ package dns import ( "fmt" "testing" + + "gotest.tools/v3/assert" ) func TestEnsureDomainsResolveSameIPv4(t *testing.T) { @@ -11,9 +13,10 @@ func TestEnsureDomainsResolveSameIPv4(t *testing.T) { serverName string shouldValidate bool }{ - // NOTE(d1): DNS records get checked, so use something we control. if - // you're here because of a failing test, try `dig +short ` to - // ensure stuff matches first! + // NOTE(d1): DNS records get checked, so use something that is maintained + // within the federation. if you're here because of a failing test, try + // `dig +short ` to ensure stuff matches first! If flakyness + // becomes an issue we can look into mocking {"docs.coopcloud.tech", "coopcloud.tech", true}, {"docs.coopcloud.tech", "swarm.autonomic.zone", true}, @@ -36,3 +39,26 @@ func TestEnsureDomainsResolveSameIPv4(t *testing.T) { } } } + +func TestEnsureIpv4(t *testing.T) { + // NOTE(d1): DNS records get checked, so use something that is maintained + // within the federation. if you're here because of a failing test, try `dig + // +short ` to ensure stuff matches first! If flakyness becomes an + // issue we can look into mocking + domainName := "collabora.ostrom.collective.tools" + serverName := "ostrom.collective.tools" + + for i := 0; i < 15; i++ { + domainIpv4, err := EnsureIPv4(domainName) + if err != nil { + t.Fatal(err) + } + + serverIpv4, err := EnsureIPv4(serverName) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, domainIpv4, serverIpv4) + } +} diff --git a/pkg/git/add.go b/pkg/git/add.go new file mode 100644 index 00000000..36668568 --- /dev/null +++ b/pkg/git/add.go @@ -0,0 +1,27 @@ +package git + +import ( + "github.com/go-git/go-git/v5" + "github.com/sirupsen/logrus" +) + +// Add adds a file to the git index. +func Add(repoPath, path string, dryRun bool) error { + repo, err := git.PlainOpen(repoPath) + if err != nil { + return err + } + + worktree, err := repo.Worktree() + if err != nil { + return err + } + + if dryRun { + logrus.Debugf("dry run: adding %s", path) + } else { + worktree.Add(path) + } + + return nil +} diff --git a/pkg/git/commit.go b/pkg/git/commit.go index a3c6433f..4b0e554b 100644 --- a/pkg/git/commit.go +++ b/pkg/git/commit.go @@ -8,7 +8,7 @@ import ( ) // Commit runs a git commit -func Commit(repoPath, glob, commitMessage string, dryRun bool) error { +func Commit(repoPath, commitMessage string, dryRun bool) error { if commitMessage == "" { return fmt.Errorf("no commit message specified?") } @@ -33,17 +33,8 @@ func Commit(repoPath, glob, commitMessage string, dryRun bool) error { } if !dryRun { - err = commitWorktree.AddGlob(glob) - if err != nil { - return err - } - logrus.Debugf("staged %s for commit", glob) - } else { - logrus.Debugf("dry run: did not stage %s for commit", glob) - } - - if !dryRun { - _, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{}) + // NOTE(d1): `All: true` does not include untracked files + _, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{All: true}) if err != nil { return err } diff --git a/pkg/git/diff.go b/pkg/git/diff.go new file mode 100644 index 00000000..65a658f8 --- /dev/null +++ b/pkg/git/diff.go @@ -0,0 +1,42 @@ +package git + +import ( + "fmt" + "os/exec" + + "github.com/sirupsen/logrus" +) + +// getGitDiffArgs builds the `git diff` invocation args. It removes the usage +// of a pager and ensures that colours are specified even when Git might detect +// otherwise. +func getGitDiffArgs(repoPath string) []string { + return []string{ + "-C", + repoPath, + "--no-pager", + "-c", + "color.diff=always", + "diff", + } +} + +// DiffUnstaged shows a `git diff`. Due to limitations in the underlying go-git +// library, this implementation requires the /usr/bin/git binary. It gracefully +// skips if it cannot find the command on the system. +func DiffUnstaged(path string) error { + if _, err := exec.LookPath("git"); err != nil { + logrus.Warnf("unable to locate git command, cannot output diff") + return nil + } + + gitDiffArgs := getGitDiffArgs(path) + diff, err := exec.Command("git", gitDiffArgs...).Output() + if err != nil { + return nil + } + + fmt.Print(string(diff)) + + return nil +} diff --git a/pkg/jsontable/jsontable.go b/pkg/jsontable/jsontable.go index ef49137c..51dd6020 100644 --- a/pkg/jsontable/jsontable.go +++ b/pkg/jsontable/jsontable.go @@ -3,6 +3,7 @@ package jsontable import ( "fmt" "io" + "strings" "github.com/olekukonko/tablewriter" ) @@ -109,6 +110,9 @@ func (t *JSONTable) _JSONRenderInner() { } writeChar(t.out, '{') for keyidx, key := range t.keys { + key := strings.ToLower(key) + key = strings.ReplaceAll(key, " ", "-") + value := "nil" if keyidx < len(row) { value = row[keyidx] @@ -138,10 +142,8 @@ func (t *JSONTable) JSONRender() { if t.hasCaption { fmt.Fprintf(t.out, "\"%s\":\"%s\",", t.captionLabel, t.caption) - } fmt.Fprintf(t.out, "\"%s\":", t.dataLabel) - } // write list diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index 622d24c7..a5d6d2eb 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -7,6 +7,7 @@ import ( "os" "path" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -31,7 +32,7 @@ import ( // RecipeCatalogueURL is the only current recipe catalogue available. const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json" -// ReposMetadataURL is the recipe repository metadata +// ReposMetadataURL is the recipe repository metadata. const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos" // tag represents a git tag. @@ -63,6 +64,11 @@ type RecipeMeta struct { Website string `json:"website"` } +// TopicMeta represents a list of topics for a repository. +type TopicMeta struct { + Topics []string `json:"topics"` +} + // LatestVersion returns the latest version of a recipe. func (r RecipeMeta) LatestVersion() string { var version string @@ -258,6 +264,20 @@ func (r Recipe) SampleEnv() (map[string]string, error) { return sampleEnv, nil } +// Ensure makes sure the recipe exists, is up to date and has the latest version checked out. +func Ensure(recipeName string) error { + if err := EnsureExists(recipeName); err != nil { + return err + } + if err := EnsureUpToDate(recipeName); err != nil { + return err + } + if err := EnsureLatest(recipeName); err != nil { + return err + } + return nil +} + // EnsureExists ensures that a recipe is locally cloned func EnsureExists(recipeName string) error { recipeDir := path.Join(config.RECIPES_DIR, recipeName) @@ -822,7 +842,16 @@ func ReadReposMetadata() (RepoCatalogue, error) { } for idx, repo := range reposList { - reposMeta[repo.Name] = reposList[idx] + var topicMeta TopicMeta + + topicsURL := getReposTopicUrl(repo.Name) + if err := web.ReadJSON(topicsURL, &topicMeta); err != nil { + return reposMeta, err + } + + if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" { + reposMeta[repo.Name] = reposList[idx] + } } pageIdx++ @@ -1002,14 +1031,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error { retrieveBar.Add(1) return } - if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists { - ch <- rm.Name - retrieveBar.Add(1) - return - } recipeDir := path.Join(config.RECIPES_DIR, rm.Name) - if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil { logrus.Fatal(err) } @@ -1025,3 +1048,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error { return nil } + +// getReposTopicUrl retrieves the repository specific topic listing. +func getReposTopicUrl(repoName string) string { + return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName) +} diff --git a/pkg/recipe/recipe_test.go b/pkg/recipe/recipe_test.go index fb0f29ae..59a57ebc 100644 --- a/pkg/recipe/recipe_test.go +++ b/pkg/recipe/recipe_test.go @@ -3,15 +3,10 @@ package recipe import ( "testing" - "coopcloud.tech/abra/pkg/catalogue" "github.com/stretchr/testify/assert" ) func TestGetVersionLabelLocalDoesNotUseTimeoutLabel(t *testing.T) { - if err := catalogue.EnsureCatalogue(); err != nil { - t.Fatal(err) - } - offline := true recipe, err := Get("traefik", offline) if err != nil { diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index 45cd94e7..d15f302d 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -4,6 +4,7 @@ package secret import ( + "context" "fmt" "slices" "strconv" @@ -15,15 +16,29 @@ import ( "coopcloud.tech/abra/pkg/upstream/stack" loader "coopcloud.tech/abra/pkg/upstream/stack" "github.com/decentral1se/passgen" + "github.com/docker/docker/api/types" dockerClient "github.com/docker/docker/client" "github.com/sirupsen/logrus" ) -// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config -// secret definition. -type secretValue struct { +// Secret represents a secret. +type Secret struct { + // Version comes from the secret version environment variable. + // For example: + // SECRET_FOO=v1 Version string - Length int + // Length comes from the length modifier at the secret version environment + // variable. For Example: + // SECRET_FOO=v1 # length=12 + Length int + // RemoteName is the name of the secret on the server. For example: + // name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION} + // With the following: + // STACK_NAME=test_example_com + // SECRET_TEST_PASS_TWO_VERSION=v2 + // Will have this remote name: + // test_example_com_test_pass_two_v2 + RemoteName string } // GeneratePasswords generates passwords. @@ -33,7 +48,6 @@ func GeneratePasswords(count, length uint) ([]string, error) { length, passgen.AlphabetDefault, ) - if err != nil { return nil, err } @@ -52,7 +66,6 @@ func GeneratePassphrases(count uint) ([]string, error) { passgen.PassphraseCasingDefault, passgen.WordListDefault, ) - if err != nil { return nil, err } @@ -67,13 +80,23 @@ func GeneratePassphrases(count uint) ([]string, error) { // and some times you don't (as the caller). We need to be able to handle the // "app new" case where we pass in the .env.sample and the "secret generate" // case where the app is created. -func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeName string) (map[string]string, error) { - secretConfigs := make(map[string]string) +func ReadSecretsConfig(appEnvPath string, composeFiles []string, stackName string) (map[string]Secret, error) { + appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath) + if err != nil { + return nil, err + } + // Set the STACK_NAME to be able to generate the remote name correctly. + appEnv["STACK_NAME"] = stackName opts := stack.Deploy{Composefiles: composeFiles} config, err := loader.LoadComposefile(opts, appEnv) if err != nil { - return secretConfigs, err + return nil, err + } + // Read the compose files without injecting environment variables. + configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation) + if err != nil { + return nil, err } var enabledSecrets []string @@ -85,83 +108,74 @@ func ReadSecretsConfig(appEnv map[string]string, composeFiles []string, recipeNa if len(enabledSecrets) == 0 { logrus.Debugf("not generating app secrets, none enabled in recipe config") - return secretConfigs, nil + return nil, nil } + secretValues := map[string]Secret{} for secretId, secretConfig := range config.Secrets { if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" { - return secretConfigs, fmt.Errorf("missing version for secret? (%s)", secretId) + return nil, fmt.Errorf("missing version for secret? (%s)", secretId) } if !(slices.Contains(enabledSecrets, secretId)) { - logrus.Warnf("%s not enabled in recipe config, not generating", secretId) + logrus.Warnf("%s not enabled in recipe config, skipping", secretId) continue } lastIdx := strings.LastIndex(secretConfig.Name, "_") secretVersion := secretConfig.Name[lastIdx+1:] - secretConfigs[secretId] = secretVersion + value := Secret{Version: secretVersion, RemoteName: secretConfig.Name} + + // Check if the length modifier is set for this secret. + for envName, modifierValues := range appModifiers { + // configWithoutEnv contains the raw name as defined in the compose.yaml + // The name will look something like this: + // name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION} + // To check if the current modifier is for the current secret we check + // if the raw name contains the env name (e.g. SECRET_TEST_PASS_TWO_VERSION). + if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, envName) { + continue + } + lengthRaw, ok := modifierValues["length"] + if ok { + length, err := strconv.Atoi(lengthRaw) + if err != nil { + return nil, err + } + value.Length = length + } + break + } + secretValues[secretId] = value } - return secretConfigs, nil -} - -func ParseSecretValue(secret string) (secretValue, error) { - values := strings.Split(secret, "#") - if len(values) == 0 { - return secretValue{}, fmt.Errorf("unable to parse %s", secret) - } - - if len(values) == 1 { - return secretValue{Version: values[0], Length: 0}, nil - } - - split := strings.Split(values[1], "=") - parsed := split[len(split)-1] - stripped := strings.ReplaceAll(parsed, " ", "") - length, err := strconv.Atoi(stripped) - if err != nil { - return secretValue{}, err - } - version := strings.ReplaceAll(values[0], " ", "") - - logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret) - - return secretValue{Version: version, Length: length}, nil + return secretValues, nil } // GenerateSecrets generates secrets locally and sends them to a remote server for storage. -func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, appName, server string) (map[string]string, error) { - secrets := make(map[string]string) - +func GenerateSecrets(cl *dockerClient.Client, secrets map[string]Secret, server string) (map[string]string, error) { + secretsGenerated := map[string]string{} var mutex sync.Mutex var wg sync.WaitGroup - ch := make(chan error, len(secretsFromConfig)) - for n, v := range secretsFromConfig { + ch := make(chan error, len(secrets)) + for n, v := range secrets { wg.Add(1) - go func(secretName, secretValue string) { + go func(secretName string, secret Secret) { defer wg.Done() - parsedSecretValue, err := ParseSecretValue(secretValue) - if err != nil { - ch <- err - return - } + logrus.Debugf("attempting to generate and store %s on %s", secret.RemoteName, server) - secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version) - logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server) - - if parsedSecretValue.Length > 0 { - passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length)) + if secret.Length > 0 { + passwords, err := GeneratePasswords(1, uint(secret.Length)) if err != nil { ch <- err return } - if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil { + if err := client.StoreSecret(cl, secret.RemoteName, passwords[0], server); err != nil { if strings.Contains(err.Error(), "AlreadyExists") { - logrus.Warnf("%s already exists, moving on...", secretRemoteName) + logrus.Warnf("%s already exists, moving on...", secret.RemoteName) ch <- nil } else { ch <- err @@ -171,7 +185,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin mutex.Lock() defer mutex.Unlock() - secrets[secretName] = passwords[0] + secretsGenerated[secretName] = passwords[0] } else { passphrases, err := GeneratePassphrases(1) if err != nil { @@ -179,9 +193,9 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin return } - if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil { + if err := client.StoreSecret(cl, secret.RemoteName, passphrases[0], server); err != nil { if strings.Contains(err.Error(), "AlreadyExists") { - logrus.Warnf("%s already exists, moving on...", secretRemoteName) + logrus.Warnf("%s already exists, moving on...", secret.RemoteName) ch <- nil } else { ch <- err @@ -191,7 +205,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin mutex.Lock() defer mutex.Unlock() - secrets[secretName] = passphrases[0] + secretsGenerated[secretName] = passphrases[0] } ch <- nil }(n, v) @@ -199,16 +213,74 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin wg.Wait() - for range secretsFromConfig { + for range secrets { err := <-ch if err != nil { return nil, err } } - logrus.Debugf("generated and stored %s on %s", secrets, server) + logrus.Debugf("generated and stored %v on %s", secrets, server) - return secrets, nil + return secretsGenerated, nil +} + +type secretStatus struct { + LocalName string + RemoteName string + Version string + CreatedOnRemote bool +} + +type secretStatuses []secretStatus + +// PollSecretsStatus checks status of secrets by comparing the local recipe +// config and deploymend server state. +func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses, error) { + var secStats secretStatuses + + composeFiles, err := config.GetComposeFiles(app.Recipe, app.Env) + if err != nil { + return secStats, err + } + + secretsConfig, err := ReadSecretsConfig(app.Path, composeFiles, app.StackName()) + if err != nil { + return secStats, err + } + + filters, err := app.Filters(false, false) + if err != nil { + return secStats, err + } + + secretList, err := cl.SecretList(context.Background(), types.SecretListOptions{Filters: filters}) + if err != nil { + return secStats, err + } + + remoteSecretNames := make(map[string]bool) + for _, cont := range secretList { + remoteSecretNames[cont.Spec.Annotations.Name] = true + } + + for secretName, val := range secretsConfig { + createdRemote := false + + secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version) + if _, ok := remoteSecretNames[secretRemoteName]; ok { + createdRemote = true + } + + secStats = append(secStats, secretStatus{ + LocalName: secretName, + RemoteName: secretRemoteName, + Version: val.Version, + CreatedOnRemote: createdRemote, + }) + } + + return secStats, nil } func EnsureSecretLengths(secrets map[string]string, domainName string, sanitisedAppName string) error { diff --git a/pkg/secret/secret_test.go b/pkg/secret/secret_test.go index 82dd200c..fc10c098 100644 --- a/pkg/secret/secret_test.go +++ b/pkg/secret/secret_test.go @@ -1,46 +1,30 @@ package secret import ( - "path" "testing" - "coopcloud.tech/abra/pkg/catalogue" - "coopcloud.tech/abra/pkg/config" - "coopcloud.tech/abra/pkg/recipe" - "coopcloud.tech/abra/pkg/upstream/stack" - loader "coopcloud.tech/abra/pkg/upstream/stack" "github.com/stretchr/testify/assert" ) func TestReadSecretsConfig(t *testing.T) { - if err := catalogue.EnsureCatalogue(); err != nil { - t.Fatal(err) - } - - offline := true - recipe, err := recipe.Get("matrix-synapse", offline) + composeFiles := []string{"./testdir/compose.yaml"} + secretsFromConfig, err := ReadSecretsConfig("./testdir/.env.sample", composeFiles, "test_example_com") if err != nil { t.Fatal(err) } - sampleEnv, err := recipe.SampleEnv() - if err != nil { - t.Fatal(err) - } + // Simple secret + assert.Equal(t, "test_example_com_test_pass_one_v2", secretsFromConfig["test_pass_one"].RemoteName) + assert.Equal(t, "v2", secretsFromConfig["test_pass_one"].Version) + assert.Equal(t, 0, secretsFromConfig["test_pass_one"].Length) - composeFiles := []string{path.Join(config.RECIPES_DIR, recipe.Name, "compose.yml")} - secretsFromConfig, err := ReadSecretsConfig(sampleEnv, composeFiles, recipe.Name) - if err != nil { - t.Fatal(err) - } + // Has a length modifier + assert.Equal(t, "test_example_com_test_pass_two_v1", secretsFromConfig["test_pass_two"].RemoteName) + assert.Equal(t, "v1", secretsFromConfig["test_pass_two"].Version) + assert.Equal(t, 10, secretsFromConfig["test_pass_two"].Length) - opts := stack.Deploy{Composefiles: composeFiles} - config, err := loader.LoadComposefile(opts, sampleEnv) - if err != nil { - t.Fatal(err) - } - - for secretId := range config.Secrets { - assert.Contains(t, secretsFromConfig, secretId) - } + // Secret name does not include the secret id + assert.Equal(t, "test_example_com_pass_three_v2", secretsFromConfig["test_pass_three"].RemoteName) + assert.Equal(t, "v2", secretsFromConfig["test_pass_three"].Version) + assert.Equal(t, 0, secretsFromConfig["test_pass_three"].Length) } diff --git a/pkg/secret/testdir/.env.sample b/pkg/secret/testdir/.env.sample new file mode 100644 index 00000000..bbcc2904 --- /dev/null +++ b/pkg/secret/testdir/.env.sample @@ -0,0 +1,3 @@ +SECRET_TEST_PASS_ONE_VERSION=v2 +SECRET_TEST_PASS_TWO_VERSION=v1 # length=10 +SECRET_TEST_PASS_THREE_VERSION=v2 diff --git a/pkg/secret/testdir/compose.yaml b/pkg/secret/testdir/compose.yaml new file mode 100644 index 00000000..86cc2575 --- /dev/null +++ b/pkg/secret/testdir/compose.yaml @@ -0,0 +1,21 @@ +--- +version: "3.8" + +services: + app: + image: nginx:1.21.0 + secrets: + - test_pass_one + - test_pass_two + - test_pass_three + +secrets: + test_pass_one: + external: true + name: ${STACK_NAME}_test_pass_one_${SECRET_TEST_PASS_ONE_VERSION} # should be removed + test_pass_two: + external: true + name: ${STACK_NAME}_test_pass_two_${SECRET_TEST_PASS_TWO_VERSION} + test_pass_three: + external: true + name: ${STACK_NAME}_pass_three_${SECRET_TEST_PASS_THREE_VERSION} # secretId and name don't match diff --git a/pkg/service/service.go b/pkg/service/service.go index 3d92d821..48cdce75 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -14,6 +14,70 @@ import ( "github.com/sirupsen/logrus" ) +// GetService retrieves a service container based on a label. If prompt is true +// and the retrievd count of service containers does not match 1, then a prompt +// is presented to let the user choose. An error is returned when no service is +// found. +func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) { + services, err := cl.ServiceList(c, types.ServiceListOptions{}) + if err != nil { + return swarm.Service{}, err + } + + if len(services) == 0 { + return swarm.Service{}, fmt.Errorf("no services deployed?") + } + + var matchingServices []swarm.Service + for _, service := range services { + if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" { + matchingServices = append(matchingServices, service) + } + } + + if len(matchingServices) == 0 { + return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label) + } + + if len(matchingServices) > 1 { + var servicesRaw []string + for _, service := range matchingServices { + serviceName := service.Spec.Name + created := formatter.HumanDuration(service.CreatedAt.Unix()) + servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created)) + } + + if !prompt { + err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " ")) + return swarm.Service{}, err + } + + logrus.Warnf("ambiguous service list received, prompting for input") + + var response string + prompt := &survey.Select{ + Message: "which service are you looking for?", + Options: servicesRaw, + } + + if err := survey.AskOne(prompt, &response); err != nil { + return swarm.Service{}, err + } + + chosenService := strings.TrimSpace(strings.Split(response, " ")[0]) + for _, service := range matchingServices { + serviceName := strings.ToLower(service.Spec.Name) + if serviceName == chosenService { + return service, nil + } + } + + logrus.Panic("failed to match chosen service") + } + + return matchingServices[0], nil +} + // GetService retrieves a service container. If prompt is true and the retrievd // count of service containers does not match 1, then a prompt is presented to // let the user choose. A count of 0 is handled gracefully. diff --git a/pkg/upstream/commandconn/connection.go b/pkg/upstream/commandconn/connection.go index 259b71d1..b2eebdbe 100644 --- a/pkg/upstream/commandconn/connection.go +++ b/pkg/upstream/commandconn/connection.go @@ -18,7 +18,7 @@ import ( // // ssh://@ URL requires Docker 18.09 or later on the remote host. func GetConnectionHelper(daemonURL string) (*connhelper.ConnectionHelper, error) { - return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=5"}) + return getConnectionHelper(daemonURL, []string{"-o ConnectTimeout=60"}) } func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.ConnectionHelper, error) { diff --git a/pkg/upstream/container/exec.go b/pkg/upstream/container/exec.go index e811481a..82a2c570 100644 --- a/pkg/upstream/container/exec.go +++ b/pkg/upstream/container/exec.go @@ -13,7 +13,10 @@ import ( "github.com/sirupsen/logrus" ) -func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error { +// RunExec runs a command on a remote container. io.Writer corresponds to the +// command output. +func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, + execConfig *types.ExecConfig) (io.Writer, error) { ctx := context.Background() // We need to check the tty _before_ we do the ContainerExecCreate, because @@ -21,22 +24,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string // there's no easy way to clean those up). But also in order to make "not // exist" errors take precedence we do a dummy inspect first. if _, err := client.ContainerInspect(ctx, containerID); err != nil { - return err + return nil, err } if !execConfig.Detach { if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil { - return err + return nil, err } } response, err := client.ContainerExecCreate(ctx, containerID, *execConfig) if err != nil { - return err + return nil, err } execID := response.ID if execID == "" { - return errors.New("exec ID empty") + return nil, errors.New("exec ID empty") } if execConfig.Detach { @@ -44,13 +47,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string Detach: execConfig.Detach, Tty: execConfig.Tty, } - return client.ContainerExecStart(ctx, execID, execStartCheck) + return nil, client.ContainerExecStart(ctx, execID, execStartCheck) } return interactiveExec(ctx, dockerCli, client, execConfig, execID) } func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client, - execConfig *types.ExecConfig, execID string) error { + execConfig *types.ExecConfig, execID string) (io.Writer, error) { // Interactive exec requested. var ( out, stderr io.Writer @@ -76,7 +79,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie } resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck) if err != nil { - return err + return out, err } defer resp.Close() @@ -107,10 +110,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie if err := <-errCh; err != nil { logrus.Debugf("Error hijack: %s", err) - return err + return out, err } - return getExecExitStatus(ctx, client, execID) + return out, getExecExitStatus(ctx, client, execID) } func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error { diff --git a/pkg/upstream/convert/service.go b/pkg/upstream/convert/service.go index 44cfd569..16cd0cfa 100644 --- a/pkg/upstream/convert/service.go +++ b/pkg/upstream/convert/service.go @@ -420,7 +420,7 @@ func convertServiceSecrets( return nil, err } - // NOTE(d1): strip # length=... modifiers + // NOTE(d1): strip all comments if strings.Contains(obj.Name, "#") { vals := strings.Split(obj.Name, "#") obj.Name = strings.TrimSpace(vals[0]) diff --git a/pkg/upstream/stack/loader.go b/pkg/upstream/stack/loader.go index 51b678b0..9790903f 100644 --- a/pkg/upstream/stack/loader.go +++ b/pkg/upstream/stack/loader.go @@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) { opts.SkipValidation = false } +// SkipInterpolation skip interpolating environment variables. +func SkipInterpolation(opts *loader.Options) { + opts.SkipInterpolation = true +} + // LoadComposefile parse the composefile specified in the cli and returns its Config and version. -func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) { +func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) { configDetails, err := getConfigDetails(opts.Composefiles, appEnv) if err != nil { return nil, err } + if options == nil { + options = []func(*loader.Options){DontSkipValidation} + } + dicts := getDictsFrom(configDetails.ConfigFiles) - config, err := loader.Load(configDetails, DontSkipValidation) + config, err := loader.Load(configDetails, options...) if err != nil { if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok { return nil, fmt.Errorf("compose file contains unsupported options: %s", diff --git a/pkg/upstream/stack/stack.go b/pkg/upstream/stack/stack.go index c2d1e214..d8e7bdd9 100644 --- a/pkg/upstream/stack/stack.go +++ b/pkg/upstream/stack/stack.go @@ -233,7 +233,7 @@ func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAP network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{}) switch { case dockerClient.IsErrNotFound(err): - return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName) + return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed, which you can do by running this on the server: docker network create -d overlay proxy", networkName) case err != nil: return err case network.Scope != "swarm": diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh new file mode 100755 index 00000000..4227c3d3 --- /dev/null +++ b/scripts/docker/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ ! -f .envrc ]; then + . .envrc.sample +else + . .envrc +fi + +git config --global --add safe.directory /abra # work around funky file permissions + +make build diff --git a/scripts/installer/installer b/scripts/installer/installer index 87a72393..b6e0e543 100755 --- a/scripts/installer/installer +++ b/scripts/installer/installer @@ -1,8 +1,8 @@ #!/usr/bin/env bash -ABRA_VERSION="0.8.0-beta" +ABRA_VERSION="0.9.0-beta" ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION" -RC_VERSION="0.8.0-beta" +RC_VERSION="0.8.0-rc1-beta" RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION" for arg in "$@"; do @@ -45,7 +45,9 @@ function install_abra_release { fi ARCH=$(uname -m) - if [[ $ARCH =~ "aarch64" ]]; then + if [[ $ARCH =~ "x86_64" ]]; then + ARCH="amd64" + elif [[ $ARCH =~ "aarch64" ]]; then ARCH="arm64" elif [[ $ARCH =~ "armv5l" ]]; then ARCH="armv5" @@ -55,7 +57,7 @@ function install_abra_release { ARCH="armv7" fi PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')_$ARCH - FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM"" + FILENAME="abra_"$ABRA_VERSION"_"$PLATFORM".tar.gz" sed_command_rel='s/.*"assets":\[\{[^]]*"name":"'$FILENAME'"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p' sed_command_checksums='s/.*"assets":\[\{[^\]*"name":"checksums.txt"[^}]*"browser_download_url":"([^"]*)".*\].*/\1/p' @@ -65,17 +67,22 @@ function install_abra_release { checksums=$(wget -q -O- $checksums_url) checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p') + abra_download="/tmp/abra-download.tar.gz" echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..." - wget -q "$release_url" -O "$HOME/.local/bin/.abra-download" - localsum=$(sha256sum $HOME/.local/bin/.abra-download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p') + + wget -q "$release_url" -O $abra_download + localsum=$(sha256sum $abra_download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p') echo "checking if checksums match..." if [[ "$localsum" != "$checksum" ]]; then print_checksum_error exit 1 fi echo "$(tput setaf 2)check successful!$(tput sgr0)" - mv "$HOME/.local/bin/.abra-download" "$HOME/.local/bin/abra" + cd /tmp/ + tar xf abra-download.tar.gz + mv abra "$HOME/.local/bin/abra" + tar tf abra-download.tar.gz | xargs rm -f chmod +x "$HOME/.local/bin/abra" x=$(echo $PATH | grep $HOME/.local/bin) diff --git a/tests/integration/app_check.bats b/tests/integration/app_check.bats index 04d99fe2..e296e3cb 100644 --- a/tests/integration/app_check.bats +++ b/tests/integration/app_check.bats @@ -35,7 +35,7 @@ setup(){ run $ABRA app check "$TEST_APP_DOMAIN" assert_success - assert_output --partial 'all necessary environment variables defined' + refute_output --partial '❌' assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE" } @@ -70,13 +70,13 @@ setup(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA app check "$TEST_APP_DOMAIN" assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _reset_recipe } @@ -86,7 +86,7 @@ setup(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 1' + assert_output --partial "Your branch is behind 'origin/main' by 1 commit" # NOTE(d1): we can't quite tell if this will fail or not in the future, so, # since it isn't an important part of what we're testing here, we don't check @@ -94,7 +94,7 @@ setup(){ run $ABRA app check "$TEST_APP_DOMAIN" --offline run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 1' + assert_output --partial "Your branch is behind 'origin/main' by 1 commit" _reset_recipe } @@ -111,12 +111,16 @@ setup(){ } @test "error if missing env var" { + run $ABRA app check "$TEST_APP_DOMAIN" + assert_success + refute_output --partial '❌' + run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"' assert_success run $ABRA app check "$TEST_APP_DOMAIN" --chaos - assert_failure - assert_output --partial "NEW_VAR" + assert_success + assert_output --partial '❌' _checkout_recipe } diff --git a/tests/integration/app_cmd.bats b/tests/integration/app_cmd.bats index b7762600..fd5e6695 100644 --- a/tests/integration/app_cmd.bats +++ b/tests/integration/app_cmd.bats @@ -25,6 +25,24 @@ teardown(){ fi } +# bats test_tags=slow +@test "autocomplete" { + run $ABRA app cmd --generate-bash-completion + assert_success + assert_output "$TEST_APP_DOMAIN" + + run $ABRA app cmd "$TEST_APP_DOMAIN" --generate-bash-completion + assert_success + assert_output "app" + + run $ABRA app cmd "$TEST_APP_DOMAIN" app --generate-bash-completion + assert_success + assert_output "test_cmd +test_cmd_arg +test_cmd_args +test_cmd_export" +} + @test "validate app argument" { run $ABRA app cmd assert_failure @@ -40,7 +58,7 @@ teardown(){ assert_success assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE" - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local + run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd assert_success assert_output --partial 'baz' @@ -52,7 +70,7 @@ teardown(){ assert_success assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo" - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local + run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd assert_failure assert_output --partial 'locally unstaged changes' @@ -65,7 +83,7 @@ teardown(){ assert_success assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo" - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos + run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd assert_success assert_output --partial 'baz' @@ -78,14 +96,14 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local + run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd assert_success assert_output --partial 'baz' run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --partial "up to date" _reset_recipe "$TEST_RECIPE" } @@ -95,14 +113,14 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --offline + run $ABRA app cmd --local --offline "$TEST_APP_DOMAIN" test_cmd assert_success assert_output --partial 'baz' run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _reset_recipe "$TEST_RECIPE" } @@ -114,13 +132,13 @@ teardown(){ } @test "error if missing arguments when passing --local" { - run $ABRA app cmd "$TEST_APP_DOMAIN" --local + run $ABRA app cmd --local "$TEST_APP_DOMAIN" assert_failure assert_output --partial 'missing arguments' } @test "cannot use --local and --user at same time" { - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --user root + run $ABRA app cmd --local --user root "$TEST_APP_DOMAIN" test_cmd assert_failure assert_output --partial 'cannot use --local & --user together' } @@ -129,7 +147,7 @@ teardown(){ run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh" assert_success - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local --chaos + run $ABRA app cmd --local --chaos "$TEST_APP_DOMAIN" test_cmd assert_failure assert_output --partial "$ABRA_DIR/recipes/$TEST_RECIPE/abra.sh does not exist" @@ -137,25 +155,25 @@ teardown(){ } @test "error if missing command" { - run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist --local + run $ABRA app cmd --local "$TEST_APP_DOMAIN" doesnt_exist assert_failure assert_output --partial "doesn't have a doesnt_exist function" } @test "run --local command" { - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd --local + run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd assert_success assert_output --partial 'baz' } @test "run command with single arg" { - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_arg --local -- bing + run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_arg -- bing assert_success assert_output --partial 'bing' } @test "run command with several args" { - run $ABRA app cmd "$TEST_APP_DOMAIN" test_cmd_args --local -- bong bang + run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_args -- bong bang assert_success assert_output --partial 'bong bang' } diff --git a/tests/integration/app_cp.bats b/tests/integration/app_cp.bats index 2687e9d4..524ff866 100644 --- a/tests/integration/app_cp.bats +++ b/tests/integration/app_cp.bats @@ -5,9 +5,11 @@ setup_file(){ _common_setup _add_server _new_app + _deploy_app } teardown_file(){ + _undeploy_app _rm_app _rm_server } @@ -17,13 +19,6 @@ setup(){ _common_setup } -teardown(){ - # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888 - if [[ -z "${BATS_TEST_COMPLETED}" ]]; then - _undeploy_app - fi -} - @test "validate app argument" { run $ABRA app cp assert_failure @@ -54,68 +49,120 @@ teardown(){ assert_output --partial 'arguments must take $SERVICE:$PATH form' } -@test "detect 'coming FROM' syntax" { - run $ABRA app cp "$TEST_APP_DOMAIN" app:/myfile.txt . --debug - assert_failure - assert_output --partial 'coming FROM the container' -} - -@test "detect 'going TO' syntax" { - run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere --debug - assert_failure - assert_output --partial 'going TO the container' -} - @test "error if local file missing" { - run $ABRA app cp "$TEST_APP_DOMAIN" myfile.txt app:/somewhere + run $ABRA app cp "$TEST_APP_DOMAIN" thisfileshouldnotexist.txt app:/somewhere assert_failure - assert_output --partial 'myfile.txt does not exist locally?' + assert_output --partial 'local stat thisfileshouldnotexist.txt: no such file or directory' } # bats test_tags=slow @test "error if service doesn't exist" { - _deploy_app + _mkfile "$BATS_TMPDIR/myfile.txt" "foo" - run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt" - assert_success - - run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ + run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ --debug assert_failure assert_output --partial 'no containers matching' - run rm -rf "$BATS_TMPDIR/myfile.txt" - assert_success - - _undeploy_app + _rm "$BATS_TMPDIR/myfile.txt" } # bats test_tags=slow -@test "copy to container" { - _deploy_app - - run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt" - assert_success +@test "copy local file to container directory" { + _mkfile "$BATS_TMPDIR/myfile.txt" "foo" run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc assert_success - run rm -rf "$BATS_TMPDIR/myfile.txt" + run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt assert_success + assert_output --partial "foo" - _undeploy_app + _rm "$BATS_TMPDIR/myfile.txt" + _rm_remote "/etc/myfile.txt" } # bats test_tags=slow -@test "copy from container" { - _deploy_app +@test "copy local file to container file (and override on remote)" { + _mkfile "$BATS_TMPDIR/myfile.txt" "foo" - run bash -c "echo foo >> $BATS_TMPDIR/myfile.txt" + # create + run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt assert_success - run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc + run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt + assert_success + assert_output --partial "foo" + + _mkfile "$BATS_TMPDIR/myfile.txt" "bar" + + # override + run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile.txt assert_success - run rm -rf "$BATS_TMPDIR/myfile.txt" + run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt + assert_success + assert_output --partial "bar" + + _rm "$BATS_TMPDIR/myfile.txt" + _rm_remote "/etc/myfile.txt" +} + +# bats test_tags=slow +@test "copy local file to container file (and rename)" { + _mkfile "$BATS_TMPDIR/myfile.txt" "foo" + + # rename + run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" app:/etc/myfile2.txt + assert_success + + run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile2.txt + assert_success + assert_output --partial "foo" + + _rm "$BATS_TMPDIR/myfile.txt" + _rm_remote "/etc/myfile2.txt" +} + +# bats test_tags=slow +@test "copy local directory to container directory (and creates missing directory)" { + _mkdir "$BATS_TMPDIR/mydir" + _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo" + + run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir" app:/etc + assert_success + + run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/mydir + assert_success + assert_output --partial "myfile.txt" + + _rm "$BATS_TMPDIR/mydir" + _rm_remote "/etc/mydir" +} + +# bats test_tags=slow +@test "copy local files to container directory" { + _mkdir "$BATS_TMPDIR/mydir" + _mkfile "$BATS_TMPDIR/mydir/myfile.txt" "foo" + _mkfile "$BATS_TMPDIR/mydir/myfile2.txt" "foo" + + run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/mydir/" app:/etc + assert_success + + run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile.txt + assert_success + assert_output --partial "myfile.txt" + + run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile2.txt + assert_success + assert_output --partial "myfile2.txt" + + _rm "$BATS_TMPDIR/mydir" + _rm_remote "/etc/myfile*" +} + +# bats test_tags=slow +@test "copy container file to local directory" { + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt" assert_success run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR" @@ -123,8 +170,76 @@ teardown(){ assert_exists "$BATS_TMPDIR/myfile.txt" assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo" - run rm -rf "$BATS_TMPDIR/myfile.txt" + _rm "$BATS_TMPDIR/myfile.txt" + _rm_remote "/etc/myfile.txt" +} + +# bats test_tags=slow +@test "copy container file to local file" { + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt" assert_success - _undeploy_app + run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile.txt" + assert_success + assert_exists "$BATS_TMPDIR/myfile.txt" + assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo" + + _rm "$BATS_TMPDIR/myfile.txt" + _rm_remote "/etc/myfile.txt" +} + +# bats test_tags=slow +@test "copy container file to local file and rename" { + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt" + assert_success + + run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/myfile.txt "$BATS_TMPDIR/myfile2.txt" + assert_success + assert_exists "$BATS_TMPDIR/myfile2.txt" + assert bash -c "cat $BATS_TMPDIR/myfile2.txt | grep -q foo" + + _rm "$BATS_TMPDIR/myfile2.txt" + _rm_remote "/etc/myfile.txt" +} + +# bats test_tags=slow +@test "copy container directory to local directory" { + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt" + assert_success + + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt" + assert_success + + mkdir "$BATS_TMPDIR/mydir" + + run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc "$BATS_TMPDIR/mydir" + assert_success + assert_exists "$BATS_TMPDIR/mydir/etc/myfile.txt" + assert_success + assert_exists "$BATS_TMPDIR/mydir/etc/myfile2.txt" + + _rm "$BATS_TMPDIR/mydir" + _rm_remote "/etc/myfile.txt" + _rm_remote "/etc/myfile2.txt" +} + +# bats test_tags=slow +@test "copy container files to local directory" { + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo foo > /etc/myfile.txt" + assert_success + + run $ABRA app run "$TEST_APP_DOMAIN" app bash -c "echo bar > /etc/myfile2.txt" + assert_success + + mkdir "$BATS_TMPDIR/mydir" + + run $ABRA app cp "$TEST_APP_DOMAIN" app:/etc/ "$BATS_TMPDIR/mydir" + assert_success + assert_exists "$BATS_TMPDIR/mydir/myfile.txt" + assert_success + assert_exists "$BATS_TMPDIR/mydir/myfile2.txt" + + _rm "$BATS_TMPDIR/mydir" + _rm_remote "/etc/myfile.txt" + _rm_remote "/etc/myfile2.txt" } diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats index 3f81d225..0df25675 100644 --- a/tests/integration/app_deploy.bats +++ b/tests/integration/app_deploy.bats @@ -16,6 +16,7 @@ teardown_file(){ setup(){ load "$PWD/tests/integration/helpers/common" _common_setup + _reset_recipe } teardown(){ @@ -82,13 +83,13 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + refute_output --regexp 'behind .* 3 commits' _reset_recipe _undeploy_app @@ -100,7 +101,7 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' # NOTE(d1): need to use --chaos to force same commit run $ABRA app deploy "$TEST_APP_DOMAIN" \ @@ -108,7 +109,7 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _undeploy_app _reset_recipe @@ -116,6 +117,9 @@ teardown(){ # bats test_tags=slow @test "deploy latest commit if no published versions and no --chaos" { + # TODO(d1): fix with a new test recipe which has no published versions? + skip "known issue, abra-test-recipe has published versions now" + latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)" _remove_tags @@ -140,7 +144,7 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)" @@ -273,6 +277,10 @@ teardown(){ } @test "ensure domain is checked" { + if [[ "$TEST_SERVER" == "default" ]]; then + skip "domain checks are disabled for local server" + fi + appDomain="custom-html.DOESNTEXIST" run $ABRA app new custom-html \ @@ -311,3 +319,54 @@ teardown(){ _undeploy_app } + +@test "bail out if specific version and chaos" { + run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \ + --chaos --no-input --no-converge-checks + assert_failure + assert_output --partial 'cannot use' +} + +# bats test_tags=slow +@test "COMPOSE_FILE with \$COMPOSE_FILE override works" { + _reset_recipe + + run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \ + "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + assert_success + + # NOTE(d1): --chaos used to bypass versions and access compose.extra_env.yml + run $ABRA app deploy "$TEST_APP_DOMAIN" \ + --no-input --no-converge-checks --chaos + assert_success + assert_output --partial "compose.yml" + assert_output --partial "compose.extra_env.yml" + + _undeploy_app + _reset_app +} + +@test "error if no secrets generated" { + run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.extra_secret.yml"/g' \ + "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + assert_success + + run sed -i 's/#SECRET_EXTRA_PASS_VERSION=v1/SECRET_EXTRA_PASS_VERSION=v1/g' \ + "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + assert_success + + run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks + assert_failure + assert_output --partial 'unable to deploy, secrets not generated' + + _reset_app +} + +@test "recipe config comments not present in values" { + run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input + assert_success + + run $ABRA app run "$TEST_APP_DOMAIN" app env + assert_success + refute_output --partial 'should be removed' +} diff --git a/tests/integration/app_new.bats b/tests/integration/app_new.bats index ac8c94fd..457ee152 100644 --- a/tests/integration/app_new.bats +++ b/tests/integration/app_new.bats @@ -18,9 +18,24 @@ setup(){ } teardown(){ + load "$PWD/tests/integration/helpers/common" _rm_app } +@test "autocomplete" { + run $ABRA app new --generate-bash-completion + assert_success + assert_output --partial "traefik" + assert_output --partial "abra-test-recipe" + + # Note: this test needs to be updated when a new version of the test recipe is published. + run $ABRA app new abra-test-recipe --generate-bash-completion + assert_success + assert_output "0.1.0+1.20.0 +0.1.1+1.20.2 +0.2.0+1.21.0" +} + @test "create new app" { run $ABRA app new "$TEST_RECIPE" \ --no-input \ @@ -28,10 +43,29 @@ teardown(){ --domain "$TEST_APP_DOMAIN" assert_success assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status + assert_output --partial "up to date" +} + +@test "create new app with version" { + run $ABRA app new "$TEST_RECIPE" 0.1.1+1.20.2 \ + --no-input \ + --server "$TEST_SERVER" \ + --domain "$TEST_APP_DOMAIN" + assert_success + assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" log -1 + assert_output --partial "453db7121c0a56a7a8f15378f18fe3bf21ccfdef" } @test "does not overwrite existing env files" { - _new_app + run $ABRA app new "$TEST_RECIPE" \ + --no-input \ + --server "$TEST_SERVER" \ + --domain "$TEST_APP_DOMAIN" + assert_success run $ABRA app new "$TEST_RECIPE" \ --no-input \ @@ -74,8 +108,7 @@ teardown(){ --no-input \ --chaos \ --server "$TEST_SERVER" \ - --domain "$TEST_APP_DOMAIN" \ - --secrets + --domain "$TEST_APP_DOMAIN" assert_success assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" @@ -88,18 +121,17 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA app new "$TEST_RECIPE" \ --no-input \ --server "$TEST_SERVER" \ - --domain "$TEST_APP_DOMAIN" \ - --secrets + --domain "$TEST_APP_DOMAIN" assert_success assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --partial "up to date" _reset_recipe } @@ -109,7 +141,7 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' # NOTE(d1): need to use --chaos to force same commit run $ABRA app new "$TEST_RECIPE" \ @@ -117,13 +149,12 @@ teardown(){ --offline \ --chaos \ --server "$TEST_SERVER" \ - --domain "$TEST_APP_DOMAIN" \ - --secrets + --domain "$TEST_APP_DOMAIN" assert_success assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _reset_recipe } diff --git a/tests/integration/app_remove.bats b/tests/integration/app_remove.bats index 8b53984c..c0be90da 100644 --- a/tests/integration/app_remove.bats +++ b/tests/integration/app_remove.bats @@ -104,10 +104,7 @@ teardown(){ _undeploy_app - # NOTE(d1): to let the stack come down before nuking volumes - sleep 5 - - run $ABRA app volume rm "$TEST_APP_DOMAIN" --force + run $ABRA app volume rm "$TEST_APP_DOMAIN" --no-input assert_success run $ABRA app volume ls "$TEST_APP_DOMAIN" @@ -132,9 +129,6 @@ teardown(){ _undeploy_app - # NOTE(d1): to let the stack come down before nuking volumes - sleep 5 - run $ABRA app rm "$TEST_APP_DOMAIN" --no-input assert_success assert_output --partial 'test-volume' diff --git a/tests/integration/app_restore.bats b/tests/integration/app_restore.bats index 77a63f63..f2cf52e8 100644 --- a/tests/integration/app_restore.bats +++ b/tests/integration/app_restore.bats @@ -109,13 +109,13 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' - run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST + run $ABRA app restore "$TEST_APP_DOMAIN" app assert_failure run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --partial "up to date" } @test "ensure recipe not up to date if --offline" { @@ -126,19 +126,19 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' - run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST --offline + run $ABRA app restore "$TEST_APP_DOMAIN" app --offline assert_failure run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit" assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --partial "HEAD detached at $latestCommit" } @test "error if missing service" { diff --git a/tests/integration/app_rollback.bats b/tests/integration/app_rollback.bats index d6cbbef1..b8200bbb 100644 --- a/tests/integration/app_rollback.bats +++ b/tests/integration/app_rollback.bats @@ -50,13 +50,13 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks assert_failure run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --partial "up to date" } @test "ensure recipe not up to date if --offline" { @@ -67,14 +67,14 @@ teardown(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA app rollback "$TEST_APP_DOMAIN" \ --no-input --no-converge-checks --offline assert_failure run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit" assert_success @@ -131,7 +131,7 @@ teardown(){ latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)" run $ABRA app deploy "$TEST_APP_DOMAIN" \ - --no-input --no-converge-checks --chaos + --no-input --chaos assert_success assert_output --partial "$latestCommit" assert_output --partial 'chaos' @@ -194,6 +194,13 @@ teardown(){ _undeploy_app } +@test "bail out if specific version and chaos" { + run $ABRA app rollback "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \ + --chaos --no-input --no-converge-checks + assert_failure + assert_output --partial 'cannot use' +} + # bats test_tags=slow @test "rollback to previous version" { run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks diff --git a/tests/integration/app_secret.bats b/tests/integration/app_secret.bats index 23451ad8..b745a939 100644 --- a/tests/integration/app_secret.bats +++ b/tests/integration/app_secret.bats @@ -8,7 +8,7 @@ setup_file(){ run $ABRA app new "$TEST_RECIPE" \ --no-input \ --server "$TEST_SERVER" \ - --domain "$TEST_APP_DOMAIN" \ + --domain "$TEST_APP_DOMAIN" assert_success assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" } @@ -19,26 +19,6 @@ teardown_file(){ _reset_recipe } -teardown(){ - # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888 - if [[ -z "${BATS_TEST_COMPLETED}" ]]; then - _undeploy_app - fi -} - -_reset_app(){ - run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" - assert_success - assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" - - run $ABRA app new "$TEST_RECIPE" \ - --no-input \ - --server "$TEST_SERVER" \ - --domain "$TEST_APP_DOMAIN" \ - assert_success - assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" -} - setup(){ load "$PWD/tests/integration/helpers/common" _common_setup @@ -123,10 +103,11 @@ setup(){ assert_success assert_output --partial 'test_pass_one' - run docker -c "$TEST_SERVER" secret ls + run bash -c '$ABRA app secret ls $TEST_APP_DOMAIN --machine | \ + jq -r ".[] | select(.name==\"test_pass_one\") | .version"' assert_success - assert_output --regexp ".*_test_pass_one_v2" - refute_output --regexp ".*_test_pass_one_v1" + assert_output --partial 'v2' + refute_output --partial 'v1' run $ABRA app secret rm "$TEST_APP_DOMAIN" --all assert_success @@ -338,6 +319,22 @@ setup(){ assert_success } +@test "ls: show secrets as machine readable" { + run $ABRA app secret ls "$TEST_APP_DOMAIN" + assert_success + assert_output --partial 'false' + + run $ABRA app secret generate "$TEST_APP_DOMAIN" --all + assert_success + + run $ABRA app secret ls "$TEST_APP_DOMAIN" --machine + assert_success + assert_output --partial '"created-on-server":"true"' + + run $ABRA app secret rm "$TEST_APP_DOMAIN" --all + assert_success +} + @test "ls: bail if unstaged changes and no --chaos" { run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo" assert_success diff --git a/tests/integration/app_upgrade.bats b/tests/integration/app_upgrade.bats index 376a75e2..0b1dafb8 100644 --- a/tests/integration/app_upgrade.bats +++ b/tests/integration/app_upgrade.bats @@ -47,6 +47,13 @@ teardown(){ _undeploy_app } +@test "bail out if specific version and chaos" { + run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" \ + --chaos --no-input --no-converge-checks + assert_failure + assert_output --partial 'cannot use' +} + # bats test_tags=slow @test "no upgrade if lint error" { _deploy_app diff --git a/tests/integration/app_version.bats b/tests/integration/app_version.bats index 2a6d6319..b11f58ef 100644 --- a/tests/integration/app_version.bats +++ b/tests/integration/app_version.bats @@ -59,6 +59,8 @@ teardown(){ # bats test_tags=slow @test "error if not in catalogue" { + skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6" + _deploy_app run $ABRA app version "$TEST_APP_DOMAIN" @@ -92,7 +94,7 @@ teardown(){ assert_success # NOTE(d1): to let the stack come down before nuking volumes - sleep 3 + sleep 5 run $ABRA app volume remove "$appDomain" --no-input assert_success diff --git a/tests/integration/app_volume.bats b/tests/integration/app_volume.bats index 5b4ff600..6327821c 100644 --- a/tests/integration/app_volume.bats +++ b/tests/integration/app_volume.bats @@ -78,9 +78,6 @@ teardown(){ _undeploy_app - # NOTE(d1): to let the stack come down before nuking volumes - sleep 5 - run $ABRA app volume rm "$TEST_APP_DOMAIN" --force assert_success assert_output --partial 'volumes removed successfully' @@ -92,9 +89,6 @@ teardown(){ _undeploy_app - # NOTE(d1): to let the stack come down before nuking volumes - sleep 5 - run $ABRA app volume rm "$TEST_APP_DOMAIN" --force assert_success assert_output --partial 'volumes removed successfully' diff --git a/tests/integration/helpers/app.bash b/tests/integration/helpers/app.bash index 17dbbc2f..32208b84 100644 --- a/tests/integration/helpers/app.bash +++ b/tests/integration/helpers/app.bash @@ -40,3 +40,16 @@ _rm_app() { run $ABRA app remove "$TEST_APP_DOMAIN" --no-input fi } + +_reset_app(){ + run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + assert_success + assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" + + run $ABRA app new "$TEST_RECIPE" \ + --no-input \ + --server "$TEST_SERVER" \ + --domain "$TEST_APP_DOMAIN" + assert_success + assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" +} diff --git a/tests/integration/helpers/common.bash b/tests/integration/helpers/common.bash index 379569ea..9f434436 100644 --- a/tests/integration/helpers/common.bash +++ b/tests/integration/helpers/common.bash @@ -1,10 +1,11 @@ #!/usr/bin/env bash _common_setup() { - load '/usr/lib/bats/bats-support/load' - load '/usr/lib/bats/bats-assert/load' - load '/usr/lib/bats/bats-file/load' + bats_load_library bats-support + bats_load_library bats-assert + bats_load_library bats-file + load "$PWD/tests/integration/helpers/file" load "$PWD/tests/integration/helpers/app" load "$PWD/tests/integration/helpers/git" load "$PWD/tests/integration/helpers/recipe" @@ -15,5 +16,5 @@ _common_setup() { export TEST_APP_NAME="$(basename "${BATS_TEST_FILENAME//./_}")" export TEST_APP_DOMAIN="$TEST_APP_NAME.$TEST_SERVER" - export TEST_RECIPE="abra-integration-test-recipe" + export TEST_RECIPE="abra-test-recipe" } diff --git a/tests/integration/helpers/file.bash b/tests/integration/helpers/file.bash new file mode 100644 index 00000000..5cb91869 --- /dev/null +++ b/tests/integration/helpers/file.bash @@ -0,0 +1,24 @@ +_mkfile() { + run bash -c "echo $2 > $1" + assert_success +} + +_mkfile_remote() { + run $ABRA app run "$TEST_APP_DOMAIN" app "bash -c \"echo $2 > $1\"" + assert_success +} + +_mkdir() { + run bash -c "mkdir -p $1" + assert_success +} + +_rm() { + run rm -rf "$1" + assert_success +} + +_rm_remote() { + run "$ABRA" app run "$TEST_APP_DOMAIN" app rm -rf "$1" + assert_success +} diff --git a/tests/integration/helpers/git.bash b/tests/integration/helpers/git.bash index f2654944..e11b76de 100644 --- a/tests/integration/helpers/git.bash +++ b/tests/integration/helpers/git.bash @@ -28,3 +28,10 @@ _reset_tags() { assert_success refute_output '0' } + +_set_git_author() { + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.email test@example.com + assert_success + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" config --local user.name test + assert_success +} diff --git a/tests/integration/helpers/server.bash b/tests/integration/helpers/server.bash index edeb9115..e3bcb320 100644 --- a/tests/integration/helpers/server.bash +++ b/tests/integration/helpers/server.bash @@ -1,13 +1,21 @@ #!/usr/bin/env bash _add_server() { - run $ABRA server add "$TEST_SERVER" + if [[ "$TEST_SERVER" == "default" ]]; then + run $ABRA server add -l + else + run $ABRA server add "$TEST_SERVER" + fi assert_success assert_exists "$ABRA_DIR/servers/$TEST_SERVER" } _rm_server() { - run $ABRA server remove --no-input "$TEST_SERVER" + if [[ "$TEST_SERVER" == "default" ]]; then + run rm -rf "$ABRA_DIR/servers/default" + else + run $ABRA server remove --no-input "$TEST_SERVER" + fi assert_success assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER" } diff --git a/tests/integration/recipe_diff.bats b/tests/integration/recipe_diff.bats new file mode 100644 index 00000000..fd384e35 --- /dev/null +++ b/tests/integration/recipe_diff.bats @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +setup() { + load "$PWD/tests/integration/helpers/common" + _common_setup +} + +@test "show unstaged changes" { + run $ABRA recipe diff "$TEST_RECIPE" + assert_success + refute_output --partial 'traefik.enable' + + run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml" + assert_success + + run $ABRA recipe diff "$TEST_RECIPE" + assert_success + assert_output --partial 'traefik.enable' + + _reset_recipe +} diff --git a/tests/integration/recipe_fetch.bats b/tests/integration/recipe_fetch.bats index a64306ab..1cc61e7c 100644 --- a/tests/integration/recipe_fetch.bats +++ b/tests/integration/recipe_fetch.bats @@ -5,7 +5,17 @@ setup() { _common_setup } -@test "recipe fetch" { +@test "recipe fetch all" { + run rm -rf "$ABRA_DIR/recipes/matrix-synapse" + assert_success + assert_not_exists "$ABRA_DIR/recipes/matrix-synapse" + + run $ABRA recipe fetch + assert_success + assert_exists "$ABRA_DIR/recipes/matrix-synapse" +} + +@test "recipe fetch single recipe" { run rm -rf "$ABRA_DIR/recipes/matrix-synapse" assert_success assert_not_exists "$ABRA_DIR/recipes/matrix-synapse" diff --git a/tests/integration/recipe_lint.bats b/tests/integration/recipe_lint.bats index 0b2617b4..401ab8fd 100644 --- a/tests/integration/recipe_lint.bats +++ b/tests/integration/recipe_lint.bats @@ -66,13 +66,13 @@ setup() { assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA recipe lint "$TEST_RECIPE" assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _reset_recipe } @@ -82,13 +82,13 @@ setup() { assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA recipe lint "$TEST_RECIPE" --offline assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _reset_recipe } diff --git a/tests/integration/recipe_release.bats b/tests/integration/recipe_release.bats index a847206c..4c10f691 100644 --- a/tests/integration/recipe_release.bats +++ b/tests/integration/recipe_release.bats @@ -15,6 +15,11 @@ teardown_file(){ setup(){ load "$PWD/tests/integration/helpers/common" _common_setup + _set_git_author +} + +teardown() { + _reset_recipe } @test "validate recipe argument" { @@ -51,8 +56,6 @@ setup(){ run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list assert_success assert_output --partial '0.2.1+1.21.6' - - _reset_recipe } # NOTE(d1): this test can't assert hardcoded versions since we upgrade a minor @@ -81,6 +84,38 @@ setup(){ run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list assert_success assert_output --regexp '0\.3\.0\+1\.2.*' - - _reset_recipe "$TEST_RECIPE" +} + +@test "unknown files not committed" { + run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch + assert_success + + run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"' + assert_success + assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo" + + run $ABRA recipe release "$TEST_RECIPE" --no-input --patch + assert_success + assert_output --partial 'no -p/--publish passed, not publishing' + + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rm foo + assert_failure + assert_output --partial "fatal: pathspec 'foo' did not match any files" +} + +# NOTE: relies on 0.2.x being the last minor version +@test "release with next release note" { + _mkfile "$ABRA_DIR/recipes/$TEST_RECIPE/release/next" "those are some release notes for the next release" + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add release/next + assert_success + run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "added some release notes" + assert_success + + run $ABRA recipe release "$TEST_RECIPE" --no-input --minor + assert_success + assert_output --partial 'no -p/--publish passed, not publishing' + + assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/next" + assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0" + assert_file_contains "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0" "those are some release notes for the next release" } diff --git a/tests/integration/recipe_reset.bats b/tests/integration/recipe_reset.bats new file mode 100644 index 00000000..22b46f61 --- /dev/null +++ b/tests/integration/recipe_reset.bats @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +setup() { + load "$PWD/tests/integration/helpers/common" + _common_setup +} + +@test "reset unstaged changes" { + run $ABRA recipe fetch "$TEST_RECIPE" + assert_success + + run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml" + assert_success + + run $ABRA recipe diff "$TEST_RECIPE" + assert_success + assert_output --partial 'traefik.enable' + + run $ABRA recipe reset "$TEST_RECIPE" + assert_success + + run $ABRA recipe diff "$TEST_RECIPE" + assert_success + refute_output --partial 'traefik.enable' +} diff --git a/tests/integration/recipe_upgrade.bats b/tests/integration/recipe_upgrade.bats index 183d1f22..a36efe26 100644 --- a/tests/integration/recipe_upgrade.bats +++ b/tests/integration/recipe_upgrade.bats @@ -61,14 +61,14 @@ setup(){ assert_success run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - assert_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' run $ABRA recipe upgrade "$TEST_RECIPE" --no-input assert_success assert_output --partial 'can upgrade service: app' run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status - refute_output --partial 'behind 3' + assert_output --regexp 'behind .* 3 commits' _reset_recipe } diff --git a/tests/integration/recipe_version.bats b/tests/integration/recipe_version.bats index 42d2e4d0..ebc5bad9 100644 --- a/tests/integration/recipe_version.bats +++ b/tests/integration/recipe_version.bats @@ -12,6 +12,8 @@ setup() { } @test "error if not present in catalogue" { + skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6" + run $ABRA recipe versions "$TEST_RECIPE" assert_failure assert_output --partial "is not published on the catalogue"