From 2c515ce70adbcb808895b8bb1efd6036e51092c1 Mon Sep 17 00:00:00 2001 From: decentral1se Date: Tue, 12 Mar 2024 10:03:42 +0100 Subject: [PATCH] Revert "feat: backup revolution" This reverts commit c5687dfbd7413f87ad202a4a275da495edae4730. This is a temporary measure to facilitate a release which won't completely explode peoples workflows (missing command logic). We re-instate this commit after the first 0.9.x release. --- cli/app/backup.go | 674 +++++++++++++++++++-------------- cli/app/cp.go | 16 +- cli/app/restore.go | 197 ++++++++-- cli/app/run.go | 2 +- cli/internal/backup.go | 86 ++--- cli/internal/command.go | 4 +- pkg/config/env.go | 2 - pkg/container/container.go | 2 +- pkg/service/service.go | 64 ---- pkg/upstream/container/exec.go | 23 +- 10 files changed, 614 insertions(+), 456 deletions(-) diff --git a/cli/app/backup.go b/cli/app/backup.go index f6c8d2ae..f9a239b1 100644 --- a/cli/app/backup.go +++ b/cli/app/backup.go @@ -1,296 +1,414 @@ package app import ( + "archive/tar" + "context" "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/recipe" + "coopcloud.tech/abra/pkg/config" + containerPkg "coopcloud.tech/abra/pkg/container" + recipePkg "coopcloud.tech/abra/pkg/recipe" + "coopcloud.tech/abra/pkg/upstream/container" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + dockerClient "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" + "github.com/klauspost/pgzip" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -var snapshot string -var snapshotFlag = &cli.StringFlag{ - Name: "snapshot, s", - Usage: "Lists specific snapshot", - Destination: &snapshot, -} - -var includePath string -var includePathFlag = &cli.StringFlag{ - Name: "path, p", - Usage: "Include path", - Destination: &includePath, -} - -var resticRepo string -var resticRepoFlag = &cli.StringFlag{ - Name: "repo, r", - Usage: "Restic repository", - Destination: &resticRepo, -} - -var appBackupListCommand = cli.Command{ - Name: "list", - Aliases: []string{"ls"}, - Flags: []cli.Flag{ - internal.DebugFlag, - internal.OfflineFlag, - snapshotFlag, - includePathFlag, - }, - Before: internal.SubCommandBefore, - Usage: "List all backups", - BashComplete: autocomplete.AppNameComplete, - Action: func(c *cli.Context) error { - app := internal.ValidateApp(c) - - if err := recipe.EnsureExists(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Chaos { - if err := recipe.EnsureIsClean(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Offline { - if err := recipe.EnsureUpToDate(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - if err := recipe.EnsureLatest(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - cl, err := client.New(app.Server) - if err != nil { - logrus.Fatal(err) - } - - targetContainer, err := internal.RetrieveBackupBotContainer(cl) - if err != nil { - logrus.Fatal(err) - } - - execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} - if snapshot != "" { - logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) - execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) - } - if includePath != "" { - logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath) - execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath)) - } - - if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil { - logrus.Fatal(err) - } - - return nil - }, -} - -var appBackupDownloadCommand = cli.Command{ - Name: "download", - Aliases: []string{"d"}, - Flags: []cli.Flag{ - internal.DebugFlag, - internal.OfflineFlag, - snapshotFlag, - includePathFlag, - }, - Before: internal.SubCommandBefore, - Usage: "Download a backup", - BashComplete: autocomplete.AppNameComplete, - Action: func(c *cli.Context) error { - app := internal.ValidateApp(c) - - if err := recipe.EnsureExists(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Chaos { - if err := recipe.EnsureIsClean(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Offline { - if err := recipe.EnsureUpToDate(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - if err := recipe.EnsureLatest(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - cl, err := client.New(app.Server) - if err != nil { - logrus.Fatal(err) - } - - targetContainer, err := internal.RetrieveBackupBotContainer(cl) - if err != nil { - logrus.Fatal(err) - } - - execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} - if snapshot != "" { - logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) - execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) - } - if includePath != "" { - logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath) - execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath)) - } - - if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil { - logrus.Fatal(err) - } - - remoteBackupDir := "/tmp/backup.tar.gz" - currentWorkingDir := "." - if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil { - logrus.Fatal(err) - } - - fmt.Println("backup successfully downloaded to current working directory") - - return nil - }, -} - -var appBackupCreateCommand = cli.Command{ - Name: "create", - Aliases: []string{"c"}, - Flags: []cli.Flag{ - internal.DebugFlag, - internal.OfflineFlag, - resticRepoFlag, - }, - Before: internal.SubCommandBefore, - Usage: "Create a new backup", - BashComplete: autocomplete.AppNameComplete, - Action: func(c *cli.Context) error { - app := internal.ValidateApp(c) - - if err := recipe.EnsureExists(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Chaos { - if err := recipe.EnsureIsClean(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Offline { - if err := recipe.EnsureUpToDate(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - if err := recipe.EnsureLatest(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - cl, err := client.New(app.Server) - if err != nil { - logrus.Fatal(err) - } - - targetContainer, err := internal.RetrieveBackupBotContainer(cl) - if err != nil { - logrus.Fatal(err) - } - - execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} - if resticRepo != "" { - logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo) - execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo)) - } - - if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil { - logrus.Fatal(err) - } - - return nil - }, -} - -var appBackupSnapshotsCommand = cli.Command{ - Name: "snapshots", - Aliases: []string{"s"}, - Flags: []cli.Flag{ - internal.DebugFlag, - internal.OfflineFlag, - snapshotFlag, - }, - Before: internal.SubCommandBefore, - Usage: "List backup snapshots", - BashComplete: autocomplete.AppNameComplete, - Action: func(c *cli.Context) error { - app := internal.ValidateApp(c) - - if err := recipe.EnsureExists(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Chaos { - if err := recipe.EnsureIsClean(app.Recipe); err != nil { - logrus.Fatal(err) - } - - if !internal.Offline { - if err := recipe.EnsureUpToDate(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - if err := recipe.EnsureLatest(app.Recipe); err != nil { - logrus.Fatal(err) - } - } - - cl, err := client.New(app.Server) - if err != nil { - logrus.Fatal(err) - } - - targetContainer, err := internal.RetrieveBackupBotContainer(cl) - if err != nil { - logrus.Fatal(err) - } - - execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} - if snapshot != "" { - logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) - execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) - } - - if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil { - logrus.Fatal(err) - } - - return nil - }, +type backupConfig struct { + preHookCmd string + postHookCmd string + backupPaths []string } var appBackupCommand = cli.Command{ Name: "backup", - Aliases: []string{"b"}, - Usage: "Manage app backups", - ArgsUsage: "", - Subcommands: []cli.Command{ - appBackupListCommand, - appBackupSnapshotsCommand, - appBackupDownloadCommand, - appBackupCreateCommand, + Aliases: []string{"bk"}, + Usage: "Run app backup", + ArgsUsage: " []", + Flags: []cli.Flag{ + internal.DebugFlag, + internal.OfflineFlag, + internal.ChaosFlag, + }, + Before: internal.SubCommandBefore, + BashComplete: autocomplete.AppNameComplete, + Description: ` +Run an app backup. + +A backup command and pre/post hook commands are defined in the recipe +configuration. Abra reads this configuration and run the comands in the context +of the deployed services. Pass if you only want to back up a single +service. All backups are placed in the ~/.abra/backups directory. + +A single backup file is produced for all backup paths specified for a service. +If we have the following backup configuration: + + - "backupbot.backup.path=/var/lib/foo,/var/lib/bar" + +And we run "abra app backup example.com app", Abra will produce a file that +looks like: + + ~/.abra/backups/example_com_app_609341138.tar.gz + +This file is a compressed archive which contains all backup paths. To see paths, run: + + tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz + +(Make sure to change the name of the backup file) + +This single file can be used to restore your app. See "abra app restore" for more. +`, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) + + recipe, err := recipePkg.Get(app.Recipe, internal.Offline) + if err != nil { + logrus.Fatal(err) + } + + if !internal.Chaos { + if err := recipePkg.EnsureIsClean(app.Recipe); err != nil { + logrus.Fatal(err) + } + + if !internal.Offline { + if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + if err := recipePkg.EnsureLatest(app.Recipe); err != nil { + logrus.Fatal(err) + } + } + + backupConfigs := make(map[string]backupConfig) + for _, service := range recipe.Config.Services { + if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok { + if backupsEnabled == "true" { + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) + bkConfig := backupConfig{} + + logrus.Debugf("backup config detected for %s", fullServiceName) + + if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok { + logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths) + bkConfig.backupPaths = strings.Split(paths, ",") + } + + if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok { + logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) + bkConfig.preHookCmd = preHookCmd + } + + if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok { + logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) + bkConfig.postHookCmd = postHookCmd + } + + backupConfigs[service.Name] = bkConfig + } + } + } + + cl, err := client.New(app.Server) + if err != nil { + logrus.Fatal(err) + } + + serviceName := c.Args().Get(1) + if serviceName != "" { + backupConfig, ok := backupConfigs[serviceName] + if !ok { + logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName) + } + + logrus.Infof("running backup for the %s service", serviceName) + + if err := runBackup(cl, app, serviceName, backupConfig); err != nil { + logrus.Fatal(err) + } + } else { + if len(backupConfigs) == 0 { + logrus.Fatalf("no backup configs discovered for %s?", app.Name) + } + + for serviceName, backupConfig := range backupConfigs { + logrus.Infof("running backup for the %s service", serviceName) + + if err := runBackup(cl, app, serviceName, backupConfig); err != nil { + logrus.Fatal(err) + } + } + } + + return nil }, } + +// TimeStamp generates a file name friendly timestamp. +func TimeStamp() string { + ts := time.Now().UTC().Format(time.RFC3339) + return strings.Replace(ts, ":", "-", -1) +} + +// runBackup does the actual backup logic. +func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error { + if len(bkConfig.backupPaths) == 0 { + return fmt.Errorf("backup paths are empty for %s?", serviceName) + } + + // FIXME: avoid instantiating a new CLI + dcli, err := command.NewDockerCli() + if err != nil { + return err + } + + filters := filters.NewArgs() + filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) + + targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) + if err != nil { + return err + } + + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) + if bkConfig.preHookCmd != "" { + splitCmd := internal.SafeSplit(bkConfig.preHookCmd) + + logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) + + preHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { + return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error()) + } + + logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd) + } + + var tempBackupPaths []string + for _, remoteBackupPath := range bkConfig.backupPaths { + sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_") + localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp())) + logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) + + logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath) + + content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath) + if err != nil { + logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) + } + defer content.Close() + + _, srcBase := archive.SplitPathDirEntry(remoteBackupPath) + preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath) + if err := copyToFile(localBackupPath, preArchive); err != nil { + logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) + } + + tempBackupPaths = append(tempBackupPaths, localBackupPath) + } + + logrus.Infof("compressing and merging archives...") + + if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil { + logrus.Debugf("failed to merge archive files: %s", err.Error()) + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + return fmt.Errorf("failed to merge archive files: %s", err.Error()) + } + + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + + if bkConfig.postHookCmd != "" { + splitCmd := internal.SafeSplit(bkConfig.postHookCmd) + + logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) + + postHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd) + } + + return nil +} + +func copyToFile(outfile string, r io.Reader) error { + tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp") + if err != nil { + return err + } + + tmpPath := tmpFile.Name() + + _, err = io.Copy(tmpFile, r) + tmpFile.Close() + + if err != nil { + os.Remove(tmpPath) + return err + } + + if err = os.Rename(tmpPath, outfile); err != nil { + os.Remove(tmpPath) + return err + } + + return nil +} + +func cleanupTempArchives(tarPaths []string) error { + for _, tarPath := range tarPaths { + if err := os.RemoveAll(tarPath); err != nil { + return err + } + + logrus.Debugf("remove temporary archive file %s", tarPath) + } + + return nil +} + +func mergeArchives(tarPaths []string, serviceName string) error { + var out io.Writer + var cout *pgzip.Writer + + localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp())) + + fout, err := os.Create(localBackupPath) + if err != nil { + return fmt.Errorf("Failed to open %s: %s", localBackupPath, err) + } + + defer fout.Close() + out = fout + + cout = pgzip.NewWriter(out) + out = cout + + tw := tar.NewWriter(out) + + for _, tarPath := range tarPaths { + if err := addTar(tw, tarPath); err != nil { + return fmt.Errorf("failed to merge %s: %v", tarPath, err) + } + } + + if err := tw.Close(); err != nil { + return fmt.Errorf("failed to close tar writer %v", err) + } + + if cout != nil { + if err := cout.Flush(); err != nil { + return fmt.Errorf("failed to flush: %s", err) + } else if err = cout.Close(); err != nil { + return fmt.Errorf("failed to close compressed writer: %s", err) + } + } + + logrus.Infof("backed up %s to %s", serviceName, localBackupPath) + + return nil +} + +func addTar(tw *tar.Writer, pth string) (err error) { + var tr *tar.Reader + var rc io.ReadCloser + var hdr *tar.Header + + if tr, rc, err = openTarFile(pth); err != nil { + return + } + + for { + if hdr, err = tr.Next(); err != nil { + if err == io.EOF { + err = nil + } + break + } + if err = tw.WriteHeader(hdr); err != nil { + break + } else if _, err = io.Copy(tw, tr); err != nil { + break + } + } + if err == nil { + err = rc.Close() + } else { + rc.Close() + } + return +} + +func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) { + var fin *os.File + var n int + buff := make([]byte, 1024) + + if fin, err = os.Open(pth); err != nil { + return + } + + if n, err = fin.Read(buff); err != nil { + fin.Close() + return + } else if n == 0 { + fin.Close() + err = fmt.Errorf("%s is empty", pth) + return + } + + if _, err = fin.Seek(0, 0); err != nil { + fin.Close() + return + } + + rc = fin + tr = tar.NewReader(rc) + + return tr, rc, nil +} diff --git a/cli/app/cp.go b/cli/app/cp.go index f27b42ce..bfc2c789 100644 --- a/cli/app/cp.go +++ b/cli/app/cp.go @@ -76,9 +76,9 @@ And if you want to copy that file back to your current working directory locally logrus.Debugf("retrieved %s as target container on %s", formatter.ShortenID(container.ID), app.Server) if toContainer { - err = CopyToContainer(cl, container.ID, srcPath, dstPath) + err = copyToContainer(cl, container.ID, srcPath, dstPath) } else { - err = CopyFromContainer(cl, container.ID, srcPath, dstPath) + err = copyFromContainer(cl, container.ID, srcPath, dstPath) } if err != nil { logrus.Fatal(err) @@ -106,9 +106,9 @@ func parseSrcAndDst(src, dst string) (srcPath string, dstPath string, service st return "", "", "", false, errServiceMissing } -// CopyToContainer copies a file or directory from the local file system to the container. +// copyToContainer copies a file or directory from the local file system to the container. // See the possible copy modes and their documentation. -func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error { +func copyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error { srcStat, err := os.Stat(srcPath) if err != nil { return fmt.Errorf("local %s ", err) @@ -140,7 +140,7 @@ func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri if err != nil { return err } - if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{ + if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{ AttachStderr: true, AttachStdin: true, AttachStdout: true, @@ -179,7 +179,7 @@ func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri if err != nil { return err } - if _, err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{ + if err := container.RunExec(dcli, cl, containerID, &types.ExecConfig{ AttachStderr: true, AttachStdin: true, AttachStdout: true, @@ -194,9 +194,9 @@ func CopyToContainer(cl *dockerClient.Client, containerID, srcPath, dstPath stri return nil } -// CopyFromContainer copies a file or directory from the given container to the local file system. +// copyFromContainer copies a file or directory from the given container to the local file system. // See the possible copy modes and their documentation. -func CopyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error { +func copyFromContainer(cl *dockerClient.Client, containerID, srcPath, dstPath string) error { srcStat, err := cl.ContainerStatPath(context.Background(), containerID, srcPath) if err != nil { if errdefs.IsNotFound(err) { diff --git a/cli/app/restore.go b/cli/app/restore.go index c80347f5..1bf9c840 100644 --- a/cli/app/restore.go +++ b/cli/app/restore.go @@ -1,82 +1,223 @@ package app import ( + "context" + "errors" "fmt" + "os" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/config" + containerPkg "coopcloud.tech/abra/pkg/container" "coopcloud.tech/abra/pkg/recipe" + recipePkg "coopcloud.tech/abra/pkg/recipe" + "coopcloud.tech/abra/pkg/upstream/container" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + dockerClient "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -var targetPath string -var targetPathFlag = &cli.StringFlag{ - Name: "target, t", - Usage: "Target path", - Destination: &targetPath, +type restoreConfig struct { + preHookCmd string + postHookCmd string } var appRestoreCommand = cli.Command{ Name: "restore", Aliases: []string{"rs"}, - Usage: "Restore an app backup", - ArgsUsage: " ", + Usage: "Run app restore", + ArgsUsage: " ", Flags: []cli.Flag{ internal.DebugFlag, internal.OfflineFlag, - targetPathFlag, + internal.ChaosFlag, }, Before: internal.SubCommandBefore, BashComplete: autocomplete.AppNameComplete, + Description: ` +Run an app restore. + +Pre/post hook commands are defined in the recipe configuration. Abra reads this +configuration and run the comands in the context of the service before +restoring the backup. + +Unlike "abra app backup", restore must be run on a per-service basis. You can +not restore all services in one go. Backup files produced by Abra are +compressed archives which use absolute paths. This allows Abra to restore +according to standard tar command logic, i.e. the backup will be restored to +the path it was originally backed up from. + +Example: + + abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz +`, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) - if err := recipe.EnsureExists(app.Recipe); err != nil { + recipe, err := recipe.Get(app.Recipe, internal.Offline) + if err != nil { logrus.Fatal(err) } if !internal.Chaos { - if err := recipe.EnsureIsClean(app.Recipe); err != nil { + if err := recipePkg.EnsureIsClean(app.Recipe); err != nil { logrus.Fatal(err) } if !internal.Offline { - if err := recipe.EnsureUpToDate(app.Recipe); err != nil { + if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil { logrus.Fatal(err) } } - if err := recipe.EnsureLatest(app.Recipe); err != nil { + if err := recipePkg.EnsureLatest(app.Recipe); err != nil { logrus.Fatal(err) } } + serviceName := c.Args().Get(1) + if serviceName == "" { + internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) + } + + backupPath := c.Args().Get(2) + if backupPath == "" { + internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) + } + + if _, err := os.Stat(backupPath); err != nil { + if os.IsNotExist(err) { + logrus.Fatalf("%s doesn't exist?", backupPath) + } + } + + restoreConfigs := make(map[string]restoreConfig) + for _, service := range recipe.Config.Services { + if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok { + if restoreEnabled == "true" { + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) + rsConfig := restoreConfig{} + + logrus.Debugf("restore config detected for %s", fullServiceName) + + if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok { + logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) + rsConfig.preHookCmd = preHookCmd + } + + if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok { + logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) + rsConfig.postHookCmd = postHookCmd + } + + restoreConfigs[service.Name] = rsConfig + } + } + } + + rsConfig, ok := restoreConfigs[serviceName] + if !ok { + rsConfig = restoreConfig{} + } + cl, err := client.New(app.Server) if err != nil { logrus.Fatal(err) } - targetContainer, err := internal.RetrieveBackupBotContainer(cl) - if err != nil { - logrus.Fatal(err) - } - - execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)} - if snapshot != "" { - logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) - execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) - } - if targetPath != "" { - logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath) - execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath)) - } - - if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil { + if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil { logrus.Fatal(err) } return nil }, } + +// runRestore does the actual restore logic. +func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error { + // FIXME: avoid instantiating a new CLI + dcli, err := command.NewDockerCli() + if err != nil { + return err + } + + filters := filters.NewArgs() + filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) + + targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) + if err != nil { + return err + } + + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) + if rsConfig.preHookCmd != "" { + splitCmd := internal.SafeSplit(rsConfig.preHookCmd) + + logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) + + preHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd) + } + + backupReader, err := os.Open(backupPath) + if err != nil { + return err + } + + content, err := archive.DecompressStream(backupReader) + if err != nil { + return err + } + + // NOTE(d1): we use absolute paths so tar knows what to do. it will restore + // files according to the paths set in the compressed archive + restorePath := "/" + + copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false} + if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil { + return err + } + + logrus.Infof("restored %s to %s", backupPath, fullServiceName) + + if rsConfig.postHookCmd != "" { + splitCmd := internal.SafeSplit(rsConfig.postHookCmd) + + logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) + + postHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd) + } + + return nil +} diff --git a/cli/app/run.go b/cli/app/run.go index b5e0a9ce..4ae68c1b 100644 --- a/cli/app/run.go +++ b/cli/app/run.go @@ -91,7 +91,7 @@ var appRunCommand = cli.Command{ logrus.Fatal(err) } - if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { + if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { logrus.Fatal(err) } diff --git a/cli/internal/backup.go b/cli/internal/backup.go index 530735c9..79951810 100644 --- a/cli/internal/backup.go +++ b/cli/internal/backup.go @@ -1,67 +1,35 @@ package internal import ( - "context" - - "coopcloud.tech/abra/pkg/config" - containerPkg "coopcloud.tech/abra/pkg/container" - "coopcloud.tech/abra/pkg/service" - "coopcloud.tech/abra/pkg/upstream/container" - "github.com/docker/cli/cli/command" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - dockerClient "github.com/docker/docker/client" - "github.com/sirupsen/logrus" + "strings" ) -// RetrieveBackupBotContainer gets the deployed backupbot container. -func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error) { - ctx := context.Background() - chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput) - if err != nil { - return types.Container{}, err +// SafeSplit splits up a string into a list of commands safely. +func SafeSplit(s string) []string { + split := strings.Split(s, " ") + + var result []string + var inquote string + var block string + for _, i := range split { + if inquote == "" { + if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") { + inquote = string(i[0]) + block = strings.TrimPrefix(i, inquote) + " " + } else { + result = append(result, i) + } + } else { + if !strings.HasSuffix(i, inquote) { + block += i + " " + } else { + block += strings.TrimSuffix(i, inquote) + inquote = "" + result = append(result, block) + block = "" + } + } } - logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name) - - filters := filters.NewArgs() - filters.Add("name", chosenService.Spec.Name) - targetContainer, err := containerPkg.GetContainer( - ctx, - cl, - filters, - NoInput, - ) - if err != nil { - return types.Container{}, err - } - - return targetContainer, nil -} - -// RunBackupCmdRemote runs a backup related command on a remote backupbot container. -func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error { - execBackupListOpts := types.ExecConfig{ - AttachStderr: true, - AttachStdin: true, - AttachStdout: true, - Cmd: []string{"/usr/bin/backup", "--", backupCmd}, - Detach: false, - Env: execEnv, - Tty: true, - } - - logrus.Debugf("running backup %s on %s with exec config %v", backupCmd, containerID, execBackupListOpts) - - // FIXME: avoid instantiating a new CLI - dcli, err := command.NewDockerCli() - if err != nil { - return err - } - - if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil { - return err - } - - return nil + return result } diff --git a/cli/internal/command.go b/cli/internal/command.go index 13c007be..6f02ae1c 100644 --- a/cli/internal/command.go +++ b/cli/internal/command.go @@ -60,7 +60,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, Tty: false, } - if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { + if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { logrus.Infof("%s does not exist for %s, use /bin/sh as fallback", shell, app.Name) shell = "/bin/sh" } @@ -85,7 +85,7 @@ func RunCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, execCreateOpts.Tty = false } - if _, err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { + if err := container.RunExec(dcli, cl, targetContainer.ID, &execCreateOpts); err != nil { return err } diff --git a/pkg/config/env.go b/pkg/config/env.go index adedc6f4..62f6a71d 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -36,8 +36,6 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud" var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json" var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git" -var BackupbotLabel = "coop-cloud.backupbot.enabled" - // envVarModifiers is a list of env var modifier strings. These are added to // env vars as comments and modify their processing by Abra, e.g. determining // how long secrets should be. diff --git a/pkg/container/container.go b/pkg/container/container.go index 1354b0dd..09d5703b 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter) } - if len(containers) > 1 { + if len(containers) != 1 { var containersRaw []string for _, container := range containers { containerName := strings.Join(container.Names, " ") diff --git a/pkg/service/service.go b/pkg/service/service.go index 48cdce75..3d92d821 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -14,70 +14,6 @@ import ( "github.com/sirupsen/logrus" ) -// GetService retrieves a service container based on a label. If prompt is true -// and the retrievd count of service containers does not match 1, then a prompt -// is presented to let the user choose. An error is returned when no service is -// found. -func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) { - services, err := cl.ServiceList(c, types.ServiceListOptions{}) - if err != nil { - return swarm.Service{}, err - } - - if len(services) == 0 { - return swarm.Service{}, fmt.Errorf("no services deployed?") - } - - var matchingServices []swarm.Service - for _, service := range services { - if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" { - matchingServices = append(matchingServices, service) - } - } - - if len(matchingServices) == 0 { - return swarm.Service{}, fmt.Errorf("no services deployed matching label '%s'?", label) - } - - if len(matchingServices) > 1 { - var servicesRaw []string - for _, service := range matchingServices { - serviceName := service.Spec.Name - created := formatter.HumanDuration(service.CreatedAt.Unix()) - servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created)) - } - - if !prompt { - err := fmt.Errorf("expected 1 service but found %v: %s", len(matchingServices), strings.Join(servicesRaw, " ")) - return swarm.Service{}, err - } - - logrus.Warnf("ambiguous service list received, prompting for input") - - var response string - prompt := &survey.Select{ - Message: "which service are you looking for?", - Options: servicesRaw, - } - - if err := survey.AskOne(prompt, &response); err != nil { - return swarm.Service{}, err - } - - chosenService := strings.TrimSpace(strings.Split(response, " ")[0]) - for _, service := range matchingServices { - serviceName := strings.ToLower(service.Spec.Name) - if serviceName == chosenService { - return service, nil - } - } - - logrus.Panic("failed to match chosen service") - } - - return matchingServices[0], nil -} - // GetService retrieves a service container. If prompt is true and the retrievd // count of service containers does not match 1, then a prompt is presented to // let the user choose. A count of 0 is handled gracefully. diff --git a/pkg/upstream/container/exec.go b/pkg/upstream/container/exec.go index 82a2c570..e811481a 100644 --- a/pkg/upstream/container/exec.go +++ b/pkg/upstream/container/exec.go @@ -13,10 +13,7 @@ import ( "github.com/sirupsen/logrus" ) -// RunExec runs a command on a remote container. io.Writer corresponds to the -// command output. -func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, - execConfig *types.ExecConfig) (io.Writer, error) { +func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error { ctx := context.Background() // We need to check the tty _before_ we do the ContainerExecCreate, because @@ -24,22 +21,22 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string // there's no easy way to clean those up). But also in order to make "not // exist" errors take precedence we do a dummy inspect first. if _, err := client.ContainerInspect(ctx, containerID); err != nil { - return nil, err + return err } if !execConfig.Detach { if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil { - return nil, err + return err } } response, err := client.ContainerExecCreate(ctx, containerID, *execConfig) if err != nil { - return nil, err + return err } execID := response.ID if execID == "" { - return nil, errors.New("exec ID empty") + return errors.New("exec ID empty") } if execConfig.Detach { @@ -47,13 +44,13 @@ func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string Detach: execConfig.Detach, Tty: execConfig.Tty, } - return nil, client.ContainerExecStart(ctx, execID, execStartCheck) + return client.ContainerExecStart(ctx, execID, execStartCheck) } return interactiveExec(ctx, dockerCli, client, execConfig, execID) } func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client, - execConfig *types.ExecConfig, execID string) (io.Writer, error) { + execConfig *types.ExecConfig, execID string) error { // Interactive exec requested. var ( out, stderr io.Writer @@ -79,7 +76,7 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie } resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck) if err != nil { - return out, err + return err } defer resp.Close() @@ -110,10 +107,10 @@ func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclie if err := <-errCh; err != nil { logrus.Debugf("Error hijack: %s", err) - return out, err + return err } - return out, getExecExitStatus(ctx, client, execID) + return getExecExitStatus(ctx, client, execID) } func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {