From b27acb2f616dd3f788cd791be950e3f0cdd0ea21 Mon Sep 17 00:00:00 2001 From: decentral1se Date: Thu, 31 Mar 2022 01:53:11 +0200 Subject: [PATCH] feat: backup/restore [ci skip] See https://git.coopcloud.tech/coop-cloud/organising/issues/30. --- cli/app/app.go | 2 + cli/app/backup.go | 205 +++++++++++++++++++++++++++++++++++++++++ cli/app/restore.go | 197 +++++++++++++++++++++++++++++++++++++++ cli/cli.go | 1 + cli/internal/backup.go | 35 +++++++ pkg/config/env.go | 1 + 6 files changed, 441 insertions(+) create mode 100644 cli/app/backup.go create mode 100644 cli/app/restore.go create mode 100644 cli/internal/backup.go diff --git a/cli/app/app.go b/cli/app/app.go index 2451378a..bb2e246c 100644 --- a/cli/app/app.go +++ b/cli/app/app.go @@ -30,5 +30,7 @@ var AppCommand = cli.Command{ appVersionCommand, appErrorsCommand, appCmdCommand, + appBackupCommand, + appRestoreCommand, }, } diff --git a/cli/app/backup.go b/cli/app/backup.go new file mode 100644 index 00000000..d8321525 --- /dev/null +++ b/cli/app/backup.go @@ -0,0 +1,205 @@ +package app + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "time" + + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/config" + containerPkg "coopcloud.tech/abra/pkg/container" + "coopcloud.tech/abra/pkg/recipe" + "coopcloud.tech/abra/pkg/upstream/container" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +type backupConfig struct { + preHookCmd string + postHookCmd string + backupPaths []string +} + +var appBackupCommand = cli.Command{ + Name: "backup", + Aliases: []string{"bk"}, + Usage: "Run app backup", + ArgsUsage: " []", + Flags: []cli.Flag{ + internal.DebugFlag, + }, + Before: internal.SubCommandBefore, + BashComplete: autocomplete.AppNameComplete, + Description: ` +This command runs an app backup. + +A backup command and pre/post hook commands are defined in the recipe +configuration. Abra reads this config and run the comands in the context of the +service. Pass if you only want to back up a single service. All +backups are placed in the ~/.abra/backups directory. + +Example: + + abra app backup example.com db +`, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) + + recipe, err := recipe.Get(app.Recipe) + if err != nil { + logrus.Fatal(err) + } + + backupConfigs := make(map[string]backupConfig) + for _, service := range recipe.Config.Services { + if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok { + if backupsEnabled == "true" { + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) + bkConfig := backupConfig{} + + logrus.Debugf("backup config detected for %s", fullServiceName) + + if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok { + logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths) + bkConfig.backupPaths = strings.Split(paths, ",") + } + + if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok { + logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) + bkConfig.preHookCmd = preHookCmd + } + + if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok { + logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) + bkConfig.postHookCmd = postHookCmd + } + + backupConfigs[service.Name] = bkConfig + } + } + } + + serviceName := c.Args().Get(1) + if serviceName != "" { + backupConfig, ok := backupConfigs[serviceName] + if !ok { + logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName) + } + if err := runBackup(app, serviceName, backupConfig); err != nil { + logrus.Fatal(err) + } + } else { + for serviceName, backupConfig := range backupConfigs { + if err := runBackup(app, serviceName, backupConfig); err != nil { + logrus.Fatal(err) + } + } + } + + return nil + }, +} + +// runBackup does the actual backup logic. +func runBackup(app config.App, serviceName string, bkConfig backupConfig) error { + if len(bkConfig.backupPaths) == 0 { + return fmt.Errorf("backup paths are empty for %s?", serviceName) + } + + cl, err := client.New(app.Server) + if err != nil { + return err + } + + // FIXME: avoid instantiating a new CLI + dcli, err := command.NewDockerCli() + if err != nil { + return err + } + + filters := filters.NewArgs() + filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) + + targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) + if err != nil { + return err + } + + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) + if bkConfig.preHookCmd != "" { + splitCmd := internal.SafeSplit(bkConfig.preHookCmd) + + logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) + + preHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd) + } + + for _, remoteBackupPath := range bkConfig.backupPaths { + timestamp := strconv.Itoa(time.Now().Nanosecond()) + sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_") + localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp)) + logrus.Debugf("backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) + + content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath) + if err != nil { + return err + } + defer content.Close() + + body, err := ioutil.ReadAll(content) + if err != nil { + return err + } + + if err := ioutil.WriteFile(localBackupPath, body, 0644); err != nil { + return err + } + + logrus.Infof("backed up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) + } + + if bkConfig.postHookCmd != "" { + splitCmd := internal.SafeSplit(bkConfig.postHookCmd) + + logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) + + postHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd) + } + + return nil +} diff --git a/cli/app/restore.go b/cli/app/restore.go new file mode 100644 index 00000000..e2e4e5db --- /dev/null +++ b/cli/app/restore.go @@ -0,0 +1,197 @@ +package app + +import ( + "context" + "errors" + "fmt" + "os" + + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/config" + containerPkg "coopcloud.tech/abra/pkg/container" + "coopcloud.tech/abra/pkg/recipe" + "coopcloud.tech/abra/pkg/upstream/container" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/pkg/archive" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +type restoreConfig struct { + preHookCmd string + postHookCmd string +} + +var appRestoreCommand = cli.Command{ + Name: "restore", + Aliases: []string{"rs"}, + Usage: "Run app restore", + ArgsUsage: " ", + Flags: []cli.Flag{ + internal.DebugFlag, + }, + Before: internal.SubCommandBefore, + BashComplete: autocomplete.AppNameComplete, + Description: ` +This command runs an app restore. + +Pre/post hook commands are defined in the recipe configuration. Abra reads this +config and run the comands in the context of the service before restoring the +backup. + +Example: + + abra app restore example.com app mybackup.tar.gz /var/lib/content +`, + Action: func(c *cli.Context) error { + app := internal.ValidateApp(c) + + serviceName := c.Args().Get(1) + if serviceName == "" { + internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) + } + + backupPath := c.Args().Get(2) + if backupPath == "" { + internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) + } + + restorePath := c.Args().Get(3) + if restorePath == "" { + internal.ShowSubcommandHelpAndError(c, errors.New("missing ?")) + } + + if _, err := os.Stat(backupPath); err != nil { + if os.IsNotExist(err) { + logrus.Fatalf("%s doesn't exist?", backupPath) + } + } + + recipe, err := recipe.Get(app.Recipe) + if err != nil { + logrus.Fatal(err) + } + + restoreConfigs := make(map[string]restoreConfig) + for _, service := range recipe.Config.Services { + if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok { + if restoreEnabled == "true" { + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) + rsConfig := restoreConfig{} + + logrus.Debugf("restore config detected for %s", fullServiceName) + + if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok { + logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) + rsConfig.preHookCmd = preHookCmd + } + + if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok { + logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) + rsConfig.postHookCmd = postHookCmd + } + + restoreConfigs[service.Name] = rsConfig + } + } + } + + rsConfig, ok := restoreConfigs[serviceName] + if !ok { + rsConfig = restoreConfig{} + } + if err := runRestore(app, backupPath, restorePath, serviceName, rsConfig); err != nil { + logrus.Fatal(err) + } + + return nil + }, +} + +// runRestore does the actual restore logic. +func runRestore(app config.App, backupPath, restorePath, serviceName string, rsConfig restoreConfig) error { + cl, err := client.New(app.Server) + if err != nil { + return err + } + + // FIXME: avoid instantiating a new CLI + dcli, err := command.NewDockerCli() + if err != nil { + return err + } + + filters := filters.NewArgs() + filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) + + targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) + if err != nil { + return err + } + + fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) + if rsConfig.preHookCmd != "" { + splitCmd := internal.SafeSplit(rsConfig.preHookCmd) + + logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) + + preHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd) + } + + backupReader, err := os.Open(backupPath) + if err != nil { + return err + } + + content, err := archive.DecompressStream(backupReader) + if err != nil { + return err + } + + copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false} + if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil { + return err + } + + logrus.Infof("restored %s to %s:%s", backupPath, fullServiceName, restorePath) + + if rsConfig.postHookCmd != "" { + splitCmd := internal.SafeSplit(rsConfig.postHookCmd) + + logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) + + postHookExecOpts := types.ExecConfig{ + AttachStderr: true, + AttachStdin: true, + AttachStdout: true, + Cmd: splitCmd, + Detach: false, + Tty: true, + } + + if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { + return err + } + + logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd) + } + + return nil +} diff --git a/cli/cli.go b/cli/cli.go index 5296592e..cf5b262d 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -172,6 +172,7 @@ func newAbraApp(version, commit string) *cli.App { path.Join(config.SERVERS_DIR), path.Join(config.RECIPES_DIR), path.Join(config.VENDOR_DIR), + path.Join(config.BACKUP_DIR), } for _, path := range paths { diff --git a/cli/internal/backup.go b/cli/internal/backup.go new file mode 100644 index 00000000..79951810 --- /dev/null +++ b/cli/internal/backup.go @@ -0,0 +1,35 @@ +package internal + +import ( + "strings" +) + +// SafeSplit splits up a string into a list of commands safely. +func SafeSplit(s string) []string { + split := strings.Split(s, " ") + + var result []string + var inquote string + var block string + for _, i := range split { + if inquote == "" { + if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") { + inquote = string(i[0]) + block = strings.TrimPrefix(i, inquote) + " " + } else { + result = append(result, i) + } + } else { + if !strings.HasSuffix(i, inquote) { + block += i + " " + } else { + block += strings.TrimSuffix(i, inquote) + inquote = "" + result = append(result, block) + block = "" + } + } + } + + return result +} diff --git a/pkg/config/env.go b/pkg/config/env.go index 3165dfb2..f6d0e9a2 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -18,6 +18,7 @@ var ABRA_DIR = os.ExpandEnv("$HOME/.abra") var SERVERS_DIR = path.Join(ABRA_DIR, "servers") var RECIPES_DIR = path.Join(ABRA_DIR, "recipes") var VENDOR_DIR = path.Join(ABRA_DIR, "vendor") +var BACKUP_DIR = path.Join(ABRA_DIR, "backups") var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json") var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud" var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"