feat: backup/restore [ci skip]

See coop-cloud/organising#30.

cli/app/app.go

@@ -30,5 +30,7 @@ var AppCommand = cli.Command{
 		appVersionCommand,
 		appErrorsCommand,
 		appCmdCommand,
+		appBackupCommand,
+		appRestoreCommand,
 	},
 }

cli/app/backup.go

@@ -0,0 +1,205 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type backupConfig struct {
+	preHookCmd  string
+	postHookCmd string
+	backupPaths []string
+}
+
+var appBackupCommand = cli.Command{
+	Name:      "backup",
+	Aliases:   []string{"bk"},
+	Usage:     "Run app backup",
+	ArgsUsage: "<domain> [<service>]",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+This command runs an app backup.
+
+A backup command and pre/post hook commands are defined in the recipe
+configuration. Abra reads this config and run the comands in the context of the
+service. Pass <service> if you only want to back up a single service. All
+backups are placed in the ~/.abra/backups directory.
+
+Example:
+
+    abra app backup example.com db
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		recipe, err := recipe.Get(app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		backupConfigs := make(map[string]backupConfig)
+		for _, service := range recipe.Config.Services {
+			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
+				if backupsEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					bkConfig := backupConfig{}
+
+					logrus.Debugf("backup config detected for %s", fullServiceName)
+
+					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
+						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
+						bkConfig.backupPaths = strings.Split(paths, ",")
+					}
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						bkConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						bkConfig.postHookCmd = postHookCmd
+					}
+
+					backupConfigs[service.Name] = bkConfig
+				}
+			}
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName != "" {
+			backupConfig, ok := backupConfigs[serviceName]
+			if !ok {
+				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
+			}
+			if err := runBackup(app, serviceName, backupConfig); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			for serviceName, backupConfig := range backupConfigs {
+				if err := runBackup(app, serviceName, backupConfig); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+// runBackup does the actual backup logic.
+func runBackup(app config.App, serviceName string, bkConfig backupConfig) error {
+	if len(bkConfig.backupPaths) == 0 {
+		return fmt.Errorf("backup paths are empty for %s?", serviceName)
+	}
+
+	cl, err := client.New(app.Server)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if bkConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
+	}
+
+	for _, remoteBackupPath := range bkConfig.backupPaths {
+		timestamp := strconv.Itoa(time.Now().Nanosecond())
+		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
+		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp))
+		logrus.Debugf("backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
+
+		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
+		if err != nil {
+			return err
+		}
+		defer content.Close()
+
+		body, err := ioutil.ReadAll(content)
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(localBackupPath, body, 0644); err != nil {
+			return err
+		}
+
+		logrus.Infof("backed up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
+	}
+
+	if bkConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
+	}
+
+	return nil
+}

cli/app/restore.go

@@ -0,0 +1,197 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	containerPkg "coopcloud.tech/abra/pkg/container"
+	"coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/upstream/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+type restoreConfig struct {
+	preHookCmd  string
+	postHookCmd string
+}
+
+var appRestoreCommand = cli.Command{
+	Name:      "restore",
+	Aliases:   []string{"rs"},
+	Usage:     "Run app restore",
+	ArgsUsage: "<domain> <service> <file> <path>",
+	Flags: []cli.Flag{
+		internal.DebugFlag,
+	},
+	Before:       internal.SubCommandBefore,
+	BashComplete: autocomplete.AppNameComplete,
+	Description: `
+This command runs an app restore.
+
+Pre/post hook commands are defined in the recipe configuration. Abra reads this
+config and run the comands in the context of the service before restoring the
+backup.
+
+Example:
+
+    abra app restore example.com app mybackup.tar.gz /var/lib/content
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
+		}
+
+		backupPath := c.Args().Get(2)
+		if backupPath == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
+		}
+
+		restorePath := c.Args().Get(3)
+		if restorePath == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing <path>?"))
+		}
+
+		if _, err := os.Stat(backupPath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("%s doesn't exist?", backupPath)
+			}
+		}
+
+		recipe, err := recipe.Get(app.Recipe)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		restoreConfigs := make(map[string]restoreConfig)
+		for _, service := range recipe.Config.Services {
+			if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
+				if restoreEnabled == "true" {
+					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
+					rsConfig := restoreConfig{}
+
+					logrus.Debugf("restore config detected for %s", fullServiceName)
+
+					if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
+						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
+						rsConfig.preHookCmd = preHookCmd
+					}
+
+					if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
+						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
+						rsConfig.postHookCmd = postHookCmd
+					}
+
+					restoreConfigs[service.Name] = rsConfig
+				}
+			}
+		}
+
+		rsConfig, ok := restoreConfigs[serviceName]
+		if !ok {
+			rsConfig = restoreConfig{}
+		}
+		if err := runRestore(app, backupPath, restorePath, serviceName, rsConfig); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+// runRestore does the actual restore logic.
+func runRestore(app config.App, backupPath, restorePath, serviceName string, rsConfig restoreConfig) error {
+	cl, err := client.New(app.Server)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: avoid instantiating a new CLI
+	dcli, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+
+	filters := filters.NewArgs()
+	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
+
+	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
+	if err != nil {
+		return err
+	}
+
+	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+	if rsConfig.preHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
+
+		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
+
+		preHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
+	}
+
+	backupReader, err := os.Open(backupPath)
+	if err != nil {
+		return err
+	}
+
+	content, err := archive.DecompressStream(backupReader)
+	if err != nil {
+		return err
+	}
+
+	copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+	if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
+		return err
+	}
+
+	logrus.Infof("restored %s to %s:%s", backupPath, fullServiceName, restorePath)
+
+	if rsConfig.postHookCmd != "" {
+		splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
+
+		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
+
+		postHookExecOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          splitCmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
+			return err
+		}
+
+		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
+	}
+
+	return nil
+}

cli/cli.go

@@ -172,6 +172,7 @@ func newAbraApp(version, commit string) *cli.App {
 			path.Join(config.SERVERS_DIR),
 			path.Join(config.RECIPES_DIR),
 			path.Join(config.VENDOR_DIR),
+			path.Join(config.BACKUP_DIR),
 		}
 
 		for _, path := range paths {

cli/internal/backup.go

@@ -0,0 +1,35 @@
+package internal
+
+import (
+	"strings"
+)
+
+// SafeSplit splits up a string into a list of commands safely.
+func SafeSplit(s string) []string {
+	split := strings.Split(s, " ")
+
+	var result []string
+	var inquote string
+	var block string
+	for _, i := range split {
+		if inquote == "" {
+			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
+				inquote = string(i[0])
+				block = strings.TrimPrefix(i, inquote) + " "
+			} else {
+				result = append(result, i)
+			}
+		} else {
+			if !strings.HasSuffix(i, inquote) {
+				block += i + " "
+			} else {
+				block += strings.TrimSuffix(i, inquote)
+				inquote = ""
+				result = append(result, block)
+				block = ""
+			}
+		}
+	}
+
+	return result
+}

pkg/config/env.go

@@ -18,6 +18,7 @@ var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
 var SERVERS_DIR = path.Join(ABRA_DIR, "servers")
 var RECIPES_DIR = path.Join(ABRA_DIR, "recipes")
 var VENDOR_DIR = path.Join(ABRA_DIR, "vendor")
+var BACKUP_DIR = path.Join(ABRA_DIR, "backups")
 var RECIPES_JSON = path.Join(ABRA_DIR, "catalogue", "recipes.json")
 var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
 var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"