wip: backup/restore v2

2024-10-29 12:03:42 +01:00 · 2024-10-29 12:03:42 +01:00 · a11ccd30f2
commit a11ccd30f2
parent 1f9b863be0
5 changed files with 224 additions and 140 deletions
--- a/cli/app/backup.go
+++ b/cli/app/backup.go
@ -15,41 +15,129 @@ var snapshot string
 var snapshotFlag = &cli.StringFlag{
 	Name:        "snapshot",
 	Aliases:     []string{"s"},
-	Usage:       "Lists specific snapshot",
+	Usage:       "List specific snapshot",
+	Value:       "latest",
 	Destination: &snapshot,
 }

+var retries string
+var retriesFlag = &cli.StringFlag{
+	Name:        "retries",
+	Aliases:     []string{"r"},
+	Usage:       "No. retry attempts",
+	Destination: &retries,
+}
+
+var showAll bool
+var showAllFlag = &cli.BoolFlag{
+	Name:        "all",
+	Aliases:     []string{"a"},
+	Usage:       "Show all paths",
+	Destination: &showAll,
+}
+
+var timestamps bool
+var timestampsFlag = &cli.BoolFlag{
+	Name:        "timestamps",
+	Aliases:     []string{"t"},
+	Usage:       "Include timestamps",
+	Destination: &timestamps,
+}
+
 var includePath string
 var includePathFlag = &cli.StringFlag{
 	Name:        "path",
 	Aliases:     []string{"p"},
-	Usage:       "Include path",
+	Usage:       "Volumes path",
 	Destination: &includePath,
 }

-var resticRepo string
-var resticRepoFlag = &cli.StringFlag{
-	Name:        "repo",
-	Aliases:     []string{"r"},
-	Usage:       "Restic repository",
-	Destination: &resticRepo,
+var includeSecrets bool
+var includeSecretsFlag = &cli.BoolFlag{
+	Name:        "secrets",
+	Aliases:     []string{"S"},
+	Usage:       "Include secrets",
+	Value:       true,
+	Destination: &includeSecrets,
 }

+// TODO: use --host <host> to make all stuff run for the app itself
+
 var appBackupListCommand = cli.Command{
 	Name:    "list",
 	Aliases: []string{"ls"},
 	Flags: []cli.Flag{
 		snapshotFlag,
-		includePathFlag,
+		showAllFlag,
+		timestampsFlag,
 	},
 	Before:        internal.SubCommandBefore,
-	Usage:         "List all backups",
+	Usage:         "List the contents of a snapshot",
 	UsageText:     "abra app backup list <domain> [options]",
 	ShellComplete: autocomplete.AppNameComplete,
 	HideHelp:      true,
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		app := internal.ValidateApp(cmd)

+		cl, err := client.New(app.Server)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+			fmt.Sprintf("INCLUDE_PATH=%s", "/var/lib/docker/volumes/gitea_test_biobulkbende_org_db"),
+		}
+
+		if snapshot != "" {
+			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
+			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		}
+		if showAll {
+			log.Debugf("including SHOW_ALL=%v in backupbot exec invocation", showAll)
+			execEnv = append(execEnv, fmt.Sprintf("SHOW_ALL=%v", showAll))
+		}
+		if timestamps {
+			log.Debugf("including TIMESTAMPS=%v in backupbot exec invocation", timestamps)
+			execEnv = append(execEnv, fmt.Sprintf("TIMESTAMPS=%v", timestamps))
+		}
+
+		if _, err = internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
+			log.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+var appBackupDownloadCommand = cli.Command{
+	Name:    "download",
+	Aliases: []string{"d"},
+	Flags: []cli.Flag{
+		snapshotFlag,
+		includePathFlag,
+		includeSecretsFlag,
+		internal.IncludeVolumesFlag,
+		internal.ChaosFlag,
+	},
+	Before: internal.SubCommandBefore,
+	Usage:  "Download a snapshot",
+	Description: `Downloads a backup.tar.gz to the current working directory.
+
+"--volumes/-v" includes data contained in volumes alongide paths specified in
+"backupbot.backup.path" labels.`,
+	UsageText:     "abra app backup download <domain> [options]",
+	ShellComplete: autocomplete.AppNameComplete,
+	HideHelp:      true,
+	Action: func(ctx context.Context, cmd *cli.Command) error {
+		app := internal.ValidateApp(cmd)
+
 		if err := app.Recipe.Ensure(internal.Chaos, internal.Offline); err != nil {
 			log.Fatal(err)
 		}
@ -64,80 +152,32 @@ var appBackupListCommand = cli.Command{
 			log.Fatal(err)
 		}

-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
 		if snapshot != "" {
 			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
 			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
 		}
+
 		if includePath != "" {
 			log.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
 			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
 		}

-		if err := internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil {
-			log.Fatal(err)
+		if includeSecrets {
+			log.Debugf("including SECRETS=%v in backupbot exec invocation", includeSecrets)
+			execEnv = append(execEnv, fmt.Sprintf("SECRETS=%v", includeSecrets))
 		}

-		return nil
-	},
-}
-
-var appBackupDownloadCommand = cli.Command{
-	Name:    "download",
-	Aliases: []string{"d"},
-	Flags: []cli.Flag{
-		snapshotFlag,
-		includePathFlag,
-	},
-	Before:        internal.SubCommandBefore,
-	Usage:         "Download a backup",
-	UsageText:     "abra app backup download <domain> [options]",
-	ShellComplete: autocomplete.AppNameComplete,
-	HideHelp:      true,
-	Action: func(ctx context.Context, cmd *cli.Command) error {
-		app := internal.ValidateApp(cmd)
-
-		if err := app.Recipe.EnsureExists(); err != nil {
-			log.Fatal(err)
+		if internal.IncludeVolumes {
+			log.Debugf("including VOLUMES=%v in backupbot exec invocation", internal.IncludeVolumes)
+			execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%v", internal.IncludeVolumes))
 		}

-		if !internal.Chaos {
-			if err := app.Recipe.EnsureIsClean(); err != nil {
-				log.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := app.Recipe.EnsureUpToDate(); err != nil {
-					log.Fatal(err)
-				}
-			}
-
-			if err := app.Recipe.EnsureLatest(); err != nil {
-				log.Fatal(err)
-			}
-		}
-
-		cl, err := client.New(app.Server)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		targetContainer, err := internal.RetrieveBackupBotContainer(cl)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
-		}
-		if includePath != "" {
-			log.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
-			execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
-		}
-
-		if err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
+		if _, err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil {
 			log.Fatal(err)
 		}

@ -147,7 +187,7 @@ var appBackupDownloadCommand = cli.Command{
 			log.Fatal(err)
 		}

-		fmt.Println("backup successfully downloaded to current working directory")
+		log.Info("snapshot successfully downloaded")

 		return nil
 	},
@ -157,36 +197,21 @@ var appBackupCreateCommand = cli.Command{
 	Name:    "create",
 	Aliases: []string{"c"},
 	Flags: []cli.Flag{
-		resticRepoFlag,
+		retriesFlag,
+		internal.ChaosFlag,
 	},
 	Before:        internal.SubCommandBefore,
-	Usage:         "Create a new backup",
+	Usage:         "Create a new snapshot",
 	UsageText:     "abra app backup create <domain> [options]",
 	ShellComplete: autocomplete.AppNameComplete,
 	HideHelp:      true,
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		app := internal.ValidateApp(cmd)

-		if err := app.Recipe.EnsureExists(); err != nil {
+		if err := app.Recipe.Ensure(internal.Chaos, internal.Offline); err != nil {
 			log.Fatal(err)
 		}

-		if !internal.Chaos {
-			if err := app.Recipe.EnsureIsClean(); err != nil {
-				log.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := app.Recipe.EnsureUpToDate(); err != nil {
-					log.Fatal(err)
-				}
-			}
-
-			if err := app.Recipe.EnsureLatest(); err != nil {
-				log.Fatal(err)
-			}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			log.Fatal(err)
@ -197,13 +222,17 @@ var appBackupCreateCommand = cli.Command{
 			log.Fatal(err)
 		}

-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if resticRepo != "" {
-			log.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
-			execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
 		}

-		if err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
+		if retries != "" {
+			log.Debugf("including RETRIES=%s in backupbot exec invocation", retries)
+			execEnv = append(execEnv, fmt.Sprintf("RETRIES=%s", retries))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil {
 			log.Fatal(err)
 		}

@ -212,39 +241,16 @@ var appBackupCreateCommand = cli.Command{
 }

 var appBackupSnapshotsCommand = cli.Command{
-	Name:    "snapshots",
-	Aliases: []string{"s"},
-	Flags: []cli.Flag{
-		snapshotFlag,
-	},
+	Name:          "snapshots",
+	Aliases:       []string{"s"},
 	Before:        internal.SubCommandBefore,
-	Usage:         "List backup snapshots",
+	Usage:         "List all snapshots",
 	UsageText:     "abra app backup snapshots <domain> [options]",
 	ShellComplete: autocomplete.AppNameComplete,
 	HideHelp:      true,
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		app := internal.ValidateApp(cmd)

-		if err := app.Recipe.EnsureExists(); err != nil {
-			log.Fatal(err)
-		}
-
-		if !internal.Chaos {
-			if err := app.Recipe.EnsureIsClean(); err != nil {
-				log.Fatal(err)
-			}
-
-			if !internal.Offline {
-				if err := app.Recipe.EnsureUpToDate(); err != nil {
-					log.Fatal(err)
-				}
-			}
-
-			if err := app.Recipe.EnsureLatest(); err != nil {
-				log.Fatal(err)
-			}
-		}
-
 		cl, err := client.New(app.Server)
 		if err != nil {
 			log.Fatal(err)
@ -255,13 +261,12 @@ var appBackupSnapshotsCommand = cli.Command{
 			log.Fatal(err)
 		}

-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
-		if snapshot != "" {
-			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
-			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
 		}

-		if err := internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
+		if _, err = internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil {
 			log.Fatal(err)
 		}

--- a/cli/app/deploy.go
+++ b/cli/app/deploy.go
@ -47,7 +47,7 @@ operations.`,
 		stackName := app.StackName()

 		specificVersion := cmd.Args().Get(1)
-		if specificVersion == "" {
+		if specificVersion == "" && !internal.Chaos {
 			specificVersion = app.Recipe.Version
 		}

--- a/cli/app/restore.go
+++ b/cli/app/restore.go
@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"fmt"
+	"strings"

 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@ -15,17 +16,53 @@ var targetPath string
 var targetPathFlag = &cli.StringFlag{
 	Name:        "target",
 	Aliases:     []string{"t"},
+	Value:       "/",
 	Usage:       "Target path",
 	Destination: &targetPath,
 }

+var hooks bool
+var hooksFlag = &cli.BoolFlag{
+	Name:        "hooks",
+	Aliases:     []string{"H"},
+	Usage:       "Enable pre/post-hook command execution",
+	Value:       false,
+	Destination: &hooks,
+}
+
+var services []string
+var servicesFlag = &cli.StringSliceFlag{
+	Name:        "service",
+	Aliases:     []string{"s"},
+	Usage:       "Restore specific services",
+	Destination: &services,
+}
+
+var volumes []string
+var volumesFlag = &cli.StringSliceFlag{
+	Name:        "volume",
+	Aliases:     []string{"v"},
+	Usage:       "Restore specific volumes",
+	Destination: &volumes,
+}
+
+// TODO: use --host <host> to make all stuff run for the app itself
+
 var appRestoreCommand = cli.Command{
-	Name:      "restore",
-	Aliases:   []string{"rs"},
-	Usage:     "Restore an app backup",
-	UsageText: "abra app restore <domain> <service> [options]",
+	Name:    "restore",
+	Aliases: []string{"rs"},
+	Usage:   "Restore a snapshot",
+	Description: `Snapshots are restored while apps are deployed.
+
+Some restore scenarios may require service / app restarts.`,
+	UsageText: "abra app restore <domain> [options]",
 	Flags: []cli.Flag{
 		targetPathFlag,
+		internal.NoInputFlag,
+		servicesFlag,
+		volumesFlag,
+		hooksFlag,
+		internal.ChaosFlag,
 	},
 	Before:        internal.SubCommandBefore,
 	ShellComplete: autocomplete.AppNameComplete,
@ -46,17 +83,44 @@ var appRestoreCommand = cli.Command{
 			log.Fatal(err)
 		}

-		execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
+		execEnv := []string{
+			fmt.Sprintf("SERVICE=%s", app.Domain),
+			"MACHINE_LOGS=true",
+		}
+
 		if snapshot != "" {
 			log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
 			execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
 		}
+
 		if targetPath != "" {
 			log.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
 			execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
 		}

-		if err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
+		if internal.NoInput {
+			log.Debugf("including NONINTERACTIVE=%v in backupbot exec invocation", internal.NoInput)
+			execEnv = append(execEnv, fmt.Sprintf("NONINTERACTIVE=%v", internal.NoInput))
+		}
+
+		if len(volumes) > 0 {
+			allVolumes := strings.Join(volumes, ",")
+			log.Debugf("including VOLUMES=%s in backupbot exec invocation", allVolumes)
+			execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%s", allVolumes))
+		}
+
+		if len(services) > 0 {
+			allServices := strings.Join(services, ",")
+			log.Debugf("including CONTAINER=%s in backupbot exec invocation", allServices)
+			execEnv = append(execEnv, fmt.Sprintf("CONTAINER=%s", allServices))
+		}
+
+		if hooks {
+			log.Debugf("including NO_COMMANDS=%v in backupbot exec invocation", false)
+			execEnv = append(execEnv, fmt.Sprintf("NO_COMMANDS=%v", false))
+		}
+
+		if _, err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil {
 			log.Fatal(err)
 		}

--- a/cli/internal/backup.go
+++ b/cli/internal/backup.go
@ -2,6 +2,8 @@ package internal

 import (
 	"context"
+	"fmt"
+	"io"

 	"coopcloud.tech/abra/pkg/config"
 	containerPkg "coopcloud.tech/abra/pkg/container"
@ -19,7 +21,7 @@ func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error
 	ctx := context.Background()
 	chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput)
 	if err != nil {
-		return types.Container{}, err
+		return types.Container{}, fmt.Errorf("no backupbot discovered, is it deployed?")
 	}

 	log.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
@ -40,7 +42,11 @@ func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error
 }

 // RunBackupCmdRemote runs a backup related command on a remote backupbot container.
-func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error {
+func RunBackupCmdRemote(
+	cl *dockerClient.Client,
+	backupCmd string,
+	containerID string,
+	execEnv []string) (io.Writer, error) {
 	execBackupListOpts := types.ExecConfig{
 		AttachStderr: true,
 		AttachStdin:  true,
@ -56,12 +62,13 @@ func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID s
 	// FIXME: avoid instantiating a new CLI
 	dcli, err := command.NewDockerCli()
 	if err != nil {
-		return err
+		return nil, err
 	}

-	if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil {
-		return err
+	out, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts)
+	if err != nil {
+		return nil, err
 	}

-	return nil
+	return out, nil
 }
--- a/cli/internal/cli.go
+++ b/cli/internal/cli.go
@ -319,6 +319,14 @@ var AllServicesFlag = &cli.BoolFlag{
 	Destination: &AllServices,
 }

+var IncludeVolumes bool
+var IncludeVolumesFlag = &cli.BoolFlag{
+	Name:        "volumes",
+	Aliases:     []string{"v"},
+	Usage:       "Include volumes",
+	Destination: &IncludeVolumes,
+}
+
 // SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
 func SubCommandBefore(ctx context.Context, cmd *cli.Command) error {
 	if Debug {