From 8fa20e2c7f67e2f0fb5788452aee9e27aec47c4c Mon Sep 17 00:00:00 2001 From: decentral1se Date: Fri, 27 Dec 2024 19:09:46 +0100 Subject: [PATCH] feat: new backup/restore --- cli/app/backup.go | 307 +++++++++++++++++++++++++++++++++++++++++ cli/app/deploy.go | 3 +- cli/app/restore.go | 135 ++++++++++++++++++ cli/app/secret.go | 8 ++ cli/internal/backup.go | 19 ++- cli/run.go | 13 +- 6 files changed, 476 insertions(+), 9 deletions(-) create mode 100644 cli/app/backup.go create mode 100644 cli/app/restore.go diff --git a/cli/app/backup.go b/cli/app/backup.go new file mode 100644 index 00000000..d784c372 --- /dev/null +++ b/cli/app/backup.go @@ -0,0 +1,307 @@ +package app + +import ( + "fmt" + + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/log" + "github.com/spf13/cobra" +) + +var AppBackupListCommand = &cobra.Command{ + Use: "list [flags]", + Aliases: []string{"ls"}, + Short: "List the contents of a snapshot", + Args: cobra.ExactArgs(1), + ValidArgsFunction: func( + cmd *cobra.Command, + args []string, + toComplete string) ([]string, cobra.ShellCompDirective) { + return autocomplete.AppNameComplete() + }, + Run: func(cmd *cobra.Command, args []string) { + app := internal.ValidateApp(args) + + cl, err := client.New(app.Server) + if err != nil { + log.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + log.Fatal(err) + } + + execEnv := []string{ + fmt.Sprintf("SERVICE=%s", app.Domain), + "MACHINE_LOGS=true", + } + + if snapshot != "" { + log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + + if showAllPaths { + log.Debugf("including SHOW_ALL=%v in backupbot exec invocation", showAllPaths) + execEnv = append(execEnv, fmt.Sprintf("SHOW_ALL=%v", showAllPaths)) + } + + if timestamps { + log.Debugf("including TIMESTAMPS=%v in backupbot exec invocation", timestamps) + execEnv = append(execEnv, fmt.Sprintf("TIMESTAMPS=%v", timestamps)) + } + + if _, err = internal.RunBackupCmdRemote(cl, "ls", targetContainer.ID, execEnv); err != nil { + log.Fatal(err) + } + }, +} + +var AppBackupDownloadCommand = &cobra.Command{ + Use: "download [flags]", + Aliases: []string{"d"}, + Short: "Download a snapshot", + Long: `Downloads a backup.tar.gz to the current working directory. + +"--volumes/-v" includes data contained in volumes alongide paths specified in +"backupbot.backup.path" labels.`, + Args: cobra.ExactArgs(1), + ValidArgsFunction: func( + cmd *cobra.Command, + args []string, + toComplete string) ([]string, cobra.ShellCompDirective) { + return autocomplete.AppNameComplete() + }, + Run: func(cmd *cobra.Command, args []string) { + app := internal.ValidateApp(args) + + if err := app.Recipe.Ensure(internal.Chaos, internal.Offline); err != nil { + log.Fatal(err) + } + + cl, err := client.New(app.Server) + if err != nil { + log.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + log.Fatal(err) + } + + execEnv := []string{ + fmt.Sprintf("SERVICE=%s", app.Domain), + "MACHINE_LOGS=true", + } + + if snapshot != "" { + log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + + if includePath != "" { + log.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath) + execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath)) + } + + if includeSecrets { + log.Debugf("including SECRETS=%v in backupbot exec invocation", includeSecrets) + execEnv = append(execEnv, fmt.Sprintf("SECRETS=%v", includeSecrets)) + } + + if includeVolumes { + log.Debugf("including VOLUMES=%v in backupbot exec invocation", includeVolumes) + execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%v", includeVolumes)) + } + + if _, err := internal.RunBackupCmdRemote(cl, "download", targetContainer.ID, execEnv); err != nil { + log.Fatal(err) + } + + remoteBackupDir := "/tmp/backup.tar.gz" + currentWorkingDir := "." + if err = CopyFromContainer(cl, targetContainer.ID, remoteBackupDir, currentWorkingDir); err != nil { + log.Fatal(err) + } + }, +} + +var AppBackupCreateCommand = &cobra.Command{ + Use: "create [flags]", + Aliases: []string{"c"}, + Short: "Create a new snapshot", + Args: cobra.ExactArgs(1), + ValidArgsFunction: func( + cmd *cobra.Command, + args []string, + toComplete string) ([]string, cobra.ShellCompDirective) { + return autocomplete.AppNameComplete() + }, + Run: func(cmd *cobra.Command, args []string) { + app := internal.ValidateApp(args) + + if err := app.Recipe.Ensure(internal.Chaos, internal.Offline); err != nil { + log.Fatal(err) + } + + cl, err := client.New(app.Server) + if err != nil { + log.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + log.Fatal(err) + } + + execEnv := []string{ + fmt.Sprintf("SERVICE=%s", app.Domain), + "MACHINE_LOGS=true", + } + + if retries != "" { + log.Debugf("including RETRIES=%s in backupbot exec invocation", retries) + execEnv = append(execEnv, fmt.Sprintf("RETRIES=%s", retries)) + } + + if _, err := internal.RunBackupCmdRemote(cl, "create", targetContainer.ID, execEnv); err != nil { + log.Fatal(err) + } + }, +} + +var AppBackupSnapshotsCommand = &cobra.Command{ + Use: "snapshots [flags]", + Aliases: []string{"s"}, + Short: "List all snapshots", + Args: cobra.ExactArgs(1), + ValidArgsFunction: func( + cmd *cobra.Command, + args []string, + toComplete string) ([]string, cobra.ShellCompDirective) { + return autocomplete.AppNameComplete() + }, + Run: func(cmd *cobra.Command, args []string) { + app := internal.ValidateApp(args) + + cl, err := client.New(app.Server) + if err != nil { + log.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + log.Fatal(err) + } + + execEnv := []string{ + fmt.Sprintf("SERVICE=%s", app.Domain), + "MACHINE_LOGS=true", + } + + if _, err = internal.RunBackupCmdRemote(cl, "snapshots", targetContainer.ID, execEnv); err != nil { + log.Fatal(err) + } + }, +} + +var AppBackupCommand = &cobra.Command{ + Use: "backup [cmd] [args] [flags]", + Aliases: []string{"b"}, + Short: "Manage app backups", +} + +var ( + snapshot string + retries string + includePath string + showAllPaths bool + timestamps bool + includeSecrets bool + includeVolumes bool +) + +func init() { + AppBackupListCommand.Flags().StringVarP( + &snapshot, + "snapshot", + "s", + "", + "list specific snapshot", + ) + + AppBackupListCommand.Flags().BoolVarP( + &showAllPaths, + "all", + "a", + false, + "show all paths", + ) + + AppBackupListCommand.Flags().BoolVarP( + ×tamps, + "timestamps", + "t", + false, + "include timestamps", + ) + + AppBackupDownloadCommand.Flags().StringVarP( + &snapshot, + "snapshot", + "s", + "", + "list specific snapshot", + ) + + AppBackupDownloadCommand.Flags().StringVarP( + &includePath, + "path", + "p", + "", + "volumes path", + ) + + AppBackupDownloadCommand.Flags().BoolVarP( + &includeSecrets, + "secrets", + "S", + false, + "include secrets", + ) + + AppBackupDownloadCommand.Flags().BoolVarP( + &includeVolumes, + "volumes", + "v", + false, + "include volumes", + ) + + AppBackupDownloadCommand.Flags().BoolVarP( + &internal.Chaos, + "chaos", + "C", + false, + "ignore uncommitted recipes changes", + ) + + AppBackupCreateCommand.Flags().StringVarP( + &retries, + "retries", + "r", + "1", + "number of retry attempts", + ) + + AppBackupCreateCommand.Flags().BoolVarP( + &internal.Chaos, + "chaos", + "C", + false, + "ignore uncommitted recipes changes", + ) +} diff --git a/cli/app/deploy.go b/cli/app/deploy.go index e5a223fb..e9473c51 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -320,7 +320,8 @@ func init() { ) AppDeployCommand.Flags().BoolVarP( - &internal.DontWaitConverge, "no-converge-checks", + &internal.DontWaitConverge, + "no-converge-checks", "c", false, "do not wait for converge logic checks", diff --git a/cli/app/restore.go b/cli/app/restore.go new file mode 100644 index 00000000..da5bfebf --- /dev/null +++ b/cli/app/restore.go @@ -0,0 +1,135 @@ +package app + +import ( + "fmt" + "strings" + + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/log" + "github.com/spf13/cobra" +) + +var AppRestoreCommand = &cobra.Command{ + Use: "restore [flags]", + Aliases: []string{"rs"}, + Short: "Restore a snapshot", + Long: `Snapshots are restored while apps are deployed. + +Some restore scenarios may require service / app restarts.`, + Args: cobra.ExactArgs(1), + ValidArgsFunction: func( + cmd *cobra.Command, + args []string, + toComplete string) ([]string, cobra.ShellCompDirective) { + return autocomplete.AppNameComplete() + }, + Run: func(cmd *cobra.Command, args []string) { + app := internal.ValidateApp(args) + + if err := app.Recipe.Ensure(internal.Chaos, internal.Offline); err != nil { + log.Fatal(err) + } + + cl, err := client.New(app.Server) + if err != nil { + log.Fatal(err) + } + + targetContainer, err := internal.RetrieveBackupBotContainer(cl) + if err != nil { + log.Fatal(err) + } + + execEnv := []string{ + fmt.Sprintf("SERVICE=%s", app.Domain), + "MACHINE_LOGS=true", + } + + if snapshot != "" { + log.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot) + execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot)) + } + + if targetPath != "" { + log.Debugf("including TARGET=%s in backupbot exec invocation", targetPath) + execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath)) + } + + if internal.NoInput { + log.Debugf("including NONINTERACTIVE=%v in backupbot exec invocation", internal.NoInput) + execEnv = append(execEnv, fmt.Sprintf("NONINTERACTIVE=%v", internal.NoInput)) + } + + if len(volumes) > 0 { + allVolumes := strings.Join(volumes, ",") + log.Debugf("including VOLUMES=%s in backupbot exec invocation", allVolumes) + execEnv = append(execEnv, fmt.Sprintf("VOLUMES=%s", allVolumes)) + } + + if len(services) > 0 { + allServices := strings.Join(services, ",") + log.Debugf("including CONTAINER=%s in backupbot exec invocation", allServices) + execEnv = append(execEnv, fmt.Sprintf("CONTAINER=%s", allServices)) + } + + if hooks { + log.Debugf("including NO_COMMANDS=%v in backupbot exec invocation", false) + execEnv = append(execEnv, fmt.Sprintf("NO_COMMANDS=%v", false)) + } + + if _, err := internal.RunBackupCmdRemote(cl, "restore", targetContainer.ID, execEnv); err != nil { + log.Fatal(err) + } + }, +} + +var ( + targetPath string + hooks bool + services []string + volumes []string +) + +func init() { + AppRestoreCommand.Flags().StringVarP( + &targetPath, + "target", + "t", + "/", + "target path", + ) + + AppRestoreCommand.Flags().StringArrayVarP( + &services, + "services", + "s", + []string{}, + "restore specific services", + ) + + AppRestoreCommand.Flags().StringArrayVarP( + &volumes, + "volumes", + "v", + []string{}, + "restore specific volumes", + ) + + AppRestoreCommand.Flags().BoolVarP( + &hooks, + "hooks", + "H", + false, + "enable pre/post-hook command execution", + ) + + AppRestoreCommand.Flags().BoolVarP( + &internal.Chaos, + "chaos", + "C", + false, + "ignore uncommitted recipes changes", + ) +} diff --git a/cli/app/secret.go b/cli/app/secret.go index 95cdffae..511135e7 100644 --- a/cli/app/secret.go +++ b/cli/app/secret.go @@ -442,6 +442,14 @@ func init() { "ignore uncommitted recipes changes", ) + AppSecretGenerateCommand.Flags().BoolVarP( + &generateAllSecrets, + "all", + "a", + false, + "generate all secrets", + ) + AppSecretInsertCommand.Flags().BoolVarP( &storeInPass, "pass", diff --git a/cli/internal/backup.go b/cli/internal/backup.go index c3a14666..1080d7ce 100644 --- a/cli/internal/backup.go +++ b/cli/internal/backup.go @@ -2,6 +2,8 @@ package internal import ( "context" + "fmt" + "io" "coopcloud.tech/abra/pkg/config" containerPkg "coopcloud.tech/abra/pkg/container" @@ -19,7 +21,7 @@ func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error ctx := context.Background() chosenService, err := service.GetServiceByLabel(ctx, cl, config.BackupbotLabel, NoInput) if err != nil { - return types.Container{}, err + return types.Container{}, fmt.Errorf("no backupbot discovered, is it deployed?") } log.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name) @@ -40,7 +42,11 @@ func RetrieveBackupBotContainer(cl *dockerClient.Client) (types.Container, error } // RunBackupCmdRemote runs a backup related command on a remote backupbot container. -func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID string, execEnv []string) error { +func RunBackupCmdRemote( + cl *dockerClient.Client, + backupCmd string, + containerID string, + execEnv []string) (io.Writer, error) { execBackupListOpts := types.ExecConfig{ AttachStderr: true, AttachStdin: true, @@ -56,12 +62,13 @@ func RunBackupCmdRemote(cl *dockerClient.Client, backupCmd string, containerID s // FIXME: avoid instantiating a new CLI dcli, err := command.NewDockerCli() if err != nil { - return err + return nil, err } - if _, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts); err != nil { - return err + out, err := container.RunExec(dcli, cl, containerID, &execBackupListOpts) + if err != nil { + return nil, err } - return nil + return out, nil } diff --git a/cli/run.go b/cli/run.go index 1020e9a9..70d89f05 100644 --- a/cli/run.go +++ b/cli/run.go @@ -159,10 +159,17 @@ func Run(version, commit string) { app.AppVolumeRemoveCommand, ) + app.AppBackupCommand.AddCommand( + app.AppBackupListCommand, + app.AppBackupDownloadCommand, + app.AppBackupCreateCommand, + app.AppBackupSnapshotsCommand, + ) + app.AppCommand.AddCommand( - app.AppRunCommand, - app.AppCmdCommand, + app.AppBackupCommand, app.AppCheckCommand, + app.AppCmdCommand, app.AppConfigCommand, app.AppCpCommand, app.AppDeployCommand, @@ -172,7 +179,9 @@ func Run(version, commit string) { app.AppPsCommand, app.AppRemoveCommand, app.AppRestartCommand, + app.AppRestoreCommand, app.AppRollbackCommand, + app.AppRunCommand, app.AppSecretCommand, app.AppServicesCommand, app.AppUndeployCommand, -- 2.47.2