Compare commits

..

1 Commits

Author SHA1 Message Date
5fea97c776 WIP: filter by topic when building catalogue
See coop-cloud/organising#377
2023-10-10 07:30:26 +02:00
12 changed files with 582 additions and 591 deletions

View File

@ -1,422 +1,414 @@
package app
import (
"archive/tar"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/autocomplete"
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
containerPkg "coopcloud.tech/abra/pkg/container"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/service"
recipePkg "coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/upstream/container"
"github.com/docker/cli/cli/command"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerClient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/klauspost/pgzip"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
var snapshot string
var snapshotFlag = &cli.StringFlag{
Name: "snapshot, s",
Usage: "Lists specific snapshot",
Destination: &snapshot,
}
var includePath string
var includePathFlag = &cli.StringFlag{
Name: "path, p",
Usage: "Include path",
Destination: &includePath,
}
var resticRepo string
var resticRepoFlag = &cli.StringFlag{
Name: "repo, r",
Usage: "Restic repository",
Destination: &resticRepo,
}
var appBackupListCommand = cli.Command{
Name: "list",
Aliases: []string{"ls"},
Flags: []cli.Flag{
internal.DebugFlag,
internal.OfflineFlag,
snapshotFlag,
includePathFlag,
},
Before: internal.SubCommandBefore,
Usage: "List all backups",
BashComplete: autocomplete.AppNameComplete,
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
if err := recipe.EnsureExists(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipe.EnsureIsClean(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Offline {
if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
if err := recipe.EnsureLatest(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
filters := filters.NewArgs()
filters.Add("name", chosenService.Spec.Name)
targetContainer, err := containerPkg.GetContainer(
context.Background(),
cl,
filters,
internal.NoInput,
)
if err != nil {
logrus.Fatal(err)
}
execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
if snapshot != "" {
logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
}
if includePath != "" {
logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
}
execBackupListOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: []string{"/usr/bin/backup", "--", "ls"},
Detach: false,
Env: execEnv,
Tty: true,
}
logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
logrus.Fatal(err)
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
logrus.Fatal(err)
}
return nil
},
}
var appBackupDownloadCommand = cli.Command{
Name: "download",
Aliases: []string{"d"},
Flags: []cli.Flag{
internal.DebugFlag,
internal.OfflineFlag,
snapshotFlag,
includePathFlag,
},
Before: internal.SubCommandBefore,
Usage: "Download a backup",
BashComplete: autocomplete.AppNameComplete,
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
if err := recipe.EnsureExists(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipe.EnsureIsClean(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Offline {
if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
if err := recipe.EnsureLatest(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
filters := filters.NewArgs()
filters.Add("name", chosenService.Spec.Name)
targetContainer, err := containerPkg.GetContainer(
context.Background(),
cl,
filters,
internal.NoInput,
)
if err != nil {
logrus.Fatal(err)
}
execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
if snapshot != "" {
logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
}
if includePath != "" {
logrus.Debugf("including INCLUDE_PATH=%s in backupbot exec invocation", includePath)
execEnv = append(execEnv, fmt.Sprintf("INCLUDE_PATH=%s", includePath))
}
execBackupListOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: []string{"/usr/bin/backup", "--", "download"},
Detach: false,
Env: execEnv,
Tty: true,
}
logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
logrus.Fatal(err)
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
logrus.Fatal(err)
}
return nil
},
}
var appBackupCreateCommand = cli.Command{
Name: "create",
Aliases: []string{"c"},
Flags: []cli.Flag{
internal.DebugFlag,
internal.OfflineFlag,
resticRepoFlag,
},
Before: internal.SubCommandBefore,
Usage: "Create a new backup",
BashComplete: autocomplete.AppNameComplete,
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
if err := recipe.EnsureExists(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipe.EnsureIsClean(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Offline {
if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
if err := recipe.EnsureLatest(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
filters := filters.NewArgs()
filters.Add("name", chosenService.Spec.Name)
targetContainer, err := containerPkg.GetContainer(
context.Background(),
cl,
filters,
internal.NoInput,
)
if err != nil {
logrus.Fatal(err)
}
execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
if resticRepo != "" {
logrus.Debugf("including RESTIC_REPO=%s in backupbot exec invocation", resticRepo)
execEnv = append(execEnv, fmt.Sprintf("RESTIC_REPO=%s", resticRepo))
}
execBackupListOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: []string{"/usr/bin/backup", "--", "create"},
Detach: false,
Env: execEnv,
Tty: true,
}
logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
logrus.Fatal(err)
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
logrus.Fatal(err)
}
return nil
},
}
var appBackupSnapshotsCommand = cli.Command{
Name: "snapshots",
Aliases: []string{"s"},
Flags: []cli.Flag{
internal.DebugFlag,
internal.OfflineFlag,
snapshotFlag,
},
Before: internal.SubCommandBefore,
Usage: "List backup snapshots",
BashComplete: autocomplete.AppNameComplete,
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
if err := recipe.EnsureExists(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipe.EnsureIsClean(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Offline {
if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
if err := recipe.EnsureLatest(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
filters := filters.NewArgs()
filters.Add("name", chosenService.Spec.Name)
targetContainer, err := containerPkg.GetContainer(
context.Background(),
cl,
filters,
internal.NoInput,
)
if err != nil {
logrus.Fatal(err)
}
execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
if snapshot != "" {
logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
}
execBackupListOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: []string{"/usr/bin/backup", "--", "snapshots"},
Detach: false,
Env: execEnv,
Tty: true,
}
logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
logrus.Fatal(err)
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
logrus.Fatal(err)
}
return nil
},
type backupConfig struct {
preHookCmd string
postHookCmd string
backupPaths []string
}
var appBackupCommand = cli.Command{
Name: "backup",
Aliases: []string{"b"},
Usage: "Manage app backups",
ArgsUsage: "<domain>",
Subcommands: []cli.Command{
appBackupListCommand,
appBackupSnapshotsCommand,
appBackupDownloadCommand,
appBackupCreateCommand,
Aliases: []string{"bk"},
Usage: "Run app backup",
ArgsUsage: "<domain> [<service>]",
Flags: []cli.Flag{
internal.DebugFlag,
internal.OfflineFlag,
internal.ChaosFlag,
},
Before: internal.SubCommandBefore,
BashComplete: autocomplete.AppNameComplete,
Description: `
Run an app backup.
A backup command and pre/post hook commands are defined in the recipe
configuration. Abra reads this configuration and run the comands in the context
of the deployed services. Pass <service> if you only want to back up a single
service. All backups are placed in the ~/.abra/backups directory.
A single backup file is produced for all backup paths specified for a service.
If we have the following backup configuration:
- "backupbot.backup.path=/var/lib/foo,/var/lib/bar"
And we run "abra app backup example.com app", Abra will produce a file that
looks like:
~/.abra/backups/example_com_app_609341138.tar.gz
This file is a compressed archive which contains all backup paths. To see paths, run:
tar -tf ~/.abra/backups/example_com_app_609341138.tar.gz
(Make sure to change the name of the backup file)
This single file can be used to restore your app. See "abra app restore" for more.
`,
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
recipe, err := recipePkg.Get(app.Recipe, internal.Offline)
if err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Offline {
if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
backupConfigs := make(map[string]backupConfig)
for _, service := range recipe.Config.Services {
if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
if backupsEnabled == "true" {
fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
bkConfig := backupConfig{}
logrus.Debugf("backup config detected for %s", fullServiceName)
if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
bkConfig.backupPaths = strings.Split(paths, ",")
}
if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
bkConfig.preHookCmd = preHookCmd
}
if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
bkConfig.postHookCmd = postHookCmd
}
backupConfigs[service.Name] = bkConfig
}
}
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
serviceName := c.Args().Get(1)
if serviceName != "" {
backupConfig, ok := backupConfigs[serviceName]
if !ok {
logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
}
logrus.Infof("running backup for the %s service", serviceName)
if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
logrus.Fatal(err)
}
} else {
if len(backupConfigs) == 0 {
logrus.Fatalf("no backup configs discovered for %s?", app.Name)
}
for serviceName, backupConfig := range backupConfigs {
logrus.Infof("running backup for the %s service", serviceName)
if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
logrus.Fatal(err)
}
}
}
return nil
},
}
// TimeStamp generates a file name friendly timestamp.
func TimeStamp() string {
ts := time.Now().UTC().Format(time.RFC3339)
return strings.Replace(ts, ":", "-", -1)
}
// runBackup does the actual backup logic.
func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
if len(bkConfig.backupPaths) == 0 {
return fmt.Errorf("backup paths are empty for %s?", serviceName)
}
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
return err
}
filters := filters.NewArgs()
filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
if err != nil {
return err
}
fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
if bkConfig.preHookCmd != "" {
splitCmd := internal.SafeSplit(bkConfig.preHookCmd)
logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
preHookExecOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: splitCmd,
Detach: false,
Tty: true,
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
}
logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
}
var tempBackupPaths []string
for _, remoteBackupPath := range bkConfig.backupPaths {
sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, TimeStamp()))
logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)
logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)
content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
if err != nil {
logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
if err := cleanupTempArchives(tempBackupPaths); err != nil {
return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
}
return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
}
defer content.Close()
_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
if err := copyToFile(localBackupPath, preArchive); err != nil {
logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
if err := cleanupTempArchives(tempBackupPaths); err != nil {
return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
}
return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
}
tempBackupPaths = append(tempBackupPaths, localBackupPath)
}
logrus.Infof("compressing and merging archives...")
if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
logrus.Debugf("failed to merge archive files: %s", err.Error())
if err := cleanupTempArchives(tempBackupPaths); err != nil {
return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
}
return fmt.Errorf("failed to merge archive files: %s", err.Error())
}
if err := cleanupTempArchives(tempBackupPaths); err != nil {
return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
}
if bkConfig.postHookCmd != "" {
splitCmd := internal.SafeSplit(bkConfig.postHookCmd)
logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
postHookExecOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: splitCmd,
Detach: false,
Tty: true,
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
return err
}
logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
}
return nil
}
func copyToFile(outfile string, r io.Reader) error {
tmpFile, err := os.CreateTemp(filepath.Dir(outfile), ".tar_temp")
if err != nil {
return err
}
tmpPath := tmpFile.Name()
_, err = io.Copy(tmpFile, r)
tmpFile.Close()
if err != nil {
os.Remove(tmpPath)
return err
}
if err = os.Rename(tmpPath, outfile); err != nil {
os.Remove(tmpPath)
return err
}
return nil
}
func cleanupTempArchives(tarPaths []string) error {
for _, tarPath := range tarPaths {
if err := os.RemoveAll(tarPath); err != nil {
return err
}
logrus.Debugf("remove temporary archive file %s", tarPath)
}
return nil
}
func mergeArchives(tarPaths []string, serviceName string) error {
var out io.Writer
var cout *pgzip.Writer
localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, TimeStamp()))
fout, err := os.Create(localBackupPath)
if err != nil {
return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
}
defer fout.Close()
out = fout
cout = pgzip.NewWriter(out)
out = cout
tw := tar.NewWriter(out)
for _, tarPath := range tarPaths {
if err := addTar(tw, tarPath); err != nil {
return fmt.Errorf("failed to merge %s: %v", tarPath, err)
}
}
if err := tw.Close(); err != nil {
return fmt.Errorf("failed to close tar writer %v", err)
}
if cout != nil {
if err := cout.Flush(); err != nil {
return fmt.Errorf("failed to flush: %s", err)
} else if err = cout.Close(); err != nil {
return fmt.Errorf("failed to close compressed writer: %s", err)
}
}
logrus.Infof("backed up %s to %s", serviceName, localBackupPath)
return nil
}
func addTar(tw *tar.Writer, pth string) (err error) {
var tr *tar.Reader
var rc io.ReadCloser
var hdr *tar.Header
if tr, rc, err = openTarFile(pth); err != nil {
return
}
for {
if hdr, err = tr.Next(); err != nil {
if err == io.EOF {
err = nil
}
break
}
if err = tw.WriteHeader(hdr); err != nil {
break
} else if _, err = io.Copy(tw, tr); err != nil {
break
}
}
if err == nil {
err = rc.Close()
} else {
rc.Close()
}
return
}
func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
var fin *os.File
var n int
buff := make([]byte, 1024)
if fin, err = os.Open(pth); err != nil {
return
}
if n, err = fin.Read(buff); err != nil {
fin.Close()
return
} else if n == 0 {
fin.Close()
err = fmt.Errorf("%s is empty", pth)
return
}
if _, err = fin.Seek(0, 0); err != nil {
fin.Close()
return
}
rc = fin
tr = tar.NewReader(rc)
return tr, rc, nil
}

View File

@ -2,7 +2,9 @@ package app
import (
"context"
"errors"
"fmt"
"os"
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/autocomplete"
@ -10,112 +12,212 @@ import (
"coopcloud.tech/abra/pkg/config"
containerPkg "coopcloud.tech/abra/pkg/container"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/service"
recipePkg "coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/upstream/container"
"github.com/docker/cli/cli/command"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerClient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
var targetPath string
var targetPathFlag = &cli.StringFlag{
Name: "target, t",
Usage: "Target path",
Destination: &targetPath,
type restoreConfig struct {
preHookCmd string
postHookCmd string
}
var appRestoreCommand = cli.Command{
Name: "restore",
Aliases: []string{"rs"},
Usage: "Restore an app backup",
ArgsUsage: "<domain> <service>",
Usage: "Run app restore",
ArgsUsage: "<domain> <service> <file>",
Flags: []cli.Flag{
internal.DebugFlag,
internal.OfflineFlag,
targetPathFlag,
internal.ChaosFlag,
},
Before: internal.SubCommandBefore,
BashComplete: autocomplete.AppNameComplete,
Description: `
Run an app restore.
Pre/post hook commands are defined in the recipe configuration. Abra reads this
configuration and run the comands in the context of the service before
restoring the backup.
Unlike "abra app backup", restore must be run on a per-service basis. You can
not restore all services in one go. Backup files produced by Abra are
compressed archives which use absolute paths. This allows Abra to restore
according to standard tar command logic, i.e. the backup will be restored to
the path it was originally backed up from.
Example:
abra app restore example.com app ~/.abra/backups/example_com_app_609341138.tar.gz
`,
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
if err := recipe.EnsureExists(app.Recipe); err != nil {
recipe, err := recipe.Get(app.Recipe, internal.Offline)
if err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipe.EnsureIsClean(app.Recipe); err != nil {
if err := recipePkg.EnsureIsClean(app.Recipe); err != nil {
logrus.Fatal(err)
}
if !internal.Offline {
if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
if err := recipePkg.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
if err := recipe.EnsureLatest(app.Recipe); err != nil {
if err := recipePkg.EnsureLatest(app.Recipe); err != nil {
logrus.Fatal(err)
}
}
serviceName := c.Args().Get(1)
if serviceName == "" {
internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
}
backupPath := c.Args().Get(2)
if backupPath == "" {
internal.ShowSubcommandHelpAndError(c, errors.New("missing <file>?"))
}
if _, err := os.Stat(backupPath); err != nil {
if os.IsNotExist(err) {
logrus.Fatalf("%s doesn't exist?", backupPath)
}
}
restoreConfigs := make(map[string]restoreConfig)
for _, service := range recipe.Config.Services {
if restoreEnabled, ok := service.Deploy.Labels["backupbot.restore"]; ok {
if restoreEnabled == "true" {
fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
rsConfig := restoreConfig{}
logrus.Debugf("restore config detected for %s", fullServiceName)
if preHookCmd, ok := service.Deploy.Labels["backupbot.restore.pre-hook"]; ok {
logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
rsConfig.preHookCmd = preHookCmd
}
if postHookCmd, ok := service.Deploy.Labels["backupbot.restore.post-hook"]; ok {
logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
rsConfig.postHookCmd = postHookCmd
}
restoreConfigs[service.Name] = rsConfig
}
}
}
rsConfig, ok := restoreConfigs[serviceName]
if !ok {
rsConfig = restoreConfig{}
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
chosenService, err := service.GetServiceByLabel(context.Background(), cl, config.BackupbotLabel, internal.NoInput)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved %s as backup enabled service", chosenService.Spec.Name)
filters := filters.NewArgs()
filters.Add("name", chosenService.Spec.Name)
targetContainer, err := containerPkg.GetContainer(
context.Background(),
cl,
filters,
internal.NoInput,
)
if err != nil {
logrus.Fatal(err)
}
execEnv := []string{fmt.Sprintf("SERVICE=%s", app.Domain)}
if snapshot != "" {
logrus.Debugf("including SNAPSHOT=%s in backupbot exec invocation", snapshot)
execEnv = append(execEnv, fmt.Sprintf("SNAPSHOT=%s", snapshot))
}
if targetPath != "" {
logrus.Debugf("including TARGET=%s in backupbot exec invocation", targetPath)
execEnv = append(execEnv, fmt.Sprintf("TARGET=%s", targetPath))
}
execBackupListOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: []string{"/usr/bin/backup", "--", "restore"},
Detach: false,
Env: execEnv,
Tty: true,
}
logrus.Debugf("running backup list on %s with exec config %v", targetContainer.ID, execBackupListOpts)
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
logrus.Fatal(err)
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &execBackupListOpts); err != nil {
if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
logrus.Fatal(err)
}
return nil
},
}
// runRestore does the actual restore logic.
func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {
return err
}
filters := filters.NewArgs()
filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))
targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
if err != nil {
return err
}
fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
if rsConfig.preHookCmd != "" {
splitCmd := internal.SafeSplit(rsConfig.preHookCmd)
logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)
preHookExecOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: splitCmd,
Detach: false,
Tty: true,
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
return err
}
logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, rsConfig.preHookCmd)
}
backupReader, err := os.Open(backupPath)
if err != nil {
return err
}
content, err := archive.DecompressStream(backupReader)
if err != nil {
return err
}
// NOTE(d1): we use absolute paths so tar knows what to do. it will restore
// files according to the paths set in the compressed archive
restorePath := "/"
copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
if err := cl.CopyToContainer(context.Background(), targetContainer.ID, restorePath, content, copyOpts); err != nil {
return err
}
logrus.Infof("restored %s to %s", backupPath, fullServiceName)
if rsConfig.postHookCmd != "" {
splitCmd := internal.SafeSplit(rsConfig.postHookCmd)
logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)
postHookExecOpts := types.ExecConfig{
AttachStderr: true,
AttachStdin: true,
AttachStdout: true,
Cmd: splitCmd,
Detach: false,
Tty: true,
}
if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
return err
}
logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, rsConfig.postHookCmd)
}
return nil
}

View File

@ -98,11 +98,6 @@ keys configured on your account.
continue
}
if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
catlBar.Add(1)
continue
}
versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
if err != nil {
logrus.Warn(err)

View File

@ -12,46 +12,6 @@ import (
"github.com/sirupsen/logrus"
)
// CatalogueSkipList is all the repos that are not recipes.
var CatalogueSkipList = map[string]bool{
"abra": true,
"abra-apps": true,
"abra-aur": true,
"abra-bash": true,
"abra-capsul": true,
"abra-gandi": true,
"abra-hetzner": true,
"abra-test-recipe": true,
"apps": true,
"aur-abra-git": true,
"auto-mirror": true,
"auto-recipes-catalogue-json": true,
"backup-bot": true,
"backup-bot-two": true,
"beta.coopcloud.tech": true,
"comrade-renovate-bot": true,
"coopcloud.tech": true,
"coturn": true,
"docker-cp-deploy": true,
"docker-dind-bats-kcov": true,
"docs.coopcloud.tech": true,
"drone-abra": true,
"example": true,
"gardening": true,
"go-abra": true,
"organising": true,
"pyabra": true,
"radicle-seed-node": true,
"recipes-catalogue-json": true,
"recipes-wishlist": true,
"recipes.coopcloud.tech": true,
"stack-ssh-deploy": true,
"swarm-cronjob": true,
"tagcmp": true,
"traefik-cert-dumper": true,
"tyop": true,
}
// EnsureCatalogue ensures that the catalogue is cloned locally & present.
func EnsureCatalogue() error {
catalogueDir := path.Join(config.ABRA_DIR, "catalogue")

View File

@ -36,8 +36,6 @@ var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
var CATALOGUE_JSON_REPO_NAME = "recipes-catalogue-json"
var SSH_URL_TEMPLATE = "ssh://git@git.coopcloud.tech:2222/coop-cloud/%s.git"
var BackupbotLabel = "coop-cloud.backupbot.enabled"
// envVarModifiers is a list of env var modifier strings. These are added to
// env vars as comments and modify their processing by Abra, e.g. determining
// how long secrets should be.

View File

@ -28,7 +28,7 @@ func GetContainer(c context.Context, cl *client.Client, filters filters.Args, no
return types.Container{}, fmt.Errorf("no containers matching the %v filter found?", filter)
}
if len(containers) > 1 {
if len(containers) != 1 {
var containersRaw []string
for _, container := range containers {
containerName := strings.Join(container.Names, " ")

View File

@ -7,6 +7,7 @@ import (
"os"
"path"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
@ -31,7 +32,7 @@ import (
// RecipeCatalogueURL is the only current recipe catalogue available.
const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
// ReposMetadataURL is the recipe repository metadata
// ReposMetadataURL is the recipe repository metadata.
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
// tag represents a git tag.
@ -63,6 +64,11 @@ type RecipeMeta struct {
Website string `json:"website"`
}
// TopicMeta represents a list of topics for a repository.
type TopicMeta struct {
Topics []string `json:"topics"`
}
// LatestVersion returns the latest version of a recipe.
func (r RecipeMeta) LatestVersion() string {
var version string
@ -822,7 +828,16 @@ func ReadReposMetadata() (RepoCatalogue, error) {
}
for idx, repo := range reposList {
reposMeta[repo.Name] = reposList[idx]
var topicMeta TopicMeta
topicsURL := getReposTopicUrl(repo.Name)
if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
return reposMeta, err
}
if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
reposMeta[repo.Name] = reposList[idx]
}
}
pageIdx++
@ -1002,14 +1017,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
retrieveBar.Add(1)
return
}
if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
ch <- rm.Name
retrieveBar.Add(1)
return
}
recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
logrus.Fatal(err)
}
@ -1025,3 +1034,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
return nil
}
// getReposTopicUrl retrieves the repository specific topic listing.
func getReposTopicUrl(repoName string) string {
return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
}

View File

@ -14,69 +14,6 @@ import (
"github.com/sirupsen/logrus"
)
// GetService retrieves a service container based on a label. If prompt is true
// and the retrievd count of service containers does not match 1, then a prompt
// is presented to let the user choose. A count of 0 is handled gracefully.
func GetServiceByLabel(c context.Context, cl *client.Client, label string, prompt bool) (swarm.Service, error) {
services, err := cl.ServiceList(c, types.ServiceListOptions{})
if err != nil {
return swarm.Service{}, err
}
if len(services) == 0 {
return swarm.Service{}, fmt.Errorf("no services deployed?")
}
var backupServices []swarm.Service
for _, service := range services {
if enabled, exists := service.Spec.Labels[label]; exists && enabled == "true" {
backupServices = append(backupServices, service)
}
}
if len(backupServices) == 0 {
return swarm.Service{}, fmt.Errorf("no backup services deployed?")
}
if len(backupServices) > 1 {
var servicesRaw []string
for _, service := range backupServices {
serviceName := service.Spec.Name
created := formatter.HumanDuration(service.CreatedAt.Unix())
servicesRaw = append(servicesRaw, fmt.Sprintf("%s (created %v)", serviceName, created))
}
if !prompt {
err := fmt.Errorf("expected 1 service but found %v: %s", len(backupServices), strings.Join(servicesRaw, " "))
return swarm.Service{}, err
}
logrus.Warnf("ambiguous service list received, prompting for input")
var response string
prompt := &survey.Select{
Message: "which service are you looking for?",
Options: servicesRaw,
}
if err := survey.AskOne(prompt, &response); err != nil {
return swarm.Service{}, err
}
chosenService := strings.TrimSpace(strings.Split(response, " ")[0])
for _, service := range backupServices {
serviceName := strings.ToLower(service.Spec.Name)
if serviceName == chosenService {
return service, nil
}
}
logrus.Panic("failed to match chosen service")
}
return backupServices[0], nil
}
// GetService retrieves a service container. If prompt is true and the retrievd
// count of service containers does not match 1, then a prompt is presented to
// let the user choose. A count of 0 is handled gracefully.

View File

@ -1,8 +1,8 @@
#!/usr/bin/env bash
ABRA_VERSION="0.8.1-beta"
ABRA_VERSION="0.8.0-beta"
ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
RC_VERSION="0.8.1-beta"
RC_VERSION="0.8.0-beta"
RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
for arg in "$@"; do

View File

@ -35,7 +35,7 @@ setup(){
run $ABRA app check "$TEST_APP_DOMAIN"
assert_success
refute_output --partial ''
assert_output --partial 'all necessary environment variables defined'
assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
}
@ -111,16 +111,12 @@ setup(){
}
@test "error if missing env var" {
run $ABRA app check "$TEST_APP_DOMAIN"
assert_success
refute_output --partial '❌'
run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
assert_success
run $ABRA app check "$TEST_APP_DOMAIN" --chaos
assert_success
assert_output --partial '❌'
assert_failure
assert_output --partial "NEW_VAR"
_checkout_recipe
}

View File

@ -321,8 +321,6 @@ teardown(){
# bats test_tags=slow
@test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
_reset_recipe
run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
"$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
assert_success

View File

@ -110,11 +110,10 @@ setup(){
assert_success
assert_output --partial 'test_pass_one'
run bash -c '$ABRA app secret ls $TEST_APP_DOMAIN --machine | \
jq -r ".[] | select(.name==\"test_pass_one\") | .version"'
run docker -c "$TEST_SERVER" secret ls
assert_success
assert_output --partial 'v2'
refute_output --partial 'v1'
assert_output --regexp ".*_test_pass_one_v2"
refute_output --regexp ".*_test_pass_one_v1"
run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
assert_success