package app import ( "archive/tar" "context" "fmt" "io" "os" "path/filepath" "strconv" "strings" "time" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" containerPkg "coopcloud.tech/abra/pkg/container" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/container" "github.com/docker/cli/cli/command" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/system" "github.com/klauspost/pgzip" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) type backupConfig struct { preHookCmd string postHookCmd string backupPaths []string } var appBackupCommand = cli.Command{ Name: "backup", Aliases: []string{"bk"}, Usage: "Run app backup", ArgsUsage: " []", Flags: []cli.Flag{ internal.DebugFlag, }, Before: internal.SubCommandBefore, BashComplete: autocomplete.AppNameComplete, Description: ` This command runs an app backup. A backup command and pre/post hook commands are defined in the recipe configuration. Abra reads this config and run the comands in the context of the service. Pass if you only want to back up a single service. All backups are placed in the ~/.abra/backups directory. Example: abra app backup example.com db `, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) recipe, err := recipe.Get(app.Recipe) if err != nil { logrus.Fatal(err) } backupConfigs := make(map[string]backupConfig) for _, service := range recipe.Config.Services { if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok { if backupsEnabled == "true" { fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name) bkConfig := backupConfig{} logrus.Debugf("backup config detected for %s", fullServiceName) if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok { logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths) bkConfig.backupPaths = strings.Split(paths, ",") } if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok { logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd) bkConfig.preHookCmd = preHookCmd } if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok { logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd) bkConfig.postHookCmd = postHookCmd } backupConfigs[service.Name] = bkConfig } } } serviceName := c.Args().Get(1) if serviceName != "" { backupConfig, ok := backupConfigs[serviceName] if !ok { logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName) } if err := runBackup(app, serviceName, backupConfig); err != nil { logrus.Fatal(err) } } else { for serviceName, backupConfig := range backupConfigs { if err := runBackup(app, serviceName, backupConfig); err != nil { logrus.Fatal(err) } } } return nil }, } // runBackup does the actual backup logic. func runBackup(app config.App, serviceName string, bkConfig backupConfig) error { if len(bkConfig.backupPaths) == 0 { return fmt.Errorf("backup paths are empty for %s?", serviceName) } cl, err := client.New(app.Server) if err != nil { return err } // FIXME: avoid instantiating a new CLI dcli, err := command.NewDockerCli() if err != nil { return err } filters := filters.NewArgs() filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName)) targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true) if err != nil { return err } fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName) if bkConfig.preHookCmd != "" { splitCmd := internal.SafeSplit(bkConfig.preHookCmd) logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd) preHookExecOpts := types.ExecConfig{ AttachStderr: true, AttachStdin: true, AttachStdout: true, Cmd: splitCmd, Detach: false, Tty: true, } if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error()) } logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd) } var tempBackupPaths []string for _, remoteBackupPath := range bkConfig.backupPaths { timestamp := strconv.Itoa(time.Now().Nanosecond()) sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_") localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp)) logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath) content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath) if err != nil { logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) if err := cleanupTempArchives(tempBackupPaths); err != nil { return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) } return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) } defer content.Close() _, srcBase := archive.SplitPathDirEntry(remoteBackupPath) preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath) if err := copyToFile(localBackupPath, preArchive); err != nil { logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) if err := cleanupTempArchives(tempBackupPaths); err != nil { return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) } return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) } tempBackupPaths = append(tempBackupPaths, localBackupPath) } logrus.Infof("compressing and merging archives...") if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil { logrus.Debugf("failed to merge archive files: %s", err.Error()) if err := cleanupTempArchives(tempBackupPaths); err != nil { return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) } return fmt.Errorf("failed to merge archive files: %s", err.Error()) } if err := cleanupTempArchives(tempBackupPaths); err != nil { return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) } if bkConfig.postHookCmd != "" { splitCmd := internal.SafeSplit(bkConfig.postHookCmd) logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd) postHookExecOpts := types.ExecConfig{ AttachStderr: true, AttachStdin: true, AttachStdout: true, Cmd: splitCmd, Detach: false, Tty: true, } if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil { return err } logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd) } return nil } func copyToFile(outfile string, r io.Reader) error { tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp") if err != nil { return err } tmpPath := tmpFile.Name() _, err = io.Copy(tmpFile, r) tmpFile.Close() if err != nil { os.Remove(tmpPath) return err } if err = os.Rename(tmpPath, outfile); err != nil { os.Remove(tmpPath) return err } return nil } func cleanupTempArchives(tarPaths []string) error { for _, tarPath := range tarPaths { if err := os.RemoveAll(tarPath); err != nil { return err } logrus.Debugf("remove temporary archive file %s", tarPath) } return nil } func mergeArchives(tarPaths []string, serviceName string) error { var out io.Writer var cout *pgzip.Writer timestamp := strconv.Itoa(time.Now().Nanosecond()) localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, timestamp)) fout, err := os.Create(localBackupPath) if err != nil { return fmt.Errorf("Failed to open %s: %s", localBackupPath, err) } defer fout.Close() out = fout cout = pgzip.NewWriter(out) out = cout tw := tar.NewWriter(out) for _, tarPath := range tarPaths { if err := addTar(tw, tarPath); err != nil { return fmt.Errorf("failed to merge %s: %v", tarPath, err) } } if err := tw.Close(); err != nil { return fmt.Errorf("failed to close tar writer %v", err) } if cout != nil { if err := cout.Flush(); err != nil { return fmt.Errorf("failed to flush: %s", err) } else if err = cout.Close(); err != nil { return fmt.Errorf("failed to close compressed writer: %s", err) } } logrus.Infof("backed up %s to %s", serviceName, localBackupPath) return nil } func addTar(tw *tar.Writer, pth string) (err error) { var tr *tar.Reader var rc io.ReadCloser var hdr *tar.Header if tr, rc, err = openTarFile(pth); err != nil { return } for { if hdr, err = tr.Next(); err != nil { if err == io.EOF { err = nil } break } if err = tw.WriteHeader(hdr); err != nil { break } else if _, err = io.Copy(tw, tr); err != nil { break } } if err == nil { err = rc.Close() } else { rc.Close() } return } func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) { var fin *os.File var n int buff := make([]byte, 1024) if fin, err = os.Open(pth); err != nil { return } if n, err = fin.Read(buff); err != nil { fin.Close() return } else if n == 0 { fin.Close() err = fmt.Errorf("%s is empty", pth) return } if _, err = fin.Seek(0, 0); err != nil { fin.Close() return } rc = fin tr = tar.NewReader(rc) return tr, rc, nil }