abra/cli/app/backup.go

package app

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"coopcloud.tech/abra/cli/internal"
	"coopcloud.tech/abra/pkg/autocomplete"
	"coopcloud.tech/abra/pkg/client"
	"coopcloud.tech/abra/pkg/config"
	containerPkg "coopcloud.tech/abra/pkg/container"
	"coopcloud.tech/abra/pkg/recipe"
	"coopcloud.tech/abra/pkg/upstream/container"
	"github.com/docker/cli/cli/command"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/system"
	"github.com/klauspost/pgzip"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

type backupConfig struct {
	preHookCmd  string
	postHookCmd string
	backupPaths []string
}

var appBackupCommand = cli.Command{
	Name:      "backup",
	Aliases:   []string{"bk"},
	Usage:     "Run app backup",
	ArgsUsage: "<domain> [<service>]",
	Flags: []cli.Flag{
		internal.DebugFlag,
	},
	Before:       internal.SubCommandBefore,
	BashComplete: autocomplete.AppNameComplete,
	Description: `
This command runs an app backup.

A backup command and pre/post hook commands are defined in the recipe
configuration. Abra reads this config and run the comands in the context of the
service. Pass <service> if you only want to back up a single service. All
backups are placed in the ~/.abra/backups directory.

Example:

    abra app backup example.com db
`,
	Action: func(c *cli.Context) error {
		app := internal.ValidateApp(c)

		recipe, err := recipe.Get(app.Recipe)
		if err != nil {
			logrus.Fatal(err)
		}

		backupConfigs := make(map[string]backupConfig)
		for _, service := range recipe.Config.Services {
			if backupsEnabled, ok := service.Deploy.Labels["backupbot.backup"]; ok {
				if backupsEnabled == "true" {
					fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), service.Name)
					bkConfig := backupConfig{}

					logrus.Debugf("backup config detected for %s", fullServiceName)

					if paths, ok := service.Deploy.Labels["backupbot.backup.path"]; ok {
						logrus.Debugf("detected backup paths for %s: %s", fullServiceName, paths)
						bkConfig.backupPaths = strings.Split(paths, ",")
					}

					if preHookCmd, ok := service.Deploy.Labels["backupbot.backup.pre-hook"]; ok {
						logrus.Debugf("detected pre-hook command for %s: %s", fullServiceName, preHookCmd)
						bkConfig.preHookCmd = preHookCmd
					}

					if postHookCmd, ok := service.Deploy.Labels["backupbot.backup.post-hook"]; ok {
						logrus.Debugf("detected post-hook command for %s: %s", fullServiceName, postHookCmd)
						bkConfig.postHookCmd = postHookCmd
					}

					backupConfigs[service.Name] = bkConfig
				}
			}
		}

		serviceName := c.Args().Get(1)
		if serviceName != "" {
			backupConfig, ok := backupConfigs[serviceName]
			if !ok {
				logrus.Fatalf("no backup config for %s? does %s exist?", serviceName, serviceName)
			}
			if err := runBackup(app, serviceName, backupConfig); err != nil {
				logrus.Fatal(err)
			}
		} else {
			for serviceName, backupConfig := range backupConfigs {
				if err := runBackup(app, serviceName, backupConfig); err != nil {
					logrus.Fatal(err)
				}
			}
		}

		return nil
	},
}

// runBackup does the actual backup logic.
func runBackup(app config.App, serviceName string, bkConfig backupConfig) error {
	if len(bkConfig.backupPaths) == 0 {
		return fmt.Errorf("backup paths are empty for %s?", serviceName)
	}

	cl, err := client.New(app.Server)
	if err != nil {
		return err
	}

	// FIXME: avoid instantiating a new CLI
	dcli, err := command.NewDockerCli()
	if err != nil {
		return err
	}

	filters := filters.NewArgs()
	filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))

	targetContainer, err := containerPkg.GetContainer(context.Background(), cl, filters, true)
	if err != nil {
		return err
	}

	fullServiceName := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
	if bkConfig.preHookCmd != "" {
		splitCmd := internal.SafeSplit(bkConfig.preHookCmd)

		logrus.Debugf("split pre-hook command for %s into %s", fullServiceName, splitCmd)

		preHookExecOpts := types.ExecConfig{
			AttachStderr: true,
			AttachStdin:  true,
			AttachStdout: true,
			Cmd:          splitCmd,
			Detach:       false,
			Tty:          true,
		}

		if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil {
			return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error())
		}

		logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd)
	}

	var tempBackupPaths []string
	for _, remoteBackupPath := range bkConfig.backupPaths {
		timestamp := strconv.Itoa(time.Now().Nanosecond())
		sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_")
		localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp))
		logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath)

		logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath)

		content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath)
		if err != nil {
			logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
			if err := cleanupTempArchives(tempBackupPaths); err != nil {
				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
			}
			return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error())
		}
		defer content.Close()

		_, srcBase := archive.SplitPathDirEntry(remoteBackupPath)
		preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath)
		if err := copyToFile(localBackupPath, preArchive); err != nil {
			logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
			if err := cleanupTempArchives(tempBackupPaths); err != nil {
				return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
			}
			return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error())
		}

		tempBackupPaths = append(tempBackupPaths, localBackupPath)
	}

	logrus.Infof("compressing and merging archives...")

	if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil {
		logrus.Debugf("failed to merge archive files: %s", err.Error())
		if err := cleanupTempArchives(tempBackupPaths); err != nil {
			return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
		}
		return fmt.Errorf("failed to merge archive files: %s", err.Error())
	}

	if err := cleanupTempArchives(tempBackupPaths); err != nil {
		return fmt.Errorf("failed to clean up temporary archives: %s", err.Error())
	}

	if bkConfig.postHookCmd != "" {
		splitCmd := internal.SafeSplit(bkConfig.postHookCmd)

		logrus.Debugf("split post-hook command for %s into %s", fullServiceName, splitCmd)

		postHookExecOpts := types.ExecConfig{
			AttachStderr: true,
			AttachStdin:  true,
			AttachStdout: true,
			Cmd:          splitCmd,
			Detach:       false,
			Tty:          true,
		}

		if err := container.RunExec(dcli, cl, targetContainer.ID, &postHookExecOpts); err != nil {
			return err
		}

		logrus.Infof("succesfully ran %s post-hook command: %s", fullServiceName, bkConfig.postHookCmd)
	}

	return nil
}

func copyToFile(outfile string, r io.Reader) error {
	tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp")
	if err != nil {
		return err
	}

	tmpPath := tmpFile.Name()

	_, err = io.Copy(tmpFile, r)
	tmpFile.Close()

	if err != nil {
		os.Remove(tmpPath)
		return err
	}

	if err = os.Rename(tmpPath, outfile); err != nil {
		os.Remove(tmpPath)
		return err
	}

	return nil
}

func cleanupTempArchives(tarPaths []string) error {
	for _, tarPath := range tarPaths {
		if err := os.RemoveAll(tarPath); err != nil {
			return err
		}

		logrus.Debugf("remove temporary archive file %s", tarPath)
	}

	return nil
}

func mergeArchives(tarPaths []string, serviceName string) error {
	var out io.Writer
	var cout *pgzip.Writer

	timestamp := strconv.Itoa(time.Now().Nanosecond())
	localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, timestamp))

	fout, err := os.Create(localBackupPath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", localBackupPath, err)
	}

	defer fout.Close()
	out = fout

	cout = pgzip.NewWriter(out)
	out = cout

	tw := tar.NewWriter(out)

	for _, tarPath := range tarPaths {
		if err := addTar(tw, tarPath); err != nil {
			return fmt.Errorf("failed to merge %s: %v", tarPath, err)
		}
	}

	if err := tw.Close(); err != nil {
		return fmt.Errorf("failed to close tar writer %v", err)
	}

	if cout != nil {
		if err := cout.Flush(); err != nil {
			return fmt.Errorf("failed to flush: %s", err)
		} else if err = cout.Close(); err != nil {
			return fmt.Errorf("failed to close compressed writer: %s", err)
		}
	}

	logrus.Infof("backed up %s to %s", serviceName, localBackupPath)

	return nil
}

func addTar(tw *tar.Writer, pth string) (err error) {
	var tr *tar.Reader
	var rc io.ReadCloser
	var hdr *tar.Header

	if tr, rc, err = openTarFile(pth); err != nil {
		return
	}

	for {
		if hdr, err = tr.Next(); err != nil {
			if err == io.EOF {
				err = nil
			}
			break
		}
		if err = tw.WriteHeader(hdr); err != nil {
			break
		} else if _, err = io.Copy(tw, tr); err != nil {
			break
		}
	}
	if err == nil {
		err = rc.Close()
	} else {
		rc.Close()
	}
	return
}

func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) {
	var fin *os.File
	var n int
	buff := make([]byte, 1024)

	if fin, err = os.Open(pth); err != nil {
		return
	}

	if n, err = fin.Read(buff); err != nil {
		fin.Close()
		return
	} else if n == 0 {
		fin.Close()
		err = fmt.Errorf("%s is empty", pth)
		return
	}

	if _, err = fin.Seek(0, 0); err != nil {
		fin.Close()
		return
	}

	rc = fin
	tr = tar.NewReader(rc)

	return tr, rc, nil
}