From a4f1634b24310835b642a07eb4c938720571026e Mon Sep 17 00:00:00 2001 From: decentral1se Date: Tue, 19 Apr 2022 12:52:30 +0200 Subject: [PATCH] fix: backups get gzip, absolute paths, single archive file --- cli/app/backup.go | 187 +++++++++++++++++++++++++++++++++++++++++++--- go.mod | 1 + go.sum | 2 + 3 files changed, 179 insertions(+), 11 deletions(-) diff --git a/cli/app/backup.go b/cli/app/backup.go index d8321525..0f62bd1a 100644 --- a/cli/app/backup.go +++ b/cli/app/backup.go @@ -1,9 +1,11 @@ package app import ( + "archive/tar" "context" "fmt" - "io/ioutil" + "io" + "os" "path/filepath" "strconv" "strings" @@ -19,6 +21,9 @@ import ( "github.com/docker/cli/cli/command" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/system" + "github.com/klauspost/pgzip" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -150,34 +155,56 @@ func runBackup(app config.App, serviceName string, bkConfig backupConfig) error } if err := container.RunExec(dcli, cl, targetContainer.ID, &preHookExecOpts); err != nil { - return err + return fmt.Errorf("failed to run %s on %s: %s", bkConfig.preHookCmd, targetContainer.ID, err.Error()) } logrus.Infof("succesfully ran %s pre-hook command: %s", fullServiceName, bkConfig.preHookCmd) } + var tempBackupPaths []string for _, remoteBackupPath := range bkConfig.backupPaths { timestamp := strconv.Itoa(time.Now().Nanosecond()) sanitisedPath := strings.ReplaceAll(remoteBackupPath, "/", "_") localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s%s_%s.tar.gz", fullServiceName, sanitisedPath, timestamp)) - logrus.Debugf("backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) + logrus.Debugf("temporarily backing up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) + + logrus.Infof("backing up %s:%s", fullServiceName, remoteBackupPath) content, _, err := cl.CopyFromContainer(context.Background(), targetContainer.ID, remoteBackupPath) if err != nil { - return err + logrus.Debugf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + return fmt.Errorf("failed to copy %s from container: %s", remoteBackupPath, err.Error()) } defer content.Close() - body, err := ioutil.ReadAll(content) - if err != nil { - return err + _, srcBase := archive.SplitPathDirEntry(remoteBackupPath) + preArchive := archive.RebaseArchiveEntries(content, srcBase, remoteBackupPath) + if err := copyToFile(localBackupPath, preArchive); err != nil { + logrus.Debugf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + return fmt.Errorf("failed to create tar archive (%s): %s", localBackupPath, err.Error()) } - if err := ioutil.WriteFile(localBackupPath, body, 0644); err != nil { - return err - } + tempBackupPaths = append(tempBackupPaths, localBackupPath) + } - logrus.Infof("backed up %s:%s to %s", fullServiceName, remoteBackupPath, localBackupPath) + logrus.Infof("compressing and merging archives...") + + if err := mergeArchives(tempBackupPaths, fullServiceName); err != nil { + logrus.Debugf("failed to merge archive files: %s", err.Error()) + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) + } + return fmt.Errorf("failed to merge archive files: %s", err.Error()) + } + + if err := cleanupTempArchives(tempBackupPaths); err != nil { + return fmt.Errorf("failed to clean up temporary archives: %s", err.Error()) } if bkConfig.postHookCmd != "" { @@ -203,3 +230,141 @@ func runBackup(app config.App, serviceName string, bkConfig backupConfig) error return nil } + +func copyToFile(outfile string, r io.Reader) error { + tmpFile, err := system.TempFileSequential(filepath.Dir(outfile), ".tar_temp") + if err != nil { + return err + } + + tmpPath := tmpFile.Name() + + _, err = io.Copy(tmpFile, r) + tmpFile.Close() + + if err != nil { + os.Remove(tmpPath) + return err + } + + if err = os.Rename(tmpPath, outfile); err != nil { + os.Remove(tmpPath) + return err + } + + return nil +} + +func cleanupTempArchives(tarPaths []string) error { + for _, tarPath := range tarPaths { + if err := os.RemoveAll(tarPath); err != nil { + return err + } + + logrus.Debugf("remove temporary archive file %s", tarPath) + } + + return nil +} + +func mergeArchives(tarPaths []string, serviceName string) error { + var out io.Writer + var cout *pgzip.Writer + + timestamp := strconv.Itoa(time.Now().Nanosecond()) + localBackupPath := filepath.Join(config.BACKUP_DIR, fmt.Sprintf("%s_%s.tar.gz", serviceName, timestamp)) + + fout, err := os.Create(localBackupPath) + if err != nil { + return fmt.Errorf("Failed to open %s: %s", localBackupPath, err) + } + + defer fout.Close() + out = fout + + cout = pgzip.NewWriter(out) + out = cout + + tw := tar.NewWriter(out) + + for _, tarPath := range tarPaths { + if err := addTar(tw, tarPath); err != nil { + return fmt.Errorf("failed to merge %s: %v", tarPath, err) + } + } + + if err := tw.Close(); err != nil { + return fmt.Errorf("failed to close tar writer %v", err) + } + + if cout != nil { + if err := cout.Flush(); err != nil { + return fmt.Errorf("failed to flush: %s", err) + } else if err = cout.Close(); err != nil { + return fmt.Errorf("failed to close compressed writer: %s", err) + } + } + + logrus.Infof("backed up %s to %s", serviceName, localBackupPath) + + return nil +} + +func addTar(tw *tar.Writer, pth string) (err error) { + var tr *tar.Reader + var rc io.ReadCloser + var hdr *tar.Header + + if tr, rc, err = openTarFile(pth); err != nil { + return + } + + for { + if hdr, err = tr.Next(); err != nil { + if err == io.EOF { + err = nil + } + break + } + if err = tw.WriteHeader(hdr); err != nil { + break + } else if _, err = io.Copy(tw, tr); err != nil { + break + } + } + if err == nil { + err = rc.Close() + } else { + rc.Close() + } + return +} + +func openTarFile(pth string) (tr *tar.Reader, rc io.ReadCloser, err error) { + var fin *os.File + var n int + buff := make([]byte, 1024) + + if fin, err = os.Open(pth); err != nil { + return + } + + if n, err = fin.Read(buff); err != nil { + fin.Close() + return + } else if n == 0 { + fin.Close() + err = fmt.Errorf("%s is empty", pth) + return + } + + if _, err = fin.Seek(0, 0); err != nil { + fin.Close() + return + } + + rc = fin + tr = tar.NewReader(rc) + + return tr, rc, nil +} diff --git a/go.mod b/go.mod index 0f488e15..5ef96b24 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/go-retryablehttp v0.7.1 github.com/kevinburke/ssh_config v1.2.0 + github.com/klauspost/pgzip v1.2.5 github.com/libdns/gandi v1.0.2 github.com/libdns/libdns v0.2.1 github.com/moby/sys/mount v0.2.0 // indirect diff --git a/go.sum b/go.sum index b098323f..912ae840 100644 --- a/go.sum +++ b/go.sum @@ -656,7 +656,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=