From 83c915857f5f9da828b478528c816466ba451e63 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 21 Jun 2021 12:56:37 +0200 Subject: [PATCH] vendor: github.com/tonistiigi/fsutil d72af97c0eaf93c1d20360e3cb9c63c223675b83 full diff: https://github.com/tonistiigi/fsutil/compare/0834f99b7b85462efb69b4f571a4fa3ca7da5ac9...d72af97c0eaf93c1d20360e3cb9c63c223675b83 Signed-off-by: Sebastiaan van Stijn --- vendor.conf | 2 +- .../tonistiigi/fsutil/diff_containerd.go | 51 +++++++++++++++++-- vendor/github.com/tonistiigi/fsutil/go.mod | 24 ++++----- .../tonistiigi/fsutil/prefix/match.go | 45 ++++++++++++++++ .../github.com/tonistiigi/fsutil/receive.go | 13 ++++- vendor/github.com/tonistiigi/fsutil/walker.go | 31 ++--------- 6 files changed, 120 insertions(+), 46 deletions(-) create mode 100644 vendor/github.com/tonistiigi/fsutil/prefix/match.go diff --git a/vendor.conf b/vendor.conf index bbdd145a8e..64938edb90 100755 --- a/vendor.conf +++ b/vendor.conf @@ -68,7 +68,7 @@ github.com/sirupsen/logrus 6699a89a232f3db797f2e2806398 github.com/spf13/cobra 8380ddd3132bdf8fd77731725b550c181dda0aa8 # v1.1.3 github.com/spf13/pflag 2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab # v1.0.5 github.com/theupdateframework/notary bf96a202a09a312ae005cd312fc06ff4d2c166ce # v0.7.0-21-gbf96a202 -github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 +github.com/tonistiigi/fsutil d72af97c0eaf93c1d20360e3cb9c63c223675b83 github.com/tonistiigi/go-rosetta f79598599c5d34ea253b56a1d7c89bc6a96de7db github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/xeipuuv/gojsonpointer 02993c407bfbf5f6dae44c4f4b1cf6a39b5fc5bb diff --git a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go index 370271523b..d72f4e64fe 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go +++ b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go @@ -1,7 +1,9 @@ package fsutil import ( + "bytes" "context" + "io" "os" "strings" @@ -35,6 +37,8 @@ const ( // computed during a directory changes calculation. type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error +const compareChunkSize = 32 * 1024 + type currentPath struct { path string stat *types.Stat @@ -42,7 +46,7 @@ type currentPath struct { } // doubleWalkDiff walks both directories to create a diff -func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc) (err error) { +func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, filter FilterFunc, differ DiffType) (err error) { g, ctx := errgroup.WithContext(ctx) var ( @@ -116,7 +120,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil } f1 = nil case ChangeKindModify: - same, err := sameFile(f1, f2copy) + same, err := sameFile(f1, f2copy, differ) if err != nil { return err } @@ -165,7 +169,10 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { } } -func sameFile(f1, f2 *currentPath) (same bool, retErr error) { +func sameFile(f1, f2 *currentPath, differ DiffType) (same bool, retErr error) { + if differ == DiffNone { + return false, nil + } // If not a directory also check size, modtime, and content if !f1.stat.IsDir() { if f1.stat.Size_ != f2.stat.Size_ { @@ -177,7 +184,43 @@ func sameFile(f1, f2 *currentPath) (same bool, retErr error) { } } - return compareStat(f1.stat, f2.stat) + same, err := compareStat(f1.stat, f2.stat) + if err != nil || !same || differ == DiffMetadata { + return same, err + } + return compareFileContent(f1.path, f2.path) +} + +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, compareChunkSize) + b2 := make([]byte, compareChunkSize) + for { + n1, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } } // compareStat returns whether the stats are equivalent, diff --git a/vendor/github.com/tonistiigi/fsutil/go.mod b/vendor/github.com/tonistiigi/fsutil/go.mod index 075742a89f..fc4cdb0014 100644 --- a/vendor/github.com/tonistiigi/fsutil/go.mod +++ b/vendor/github.com/tonistiigi/fsutil/go.mod @@ -3,18 +3,18 @@ module github.com/tonistiigi/fsutil go 1.13 require ( - github.com/Microsoft/hcsshim v0.8.9 // indirect - github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc - github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c - github.com/gogo/protobuf v1.3.1 - github.com/moby/sys/mount v0.1.0 // indirect - github.com/moby/sys/mountinfo v0.1.3 // indirect + github.com/containerd/continuity v0.1.0 + github.com/docker/docker v20.10.3-0.20210609071616-4c2ec79bf2a8+incompatible // master (v21.xx-dev) + github.com/gogo/protobuf v1.3.2 + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/go-cmp v0.5.2 // indirect + github.com/kr/pretty v0.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/opencontainers/runc v1.0.0-rc10 // indirect github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.5.1 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20 - gotest.tools/v3 v3.0.2 // indirect + github.com/stretchr/testify v1.7.0 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gotest.tools/v3 v3.0.3 // indirect ) diff --git a/vendor/github.com/tonistiigi/fsutil/prefix/match.go b/vendor/github.com/tonistiigi/fsutil/prefix/match.go new file mode 100644 index 0000000000..57e745c681 --- /dev/null +++ b/vendor/github.com/tonistiigi/fsutil/prefix/match.go @@ -0,0 +1,45 @@ +package prefix + +import ( + "path" + "path/filepath" + "strings" +) + +// Match matches a path against a pattern. It returns m = true if the path +// matches the pattern, and partial = true if the pattern has more separators +// than the path and the common components match (for example, name = foo and +// pattern = foo/bar/*). slashSeparator determines whether the path and pattern +// are '/' delimited (true) or use the native path separator (false). +func Match(pattern, name string, slashSeparator bool) (m bool, partial bool) { + separator := filepath.Separator + if slashSeparator { + separator = '/' + } + count := strings.Count(name, string(separator)) + if strings.Count(pattern, string(separator)) > count { + pattern = trimUntilIndex(pattern, string(separator), count) + partial = true + } + if slashSeparator { + m, _ = path.Match(pattern, name) + } else { + m, _ = filepath.Match(pattern, name) + } + return m, partial +} + +func trimUntilIndex(str, sep string, count int) string { + s := str + i := 0 + c := 0 + for { + idx := strings.Index(s, sep) + s = s[idx+len(sep):] + i += idx + len(sep) + c++ + if c > count { + return str[:i-len(sep)] + } + } +} diff --git a/vendor/github.com/tonistiigi/fsutil/receive.go b/vendor/github.com/tonistiigi/fsutil/receive.go index 5c6a486978..209d1d2faf 100644 --- a/vendor/github.com/tonistiigi/fsutil/receive.go +++ b/vendor/github.com/tonistiigi/fsutil/receive.go @@ -11,12 +11,21 @@ import ( "golang.org/x/sync/errgroup" ) +type DiffType int + +const ( + DiffMetadata DiffType = iota + DiffNone + DiffContent +) + type ReceiveOpt struct { NotifyHashed ChangeFunc ContentHasher ContentHasher ProgressCb func(int, bool) Merge bool Filter FilterFunc + Differ DiffType } func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error { @@ -33,6 +42,7 @@ func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) erro progressCb: opt.ProgressCb, merge: opt.Merge, filter: opt.Filter, + differ: opt.Differ, } return r.run(ctx) } @@ -47,6 +57,7 @@ type receiver struct { progressCb func(int, bool) merge bool filter FilterFunc + differ DiffType notifyHashed ChangeFunc contentHasher ContentHasher @@ -132,7 +143,7 @@ func (r *receiver) run(ctx context.Context) error { if !r.merge { destWalker = getWalkerFn(r.dest) } - err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter) + err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill, r.filter, r.differ) if err != nil { return err } diff --git a/vendor/github.com/tonistiigi/fsutil/walker.go b/vendor/github.com/tonistiigi/fsutil/walker.go index b10383e4c5..cc6143db49 100644 --- a/vendor/github.com/tonistiigi/fsutil/walker.go +++ b/vendor/github.com/tonistiigi/fsutil/walker.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/pkg/fileutils" "github.com/pkg/errors" + "github.com/tonistiigi/fsutil/prefix" "github.com/tonistiigi/fsutil/types" ) @@ -96,8 +97,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err if !skip { matched := false partial := true - for _, p := range includePatterns { - if ok, p := matchPrefix(p, path); ok { + for _, pattern := range includePatterns { + if ok, p := prefix.Match(pattern, path, false); ok { matched = true if !p { partial = false @@ -190,32 +191,6 @@ func (s *StatInfo) Sys() interface{} { return s.Stat } -func matchPrefix(pattern, name string) (bool, bool) { - count := strings.Count(name, string(filepath.Separator)) - partial := false - if strings.Count(pattern, string(filepath.Separator)) > count { - pattern = trimUntilIndex(pattern, string(filepath.Separator), count) - partial = true - } - m, _ := filepath.Match(pattern, name) - return m, partial -} - -func trimUntilIndex(str, sep string, count int) string { - s := str - i := 0 - c := 0 - for { - idx := strings.Index(s, sep) - s = s[idx+len(sep):] - i += idx + len(sep) - c++ - if c > count { - return str[:i-len(sep)] - } - } -} - func isNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) }