From c36c61ebfeda636678689b45649a175ebe16efc7 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 14 Jul 2014 19:31:19 +0300 Subject: [PATCH 1/2] archive: add buffers to operations with tarballs Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: a377844998f6a866d59bf356790f76d0d25bf01f Component: engine --- components/engine/archive/archive.go | 23 +++++++++++++++++------ components/engine/archive/changes.go | 4 +++- components/engine/archive/common.go | 4 ++++ components/engine/archive/diff.go | 5 ++++- 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 components/engine/archive/common.go diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index bf6e0b7797..550ef3f040 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -131,7 +131,7 @@ func (compression *Compression) Extension() string { return "" } -func addTarFile(path, name string, tw *tar.Writer) error { +func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error { fi, err := os.Lstat(path) if err != nil { return err @@ -181,11 +181,18 @@ func addTarFile(path, name string, tw *tar.Writer) error { if err != nil { return err } - if _, err := io.Copy(tw, file); err != nil { - file.Close() + + twBuf.Reset(tw) + _, err = io.Copy(twBuf, file) + file.Close() + if err != nil { return err } - file.Close() + err = twBuf.Flush() + if err != nil { + return err + } + twBuf.Reset(nil) } return nil @@ -328,6 +335,8 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) options.Includes = []string{"."} } + twBuf := bufio.NewWriterSize(nil, twBufSize) + for _, include := range options.Includes { filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { if err != nil { @@ -355,7 +364,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } } - if err := addTarFile(filePath, relFilePath, tw); err != nil { + if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", srcPath, err) } return nil @@ -394,6 +403,7 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { defer decompressedArchive.Close() tr := tar.NewReader(decompressedArchive) + trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header @@ -439,7 +449,8 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { } } } - if err := createTarFile(path, dest, hdr, tr, options == nil || !options.NoLchown); err != nil { + trBuf.Reset(tr) + if err := createTarFile(path, dest, hdr, trBuf, options == nil || !options.NoLchown); err != nil { return err } diff --git a/components/engine/archive/changes.go b/components/engine/archive/changes.go index 1e588b8eb5..154a527aec 100644 --- a/components/engine/archive/changes.go +++ b/components/engine/archive/changes.go @@ -1,6 +1,7 @@ package archive import ( + "bufio" "bytes" "fmt" "io" @@ -343,6 +344,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { tw := tar.NewWriter(writer) go func() { + twBuf := bufio.NewWriterSize(nil, twBufSize) // In general we log errors here but ignore them because // during e.g. a diff operation the container can continue // mutating the filesystem and we can see transient errors @@ -365,7 +367,7 @@ func ExportChanges(dir string, changes []Change) (Archive, error) { } } else { path := filepath.Join(dir, change.Path) - if err := addTarFile(path, change.Path[1:], tw); err != nil { + if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil { utils.Debugf("Can't add file %s to tar: %s\n", path, err) } } diff --git a/components/engine/archive/common.go b/components/engine/archive/common.go new file mode 100644 index 0000000000..2aac34e840 --- /dev/null +++ b/components/engine/archive/common.go @@ -0,0 +1,4 @@ +package archive + +const twBufSize = 32 * 1024 +const trBufSize = 32 * 1024 diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index d169669126..34a887663d 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -1,6 +1,7 @@ package archive import ( + "bufio" "fmt" "io" "io/ioutil" @@ -32,6 +33,7 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } tr := tar.NewReader(layer) + trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header @@ -108,7 +110,8 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } } - srcData := io.Reader(tr) + trBuf.Reset(tr) + srcData := io.Reader(trBuf) srcHdr := hdr // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so From 597e28c8ddf76f8908b936932c73c87e7bf3492f Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 16 Jul 2014 21:50:02 +0300 Subject: [PATCH 2/2] archive: add a benchmark for TarUntar Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: 76429cc11f6e2bd731e9ee77a2a538f15ef80eea Component: engine --- components/engine/archive/archive_test.go | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/components/engine/archive/archive_test.go b/components/engine/archive/archive_test.go index 61ee0af8e7..1a68234129 100644 --- a/components/engine/archive/archive_test.go +++ b/components/engine/archive/archive_test.go @@ -199,3 +199,42 @@ func TestUntarUstarGnuConflict(t *testing.T) { t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm") } } + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} + +func BenchmarkTarUntar(b *testing.B) { + origin, err := ioutil.TempDir("", "docker-test-untar-origin") + if err != nil { + b.Fatal(err) + } + tempDir, err := ioutil.TempDir("", "docker-test-untar-destination") + if err != nil { + b.Fatal(err) + } + target := path.Join(tempDir, "dest") + n, err := prepareUntarSourceDirectory(100, origin) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + b.SetBytes(int64(n)) + defer os.RemoveAll(origin) + defer os.RemoveAll(tempDir) + for n := 0; n < b.N; n++ { + err := TarUntar(origin, target) + if err != nil { + b.Fatal(err) + } + os.RemoveAll(target) + } +}