From 81f2f2c3d5e4fc3c110e96775f31e3e9ae179d87 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 13 Jul 2017 18:56:00 -0700 Subject: [PATCH] archive: add test for prefix header With docker-17.06.0 some images pulled do not extract properly. Some files don't appear in correct directories. This may or may not cause the pull to fail. These images can't be pushed or saved. 17.06 is the first version of Docker built with go1.8. Cause There are multiple updates to the tar package in go1.8. https://go-review.googlesource.com/c/32234/ disables using "prefix" field when new tar archives are being written. Prefix field was previously set when a record in the archive used a path longer than 100 bytes. Another change https://go-review.googlesource.com/c/31444/ makes the reader ignore the "prefix" field value if the record is in GNU format. GNU format defines that same area should be used for access and modified times. If the "prefix" field is not read, a file will only be extracted by the basename. The problem is that with a previous version of the golang archive package headers could be written, that use the prefix field while at the same time setting the header format to GNU. This happens when numeric fields are big enough that they can not be written as octal strings and need to be written in binary. Usually, this shouldn't happen: uid, gid, devmajor, devminor can use up to 7 bytes, size and timestamp can use 11. If one of the records does overflow it switches the whole writer to GNU mode and all next files will be saved in GNU format. Signed-off-by: Tonis Tiigi (cherry picked from commit 4a3cfda45e37b81211fbfbf0c45dbe64860a3ad0) Signed-off-by: Andrew Hsu --- components/engine/Dockerfile.aarch64 | 1 + components/engine/Dockerfile.armhf | 1 + components/engine/Dockerfile.ppc64le | 1 + components/engine/Dockerfile.s390x | 1 + components/engine/Dockerfile.simple | 1 + components/engine/pkg/archive/archive_test.go | 19 +++++++++++++++++++ 6 files changed, 24 insertions(+) diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index b96d473d5b..34ad58f883 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -98,6 +98,7 @@ RUN set -x \ # bootstrap, so we use golang-go (1.6) as bootstrap to build Go from source code. # We don't use the official ARMv6 released binaries as a GOROOT_BOOTSTRAP, because # not all ARM64 platforms support 32-bit mode. 32-bit mode is optional for ARMv8. +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN mkdir /usr/src/go && curl -fsSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/src/go -xz --strip-components=1 \ && cd /usr/src/go/src \ diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index fe190a3fd0..13868c2431 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -71,6 +71,7 @@ RUN cd /usr/local/lvm2 \ # See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL # Install Go +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-armv6l.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index d61205573a..f6744c52f8 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -95,6 +95,7 @@ RUN set -x \ # Install Go # NOTE: official ppc64le go binaries weren't available until go 1.6.4 and 1.7.4 +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-ppc64le.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index c568480ecd..20b91137a9 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -88,6 +88,7 @@ RUN cd /usr/local/lvm2 \ && make install_device-mapper # See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-s390x.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/Dockerfile.simple b/components/engine/Dockerfile.simple index 4edc08f9ed..b4682d4cbc 100644 --- a/components/engine/Dockerfile.simple +++ b/components/engine/Dockerfile.simple @@ -53,6 +53,7 @@ RUN set -x \ # IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines # will need updating, to avoid errors. Ping #docker-maintainers on IRC # with a heads-up. +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/pkg/archive/archive_test.go b/components/engine/pkg/archive/archive_test.go index e351a0455b..1cbf0b8513 100644 --- a/components/engine/pkg/archive/archive_test.go +++ b/components/engine/pkg/archive/archive_test.go @@ -1216,6 +1216,25 @@ func TestReplaceFileTarWrapper(t *testing.T) { } } +// TestPrefixHeaderReadable tests that files that could be created with the +// version of this package that was built with <=go17 are still readable. +func TestPrefixHeaderReadable(t *testing.T) { + // https://gist.github.com/stevvooe/e2a790ad4e97425896206c0816e1a882#file-out-go + var testFile = []byte("\x1f\x8b\x08\x08\x44\x21\x68\x59\x00\x03\x74\x2e\x74\x61\x72\x00\x4b\xcb\xcf\x67\xa0\x35\x30\x80\x00\x86\x06\x10\x47\x01\xc1\x37\x40\x00\x54\xb6\xb1\xa1\xa9\x99\x09\x48\x25\x1d\x40\x69\x71\x49\x62\x91\x02\xe5\x76\xa1\x79\x84\x21\x91\xd6\x80\x72\xaf\x8f\x82\x51\x30\x0a\x46\x36\x00\x00\xf0\x1c\x1e\x95\x00\x06\x00\x00") + + tmpDir, err := ioutil.TempDir("", "prefix-test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + err = Untar(bytes.NewReader(testFile), tmpDir, nil) + require.NoError(t, err) + + baseName := "foo" + pth := strings.Repeat("a", 100-len(baseName)) + "/" + baseName + + _, err = os.Lstat(filepath.Join(tmpDir, pth)) + require.NoError(t, err) +} + func buildSourceArchive(t *testing.T, numberOfFiles int) (io.ReadCloser, func()) { srcDir, err := ioutil.TempDir("", "docker-test-srcDir") require.NoError(t, err)