From 8061bcd2a73c0753ef09d8e30c52e9b5ec3d9c25 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 13 Jul 2017 18:56:00 -0700 Subject: [PATCH] archive: add test for prefix header With docker-17.06.0 some images pulled do not extract properly. Some files don't appear in correct directories. This may or may not cause the pull to fail. These images can't be pushed or saved. 17.06 is the first version of Docker built with go1.8. Cause There are multiple updates to the tar package in go1.8. https://go-review.googlesource.com/c/32234/ disables using "prefix" field when new tar archives are being written. Prefix field was previously set when a record in the archive used a path longer than 100 bytes. Another change https://go-review.googlesource.com/c/31444/ makes the reader ignore the "prefix" field value if the record is in GNU format. GNU format defines that same area should be used for access and modified times. If the "prefix" field is not read, a file will only be extracted by the basename. The problem is that with a previous version of the golang archive package headers could be written, that use the prefix field while at the same time setting the header format to GNU. This happens when numeric fields are big enough that they can not be written as octal strings and need to be written in binary. Usually, this shouldn't happen: uid, gid, devmajor, devminor can use up to 7 bytes, size and timestamp can use 11. If one of the records does overflow it switches the whole writer to GNU mode and all next files will be saved in GNU format. Signed-off-by: Tonis Tiigi Upstream-commit: 4a3cfda45e37b81211fbfbf0c45dbe64860a3ad0 Component: engine --- components/engine/Dockerfile.aarch64 | 1 + components/engine/Dockerfile.armhf | 1 + components/engine/Dockerfile.ppc64le | 1 + components/engine/Dockerfile.s390x | 1 + components/engine/Dockerfile.simple | 1 + components/engine/pkg/archive/archive_test.go | 19 +++++++++++++++++++ 6 files changed, 24 insertions(+) diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index 7a8f5f793c..cabcda28bd 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -97,6 +97,7 @@ RUN set -x \ # bootstrap, so we use golang-go (1.6) as bootstrap to build Go from source code. # We don't use the official ARMv6 released binaries as a GOROOT_BOOTSTRAP, because # not all ARM64 platforms support 32-bit mode. 32-bit mode is optional for ARMv8. +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN mkdir /usr/src/go && curl -fsSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/src/go -xz --strip-components=1 \ && cd /usr/src/go/src \ diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 6103c5a3ad..dd1f536191 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -70,6 +70,7 @@ RUN cd /usr/local/lvm2 \ # See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL # Install Go +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-armv6l.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index e64153800c..43b84e4501 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -94,6 +94,7 @@ RUN set -x \ # Install Go # NOTE: official ppc64le go binaries weren't available until go 1.6.4 and 1.7.4 +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-ppc64le.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index c69da3c969..35ec683739 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -87,6 +87,7 @@ RUN cd /usr/local/lvm2 \ && make install_device-mapper # See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-s390x.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/Dockerfile.simple b/components/engine/Dockerfile.simple index 4edc08f9ed..b4682d4cbc 100644 --- a/components/engine/Dockerfile.simple +++ b/components/engine/Dockerfile.simple @@ -53,6 +53,7 @@ RUN set -x \ # IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines # will need updating, to avoid errors. Ping #docker-maintainers on IRC # with a heads-up. +# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -xzC /usr/local diff --git a/components/engine/pkg/archive/archive_test.go b/components/engine/pkg/archive/archive_test.go index e85878fd0f..1371b8ab12 100644 --- a/components/engine/pkg/archive/archive_test.go +++ b/components/engine/pkg/archive/archive_test.go @@ -1203,6 +1203,25 @@ func TestReplaceFileTarWrapper(t *testing.T) { } } +// TestPrefixHeaderReadable tests that files that could be created with the +// version of this package that was built with <=go17 are still readable. +func TestPrefixHeaderReadable(t *testing.T) { + // https://gist.github.com/stevvooe/e2a790ad4e97425896206c0816e1a882#file-out-go + var testFile = []byte("\x1f\x8b\x08\x08\x44\x21\x68\x59\x00\x03\x74\x2e\x74\x61\x72\x00\x4b\xcb\xcf\x67\xa0\x35\x30\x80\x00\x86\x06\x10\x47\x01\xc1\x37\x40\x00\x54\xb6\xb1\xa1\xa9\x99\x09\x48\x25\x1d\x40\x69\x71\x49\x62\x91\x02\xe5\x76\xa1\x79\x84\x21\x91\xd6\x80\x72\xaf\x8f\x82\x51\x30\x0a\x46\x36\x00\x00\xf0\x1c\x1e\x95\x00\x06\x00\x00") + + tmpDir, err := ioutil.TempDir("", "prefix-test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + err = Untar(bytes.NewReader(testFile), tmpDir, nil) + require.NoError(t, err) + + baseName := "foo" + pth := strings.Repeat("a", 100-len(baseName)) + "/" + baseName + + _, err = os.Lstat(filepath.Join(tmpDir, pth)) + require.NoError(t, err) +} + func buildSourceArchive(t *testing.T, numberOfFiles int) (io.ReadCloser, func()) { srcDir, err := ioutil.TempDir("", "docker-test-srcDir") require.NoError(t, err)