From c6bfcea4c884c56b4860c83eb65b39a0fe356c4f Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Fri, 5 Jan 2018 02:58:14 +0000 Subject: [PATCH 1/5] Upgrade the frozen images to multi-arch Upgrade the frozen images to the multi-arch ones. Since issue #35963 is not fixed yet on linux/amd64, so we keep the busybox image on amd64 untouched. Signed-off-by: Dennis Chen Upstream-commit: eaae7750efbc80314c5e028c4d43d3cd9e104edd Component: engine --- components/engine/Dockerfile | 7 ++++--- components/engine/Dockerfile.armhf | 8 ++++---- components/engine/Dockerfile.e2e | 8 ++++---- components/engine/Dockerfile.ppc64le | 8 ++++---- components/engine/Dockerfile.s390x | 8 ++++---- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 5f78eda682..2a8f9181e6 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -167,11 +167,12 @@ RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ +# TODO: when issue #35963 fixed, we can upgrade the busybox to multi-arch RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - buildpack-deps:jessie@sha256:85b379ec16065e4fe4127eb1c5fb1bcc03c559bd36dbb2e22ff496de55925fa6 \ + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ busybox:latest@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f \ - debian:jessie@sha256:72f784399fd2719b4cb4e16ef8e369a39dc67f53d978cd3e2e7bf4e502c7b793 \ - hello-world:latest@sha256:c5515758d4c5e1e838e9cd307f6c6a0d620b5e07e6f927b07d05f6d12a1ac8d7 + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) # Install tomlv, vndr, runc, containerd, tini, docker-proxy dockercli diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index bc430e8c0b..b5a4d938a4 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -131,10 +131,10 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - armhf/buildpack-deps:jessie@sha256:eb2dad77ef53e88d94c3c83862d315c806ea1ca49b6e74f4db362381365ce489 \ - armhf/busybox:latest@sha256:016a1e149d2acc2a3789a160dfa60ce870794eea27ad5e96f7a101970e5e1689 \ - armhf/debian:jessie@sha256:ac59fa18b28d0ef751eabb5ba4c4b5a9063f99398bae2f70495aa8ed6139b577 \ - armhf/hello-world:latest@sha256:9701edc932223a66e49dd6c894a11db8c2cf4eccd1414f1ec105a623bf16b426 + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) # Install tomlv, vndr, runc, containerd, tini, docker-proxy diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index 7c6bb45d91..4dec3515a7 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -16,10 +16,10 @@ WORKDIR /go/src/github.com/docker/docker/ # Generate frozen images COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \ - buildpack-deps:jessie@sha256:85b379ec16065e4fe4127eb1c5fb1bcc03c559bd36dbb2e22ff496de55925fa6 \ - busybox:latest@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f \ - debian:jessie@sha256:72f784399fd2719b4cb4e16ef8e369a39dc67f53d978cd3e2e7bf4e502c7b793 \ - hello-world:latest@sha256:c5515758d4c5e1e838e9cd307f6c6a0d620b5e07e6f927b07d05f6d12a1ac8d7 + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # Download Docker CLI binary COPY hack/dockerfile hack/dockerfile diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index fa7307b3be..106a3a6f37 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -129,10 +129,10 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - ppc64le/buildpack-deps:jessie@sha256:1a2f2d2cc8738f14b336aeffc3503b5c9dedf9e1f26c7313cb4999534ad4716f \ - ppc64le/busybox:latest@sha256:54f34c83adfab20cf0e630d879e210f07b0062cd6caaf16346a61396d50e7584 \ - ppc64le/debian:jessie@sha256:ea8c5b105e3790f075145b40e4be1e4488c9f33f55e6cc45182047b80a68f892 \ - ppc64le/hello-world:latest@sha256:7d57adf137665f748956c86089320710b66d08584db3500ed98f4bb3da637c2d + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) # Install tomlv, vndr, runc, containerd, tini, docker-proxy diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index e8e78302df..e2f81a5d7d 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -123,10 +123,10 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ - s390x/buildpack-deps:jessie@sha256:552dec28146e4d2591fc0309aebdbac9e4fb1f335d90c70a14bbf72fb8bb1be5 \ - s390x/busybox:latest@sha256:e32f40c39ca596a4317392bd32809bb188c4ae5864ea827c3219c75c50069964 \ - s390x/debian:jessie@sha256:6994e3ffa5a1dabea09d536f350b3ed2715292cb469417c42a82b70fcbff7d32 \ - s390x/hello-world:latest@sha256:602db500fee63934292260e65c0c528128ad1c1c7c6497f95bbbac7d4d5312f1 + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) # Install tomlv, vndr, runc, containerd, tini, docker-proxy From 5c2e02eb8f8e63548930447c81cf9e06dbe9ed7a Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Fri, 5 Jan 2018 03:13:24 +0000 Subject: [PATCH 2/5] Removal of TEST_IMAGE_NAMESPACE We don't need the test image namespace anymore since we've already upgrade those images to the latest multi-arch ones. Signed-off-by: Dennis Chen Upstream-commit: 662bdb4a5638e56d78e66bb2cb5d7a4a751135c9 Component: engine --- components/engine/hack/make/.detect-daemon-osarch | 6 +----- components/engine/hack/make/.integration-test-helpers | 1 - components/engine/integration-cli/fixtures/load/frozen.go | 4 ---- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/components/engine/hack/make/.detect-daemon-osarch b/components/engine/hack/make/.detect-daemon-osarch index 7376c1d855..c472710d17 100644 --- a/components/engine/hack/make/.detect-daemon-osarch +++ b/components/engine/hack/make/.detect-daemon-osarch @@ -53,7 +53,6 @@ esac export PACKAGE_ARCH DOCKERFILE='Dockerfile' -TEST_IMAGE_NAMESPACE= case "$PACKAGE_ARCH" in amd64) case "${DOCKER_ENGINE_GOOS:-$DOCKER_CLIENT_GOOS}" in @@ -64,9 +63,6 @@ case "$PACKAGE_ARCH" in ;; *) DOCKERFILE="Dockerfile.$PACKAGE_ARCH" - if [ "$PACKAGE_ARCH" != "aarch64" ]; then - TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH" - fi ;; esac -export DOCKERFILE TEST_IMAGE_NAMESPACE +export DOCKERFILE diff --git a/components/engine/hack/make/.integration-test-helpers b/components/engine/hack/make/.integration-test-helpers index abd1d0f305..3b1326dee0 100644 --- a/components/engine/hack/make/.integration-test-helpers +++ b/components/engine/hack/make/.integration-test-helpers @@ -99,7 +99,6 @@ test_env() { HOME="$ABS_DEST/fake-HOME" \ PATH="$PATH" \ TEMP="$TEMP" \ - TEST_IMAGE_NAMESPACE="$TEST_IMAGE_NAMESPACE" \ TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \ "$@" ) diff --git a/components/engine/integration-cli/fixtures/load/frozen.go b/components/engine/integration-cli/fixtures/load/frozen.go index 4bec323b51..7b91fecd93 100644 --- a/components/engine/integration-cli/fixtures/load/frozen.go +++ b/components/engine/integration-cli/fixtures/load/frozen.go @@ -26,7 +26,6 @@ const frozenImgDir = "/docker-frozen-images" // images were passed in. If the images need to be downloaded, then it will respect // the passed in images func FrozenImagesLinux(client client.APIClient, images ...string) error { - imgNS := os.Getenv("TEST_IMAGE_NAMESPACE") var loadImages []struct{ srcName, destName string } for _, img := range images { if !imageExists(client, img) { @@ -38,9 +37,6 @@ func FrozenImagesLinux(client client.APIClient, images ...string) error { if img == "hello-world:frozen" { srcName = "hello-world:latest" } - if imgNS != "" { - srcName = imgNS + "/" + srcName - } loadImages = append(loadImages, struct{ srcName, destName string }{ srcName: srcName, destName: img, From 9251d7a17a15a13de1b11f4be0abb0ce07995752 Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Fri, 5 Jan 2018 03:17:17 +0000 Subject: [PATCH 3/5] busybox: Fix `ls` compatible issue BusyBox v1.26.2 (2017-03-09 00:04:38 UTC) supports `-le` option to get the full date and time information, while BusyBox v1.27.2 (2017-11-01 23:22:25 UTC, which is used by the official multi-arch image, uses `--full-time` instead of `-e` to get the same data. As a result, we will get below error for the `DockerSuite.TestBuildLastModified` test case in case of multi-arch image used: > docker_cli_build_test.go:446: > out2 = cli.DockerCmd(c, "run", name, "ls", "-le", "/file").Combined() > o/src/github.com/docker/docker/vendor/github.com/gotestyourself/gotestyourself/icmd/command.go:61: > t.Fatalf("at %s:%d - %s\n", filepath.Base(file), line, err.Error()) > ... Error: at cli.go:33 - > Command: /usr/local/bin/docker run testbuildlastmodified ls -le /file > ExitCode: 1 > Error: exit status 1 > Stdout: > Stderr: ls: invalid option -- e > BusyBox v1.27.2 (2017-11-01 23:22:25 UTC) multi-call binary. This PR tries to fix the above compatible issue for busybox image. Signed-off-by: Dennis Chen Upstream-commit: f88c2c04ef4dbae20eaddeaf69e605878f498041 Component: engine --- .../engine/integration-cli/docker_cli_build_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 8ae5e05ed8..97024cf651 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -400,20 +400,27 @@ func (s *DockerSuite) TestBuildLastModified(c *check.C) { defer server.Close() var out, out2 string + var args []string + // Temopray workaround for #35963. Will remove this when that issue fixed + if runtime.GOARCH == "amd64" { + args = []string{"run", name, "ls", "-le", "/file"} + } else { + args = []string{"run", name, "ls", "-l", "--full-time", "/file"} + } dFmt := `FROM busybox ADD %s/file /` dockerfile := fmt.Sprintf(dFmt, server.URL()) cli.BuildCmd(c, name, build.WithoutCache, build.WithDockerfile(dockerfile)) - out = cli.DockerCmd(c, "run", name, "ls", "-le", "/file").Combined() + out = cli.DockerCmd(c, args...).Combined() // Build it again and make sure the mtime of the file didn't change. // Wait a few seconds to make sure the time changed enough to notice time.Sleep(2 * time.Second) cli.BuildCmd(c, name, build.WithoutCache, build.WithDockerfile(dockerfile)) - out2 = cli.DockerCmd(c, "run", name, "ls", "-le", "/file").Combined() + out2 = cli.DockerCmd(c, args...).Combined() if out != out2 { c.Fatalf("MTime changed:\nOrigin:%s\nNew:%s", out, out2) @@ -428,7 +435,7 @@ ADD %s/file /` dockerfile = fmt.Sprintf(dFmt, server.URL()) cli.BuildCmd(c, name, build.WithoutCache, build.WithDockerfile(dockerfile)) - out2 = cli.DockerCmd(c, "run", name, "ls", "-le", "/file").Combined() + out2 = cli.DockerCmd(c, args...).Combined() if out == out2 { c.Fatalf("MTime didn't change:\nOrigin:%s\nNew:%s", out, out2) From 54836718fd348e8cca44b4a829f8aad3f2814553 Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Mon, 8 Jan 2018 06:15:24 +0000 Subject: [PATCH 4/5] busybox: use '-v' option for 'cat' Use `cat -v` command instead of `catv` for the latest version of busybox(V1.28.0) with multi-arch Signed-off-by: Dennis Chen Upstream-commit: ec6659a1216fbfe3fead759b0220501847d12e28 Component: engine --- .../engine/integration-cli/docker_cli_run_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 1749df07c1..b5dca1ec3c 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -2239,7 +2239,16 @@ func (s *DockerSuite) TestRunSlowStdoutConsumer(c *check.C) { // TODO Windows: This should be able to run on Windows if can find an // alternate to /dev/zero and /dev/stdout. testRequires(c, DaemonIsLinux) - cont := exec.Command(dockerBinary, "run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | catv") + + // TODO will remove this if issue #35963 fixed + var args []string + if runtime.GOARCH == "amd64" { + args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | catv"} + } else { + args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"} + } + + cont := exec.Command(dockerBinary, args...) stdout, err := cont.StdoutPipe() if err != nil { From 3b2e22c98e7f509d390987d5851ea7bc219a201c Mon Sep 17 00:00:00 2001 From: Boaz Shuster Date: Sun, 14 Jan 2018 11:10:49 +0000 Subject: [PATCH 5/5] Small nitpick: create ErrVolumeTargetIsRoot in the volume package Both lcow_parser.go and linux_parser.go are duplicating the error: "invalid specification: destination can't be '/'" This commit creates a new error called "ErrVolumeTargetIsRoot" that is used by both linux_parser and lcow_parser and remove the duplication in the code. Signed-off-by: Boaz Shuster Upstream-commit: 62143af5437a29d4b95f971d1905cfef763b0847 Component: engine --- components/engine/volume/lcow_parser.go | 3 +-- components/engine/volume/linux_parser.go | 2 +- components/engine/volume/parser.go | 5 +++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/components/engine/volume/lcow_parser.go b/components/engine/volume/lcow_parser.go index 8d15f0d9d7..b6a49d84df 100644 --- a/components/engine/volume/lcow_parser.go +++ b/components/engine/volume/lcow_parser.go @@ -2,7 +2,6 @@ package volume import ( "errors" - "fmt" "path" "github.com/docker/docker/api/types/mount" @@ -10,7 +9,7 @@ import ( var lcowSpecificValidators mountValidator = func(m *mount.Mount) error { if path.Clean(m.Target) == "/" { - return fmt.Errorf("invalid specification: destination can't be '/'") + return ErrVolumeTargetIsRoot } if m.Type == mount.TypeNamedPipe { return errors.New("Linux containers on Windows do not support named pipe mounts") diff --git a/components/engine/volume/linux_parser.go b/components/engine/volume/linux_parser.go index fd54e82162..43f1915382 100644 --- a/components/engine/volume/linux_parser.go +++ b/components/engine/volume/linux_parser.go @@ -29,7 +29,7 @@ func linuxSplitRawSpec(raw string) ([]string, error) { func linuxValidateNotRoot(p string) error { p = path.Clean(strings.Replace(p, `\`, `/`, -1)) if p == "/" { - return fmt.Errorf("invalid specification: destination can't be '/'") + return ErrVolumeTargetIsRoot } return nil } diff --git a/components/engine/volume/parser.go b/components/engine/volume/parser.go index 13fd7d1489..0b720313c7 100644 --- a/components/engine/volume/parser.go +++ b/components/engine/volume/parser.go @@ -1,6 +1,7 @@ package volume import ( + "errors" "runtime" "github.com/docker/docker/api/types/mount" @@ -13,6 +14,10 @@ const ( OSWindows = "windows" ) +// ErrVolumeTargetIsRoot is returned when the target destination is root. +// It's used by both LCOW and Linux parsers. +var ErrVolumeTargetIsRoot = errors.New("invalid specification: destination can't be '/'") + // Parser represents a platform specific parser for mount expressions type Parser interface { ParseMountRaw(raw, volumeDriver string) (*MountPoint, error)