From 143dde67838f8c6fd72bfed79aa3943b821afc3d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Feb 2018 18:24:21 -0500 Subject: [PATCH 1/7] Fix TestAttachAfterDetach to work with latest client Signed-off-by: Daniel Nephin Upstream-commit: 847b610620a8b8294d61c717d3c4aa13cb7a8b33 Component: engine --- .../docker_cli_attach_unix_test.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_attach_unix_test.go b/components/engine/integration-cli/docker_cli_attach_unix_test.go index e40d7cf39c..9affb944b1 100644 --- a/components/engine/integration-cli/docker_cli_attach_unix_test.go +++ b/components/engine/integration-cli/docker_cli_attach_unix_test.go @@ -69,10 +69,10 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) { cmd.Stdout = tty cmd.Stderr = tty - errChan := make(chan error) + cmdExit := make(chan error) go func() { - errChan <- cmd.Run() - close(errChan) + cmdExit <- cmd.Run() + close(cmdExit) }() c.Assert(waitRun(name), check.IsNil) @@ -82,12 +82,7 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) { cpty.Write([]byte{17}) select { - case err := <-errChan: - if err != nil { - buff := make([]byte, 200) - tty.Read(buff) - c.Fatalf("%s: %s", err, buff) - } + case <-cmdExit: case <-time.After(5 * time.Second): c.Fatal("timeout while detaching") } @@ -102,6 +97,7 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) { err = cmd.Start() c.Assert(err, checker.IsNil) + defer cmd.Process.Kill() bytes := make([]byte, 10) var nBytes int @@ -124,11 +120,7 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) { c.Fatal("timeout waiting for attach read") } - err = cmd.Wait() - c.Assert(err, checker.IsNil) - c.Assert(string(bytes[:nBytes]), checker.Contains, "/ #") - } // TestAttachDetach checks that attach in tty mode can be detached using the long container ID From f11213a80093591f2a3ac7ed0bb5a03312ffb533 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 16 Feb 2018 13:51:30 -0500 Subject: [PATCH 2/7] Split binary installers/commit scripts Originally I worked on this for the multi-stage build Dockerfile changes. Decided to split this out as we are still waiting for multi-stage to be available on CI and rebasing these is pretty annoying. Signed-off-by: Brian Goff Upstream-commit: b529d1b0936b90ae14d584c73f7332919f8d76b7 Component: engine --- components/engine/Dockerfile | 11 +- components/engine/Dockerfile.aarch64 | 13 +- components/engine/Dockerfile.armhf | 11 +- components/engine/Dockerfile.e2e | 2 +- components/engine/Dockerfile.ppc64le | 11 +- components/engine/Dockerfile.s390x | 11 +- components/engine/Dockerfile.simple | 9 +- .../engine/hack/dockerfile/binaries-commits | 23 --- .../hack/dockerfile/install-binaries.sh | 176 ------------------ .../dockerfile/install/containerd.installer | 31 +++ .../dockerfile/install/dockercli.installer | 31 +++ .../dockerfile/install/gometalinter.installer | 12 ++ .../engine/hack/dockerfile/install/install.sh | 30 +++ .../hack/dockerfile/install/proxy.installer | 37 ++++ .../hack/dockerfile/install/runc.installer | 17 ++ .../hack/dockerfile/install/tini.installer | 14 ++ .../hack/dockerfile/install/tomlv.installer | 12 ++ .../hack/dockerfile/install/vndr.installer | 11 ++ components/engine/hack/make/.go-autogen | 4 +- components/engine/vendor.conf | 6 +- 20 files changed, 239 insertions(+), 233 deletions(-) delete mode 100644 components/engine/hack/dockerfile/binaries-commits delete mode 100755 components/engine/hack/dockerfile/install-binaries.sh create mode 100755 components/engine/hack/dockerfile/install/containerd.installer create mode 100755 components/engine/hack/dockerfile/install/dockercli.installer create mode 100755 components/engine/hack/dockerfile/install/gometalinter.installer create mode 100755 components/engine/hack/dockerfile/install/install.sh create mode 100755 components/engine/hack/dockerfile/install/proxy.installer create mode 100755 components/engine/hack/dockerfile/install/runc.installer create mode 100755 components/engine/hack/dockerfile/install/tini.installer create mode 100755 components/engine/hack/dockerfile/install/tomlv.installer create mode 100755 components/engine/hack/dockerfile/install/vndr.installer diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 30c5dcd291..bd42d81c46 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -187,11 +187,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) -# Install tomlv, vndr, runc, containerd, tini, docker-proxy dockercli -# Please edit hack/dockerfile/install-binaries.sh to update them. -COPY hack/dockerfile/binaries-commits /tmp/binaries-commits -COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh -RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter +# Install tomlv, vndr, runc, containerd, tini, proxy dockercli +# Please edit hack/dockerfile/install/.installer to update them. +COPY hack/dockerfile/install hack/dockerfile/install +RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ + do hack/dockerfile/install/install.sh $i; \ + done ENV PATH=/usr/local/cli:$PATH # Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index 8907384496..68df256e74 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -151,14 +151,17 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) +# -# Install tomlv, vndr, runc, containerd, tini, docker-proxy -# Please edit hack/dockerfile/install-binaries.sh to update them. -COPY hack/dockerfile/binaries-commits /tmp/binaries-commits -COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh -RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter +# Install tomlv, vndr, runc, containerd, tini, proxy dockercli +# Please edit hack/dockerfile/install/.installer to update them. +COPY hack/dockerfile/install hack/dockerfile/install +RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ + do hack/dockerfile/install/install.sh $i; \ + done ENV PATH=/usr/local/cli:$PATH + # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index b6a8ed301f..1978be7610 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -138,11 +138,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) -# Install tomlv, vndr, runc, containerd, tini, docker-proxy -# Please edit hack/dockerfile/install-binaries.sh to update them. -COPY hack/dockerfile/binaries-commits /tmp/binaries-commits -COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh -RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter +# Install tomlv, vndr, runc, containerd, tini, proxy dockercli +# Please edit hack/dockerfile/install/.installer to update them. +COPY hack/dockerfile/install hack/dockerfile/install +RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ + do hack/dockerfile/install/install.sh $i; \ + done ENV PATH=/usr/local/cli:$PATH ENTRYPOINT ["hack/dind"] diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index bfac86d291..88edc5a907 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -23,7 +23,7 @@ RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \ # Download Docker CLI binary COPY hack/dockerfile hack/dockerfile -RUN hack/dockerfile/install-binaries.sh dockercli +RUN hack/dockerfile/install.sh dockercli # Set tag and add sources ARG DOCKER_GITCOMMIT diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 690ed491c1..c4aaa831e8 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -136,11 +136,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) -# Install tomlv, vndr, runc, containerd, tini, docker-proxy -# Please edit hack/dockerfile/install-binaries.sh to update them. -COPY hack/dockerfile/binaries-commits /tmp/binaries-commits -COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh -RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter +# Install tomlv, vndr, runc, containerd, tini, proxy dockercli +# Please edit hack/dockerfile/install/.installer to update them. +COPY hack/dockerfile/install hack/dockerfile/install +RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ + do hack/dockerfile/install/install.sh $i; \ + done ENV PATH=/usr/local/cli:$PATH # Wrap all commands in the "docker-in-docker" script to allow nested containers diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index 53a567dcab..d6bc28ecf2 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -130,11 +130,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) -# Install tomlv, vndr, runc, containerd, tini, docker-proxy -# Please edit hack/dockerfile/install-binaries.sh to update them. -COPY hack/dockerfile/binaries-commits /tmp/binaries-commits -COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh -RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter +# Install tomlv, vndr, runc, containerd, tini, proxy dockercli +# Please edit hack/dockerfile/install/.installer to update them. +COPY hack/dockerfile/install hack/dockerfile/install +RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ + do hack/dockerfile/install/install.sh $i; \ + done ENV PATH=/usr/local/cli:$PATH # Wrap all commands in the "docker-in-docker" script to allow nested containers diff --git a/components/engine/Dockerfile.simple b/components/engine/Dockerfile.simple index 6ab59da6db..b1e5584257 100644 --- a/components/engine/Dockerfile.simple +++ b/components/engine/Dockerfile.simple @@ -50,10 +50,11 @@ ENV GOPATH /go ENV CGO_LDFLAGS -L/lib # Install runc, containerd, tini and docker-proxy -# Please edit hack/dockerfile/install-binaries.sh to update them. -COPY hack/dockerfile/binaries-commits /tmp/binaries-commits -COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh -RUN /tmp/install-binaries.sh runc containerd tini proxy dockercli +# Please edit hack/dockerfile/install/.installer to update them. +COPY hack/dockerfile/install hack/dockerfile/install +RUN for i in runc containerd tini proxy dockercli; \ + do hack/dockerfile/install/install.sh $i; \ + done ENV PATH=/usr/local/cli:$PATH ENV AUTO_GOPATH 1 diff --git a/components/engine/hack/dockerfile/binaries-commits b/components/engine/hack/dockerfile/binaries-commits deleted file mode 100644 index b04b0d3ae2..0000000000 --- a/components/engine/hack/dockerfile/binaries-commits +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# When updating TOMLV_COMMIT, consider updating github.com/BurntSushi/toml -# in vendor.conf accordingly -TOMLV_COMMIT=a368813c5e648fee92e5f6c30e3944ff9d5e8895 - -# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly -RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5 - -# containerd is also pinned in vendor.conf. When updating the binary -# version you may also need to update the vendor version to pick up bug -# fixes or new APIs. -CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2 -TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 - -# LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When -# updating the binary version, consider updating github.com/docker/libnetwork -# in vendor.conf accordingly -LIBNETWORK_COMMIT=ed2130d117c11c542327b4d5216a5db36770bc65 -VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384 - -# Linting -GOMETALINTER_COMMIT=bfcc1d6942136fd86eb6f1a6fb328de8398fbd80 diff --git a/components/engine/hack/dockerfile/install-binaries.sh b/components/engine/hack/dockerfile/install-binaries.sh deleted file mode 100755 index 69885a5eac..0000000000 --- a/components/engine/hack/dockerfile/install-binaries.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -set -e -set -x - -. $(dirname "$0")/binaries-commits - -RM_GOPATH=0 - -TMP_GOPATH=${TMP_GOPATH:-""} - -if [ -z "$TMP_GOPATH" ]; then - export GOPATH="$(mktemp -d)" - RM_GOPATH=1 -else - export GOPATH="$TMP_GOPATH" -fi - -# Do not build with ambient capabilities support -RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}" - -install_runc() { - echo "Install runc version $RUNC_COMMIT" - git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" - cd "$GOPATH/src/github.com/opencontainers/runc" - git checkout -q "$RUNC_COMMIT" - make BUILDTAGS="$RUNC_BUILDTAGS" $1 - cp runc /usr/local/bin/docker-runc -} - -install_containerd() { - echo "Install containerd version $CONTAINERD_COMMIT" - git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd" - cd "$GOPATH/src/github.com/containerd/containerd" - git checkout -q "$CONTAINERD_COMMIT" - ( - export GOPATH - make - ) - cp bin/containerd /usr/local/bin/docker-containerd - cp bin/containerd-shim /usr/local/bin/docker-containerd-shim - cp bin/ctr /usr/local/bin/docker-containerd-ctr -} - -install_containerd_static() { - echo "Install containerd version $CONTAINERD_COMMIT" - git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd" - cd "$GOPATH/src/github.com/containerd/containerd" - git checkout -q "$CONTAINERD_COMMIT" - ( - export GOPATH - make BUILDTAGS='static_build netgo' EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' - ) - cp bin/containerd /usr/local/bin/docker-containerd - cp bin/containerd-shim /usr/local/bin/docker-containerd-shim - cp bin/ctr /usr/local/bin/docker-containerd-ctr -} - -install_proxy() { - echo "Install docker-proxy version $LIBNETWORK_COMMIT" - git clone https://github.com/docker/libnetwork.git "$GOPATH/src/github.com/docker/libnetwork" - cd "$GOPATH/src/github.com/docker/libnetwork" - git checkout -q "$LIBNETWORK_COMMIT" - go build -buildmode=pie -ldflags="$PROXY_LDFLAGS" -o /usr/local/bin/docker-proxy github.com/docker/libnetwork/cmd/proxy -} - -install_dockercli() { - DOCKERCLI_CHANNEL=${DOCKERCLI_CHANNEL:-edge} - DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce} - echo "Install docker/cli version $DOCKERCLI_VERSION from $DOCKERCLI_CHANNEL" - - arch=$(uname -m) - # No official release of these platforms - if [[ "$arch" != "x86_64" ]] && [[ "$arch" != "s390x" ]]; then - build_dockercli - return - fi - - url=https://download.docker.com/linux/static - curl -Ls $url/$DOCKERCLI_CHANNEL/$arch/docker-$DOCKERCLI_VERSION.tgz | \ - tar -xz docker/docker - mv docker/docker /usr/local/bin/ - rmdir docker -} - -build_dockercli() { - DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce} - git clone https://github.com/docker/docker-ce "$GOPATH/tmp/docker-ce" - cd "$GOPATH/tmp/docker-ce" - git checkout -q "v$DOCKERCLI_VERSION" - mkdir -p "$GOPATH/src/github.com/docker" - mv components/cli "$GOPATH/src/github.com/docker/cli" - go build -buildmode=pie -o /usr/local/bin/docker github.com/docker/cli/cmd/docker -} - -install_gometalinter() { - echo "Installing gometalinter version $GOMETALINTER_COMMIT" - go get -d github.com/alecthomas/gometalinter - cd "$GOPATH/src/github.com/alecthomas/gometalinter" - git checkout -q "$GOMETALINTER_COMMIT" - go build -buildmode=pie -o /usr/local/bin/gometalinter github.com/alecthomas/gometalinter - GOBIN=/usr/local/bin gometalinter --install -} - -for prog in "$@" -do - case $prog in - tomlv) - echo "Install tomlv version $TOMLV_COMMIT" - git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" - cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT" - go build -buildmode=pie -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv - ;; - - runc) - install_runc static - ;; - - runc-dynamic) - install_runc - ;; - - containerd) - install_containerd_static - ;; - - containerd-dynamic) - install_containerd - ;; - - gometalinter) - install_gometalinter - ;; - - tini) - echo "Install tini version $TINI_COMMIT" - git clone https://github.com/krallin/tini.git "$GOPATH/tini" - cd "$GOPATH/tini" - git checkout -q "$TINI_COMMIT" - cmake . - make tini-static - cp tini-static /usr/local/bin/docker-init - ;; - - proxy) - ( - export CGO_ENABLED=0 - install_proxy - ) - ;; - - proxy-dynamic) - PROXY_LDFLAGS="-linkmode=external" install_proxy - ;; - - vndr) - echo "Install vndr version $VNDR_COMMIT" - git clone https://github.com/LK4D4/vndr.git "$GOPATH/src/github.com/LK4D4/vndr" - cd "$GOPATH/src/github.com/LK4D4/vndr" - git checkout -q "$VNDR_COMMIT" - go build -buildmode=pie -v -o /usr/local/bin/vndr . - ;; - - dockercli) - install_dockercli - ;; - - *) - echo echo "Usage: $0 [tomlv|runc|runc-dynamic|containerd|containerd-dynamic|tini|proxy|proxy-dynamic|vndr|dockercli|gometalinter]" - exit 1 - - esac -done - -if [ $RM_GOPATH -eq 1 ]; then - rm -rf "$GOPATH" -fi diff --git a/components/engine/hack/dockerfile/install/containerd.installer b/components/engine/hack/dockerfile/install/containerd.installer new file mode 100755 index 0000000000..5731a6d560 --- /dev/null +++ b/components/engine/hack/dockerfile/install/containerd.installer @@ -0,0 +1,31 @@ +#!/bin/sh + + +# containerd is also pinned in vendor.conf. When updating the binary +# version you may also need to update the vendor version to pick up bug +# fixes or new APIs. +CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2 + +install_containerd() { + echo "Install containerd version $CONTAINERD_COMMIT" + git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd" + cd "$GOPATH/src/github.com/containerd/containerd" + git checkout -q "$CONTAINERD_COMMIT" + + ( + + if [ "$1" == "static" ]; then + export BUILDTAGS='static_build netgo' + export EXTRA_FLAGS='-buildmod pie' + export EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' + fi + + make + ) + + mkdir -p ${PREFIX} + + cp bin/containerd ${PREFIX}/docker-containerd + cp bin/containerd-shim ${PREFIX}/docker-containerd-shim + cp bin/ctr ${PREFIX}/docker-containerd-ctr +} diff --git a/components/engine/hack/dockerfile/install/dockercli.installer b/components/engine/hack/dockerfile/install/dockercli.installer new file mode 100755 index 0000000000..ae3aa0dd45 --- /dev/null +++ b/components/engine/hack/dockerfile/install/dockercli.installer @@ -0,0 +1,31 @@ +#!/bin/sh + +DOCKERCLI_CHANNEL=${DOCKERCLI_CHANNEL:-edge} +DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce} + +install_dockercli() { + echo "Install docker/cli version $DOCKERCLI_VERSION from $DOCKERCLI_CHANNEL" + + arch=$(uname -m) + # No official release of these platforms + if [[ "$arch" != "x86_64" ]] && [[ "$arch" != "s390x" ]]; then + build_dockercli + return + fi + + url=https://download.docker.com/linux/static + curl -Ls $url/$DOCKERCLI_CHANNEL/$arch/docker-$DOCKERCLI_VERSION.tgz | \ + tar -xz docker/docker + mkdir -p ${PREFIX} + mv docker/docker ${PREFIX}/ + rmdir docker +} + +build_dockercli() { + git clone https://github.com/docker/docker-ce "$GOPATH/tmp/docker-ce" + cd "$GOPATH/tmp/docker-ce" + git checkout -q "v$DOCKERCLI_VERSION" + mkdir -p "$GOPATH/src/github.com/docker" + mv components/cli "$GOPATH/src/github.com/docker/cli" + go build -buildmode=pie -o ${PREFIX}/docker github.com/docker/cli/cmd/docker +} diff --git a/components/engine/hack/dockerfile/install/gometalinter.installer b/components/engine/hack/dockerfile/install/gometalinter.installer new file mode 100755 index 0000000000..13500e1c89 --- /dev/null +++ b/components/engine/hack/dockerfile/install/gometalinter.installer @@ -0,0 +1,12 @@ +#!/bin/sh + +GOMETALINTER_COMMIT=bfcc1d6942136fd86eb6f1a6fb328de8398fbd80 + +install_gometalinter() { + echo "Installing gometalinter version $GOMETALINTER_COMMIT" + go get -d github.com/alecthomas/gometalinter + cd "$GOPATH/src/github.com/alecthomas/gometalinter" + git checkout -q "$GOMETALINTER_COMMIT" + go build -buildmode=pie -o ${PREFIX}/gometalinter github.com/alecthomas/gometalinter + GOBIN=${PREFIX} ${PREFIX}/gometalinter --install +} diff --git a/components/engine/hack/dockerfile/install/install.sh b/components/engine/hack/dockerfile/install/install.sh new file mode 100755 index 0000000000..a0ff09da55 --- /dev/null +++ b/components/engine/hack/dockerfile/install/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +set -x + +RM_GOPATH=0 + +TMP_GOPATH=${TMP_GOPATH:-""} + +: ${PREFIX:="/usr/local/bin"} + +if [ -z "$TMP_GOPATH" ]; then + export GOPATH="$(mktemp -d)" + RM_GOPATH=1 +else + export GOPATH="$TMP_GOPATH" +fi + +dir="$(dirname $0)" + +bin=$1 +shift + +if [ ! -f "${dir}/${bin}.installer" ]; then + echo "Could not find installer for \"$bin\"" + exit 1 +fi + +. $dir/$bin.installer +install_$bin "$@" diff --git a/components/engine/hack/dockerfile/install/proxy.installer b/components/engine/hack/dockerfile/install/proxy.installer new file mode 100755 index 0000000000..598f67c3fb --- /dev/null +++ b/components/engine/hack/dockerfile/install/proxy.installer @@ -0,0 +1,37 @@ +#!/bin/sh + +# LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When +# updating the binary version, consider updating github.com/docker/libnetwork +# in vendor.conf accordingly +LIBNETWORK_COMMIT=ed2130d117c11c542327b4d5216a5db36770bc65 + +install_proxy() { + case "$1" in + "dynamic") + install_proxy_dynamic + return + ;; + "") + export CGO_ENABLED=0 + _install_proxy + ;; + *) + echo 'Usage: $0 [dynamic]' + ;; + esac +} + +install_proxy_dynamic() { + export PROXY_LDFLAGS="-linkmode=external" install_proxy + _install_proxy +} + +_install_proxy() { + echo "Install docker-proxy version $LIBNETWORK_COMMIT" + git clone https://github.com/docker/libnetwork.git "$GOPATH/src/github.com/docker/libnetwork" + cd "$GOPATH/src/github.com/docker/libnetwork" + git checkout -q "$LIBNETWORK_COMMIT" + go build -buildmode=pie -ldflags="$PROXY_LDFLAGS" -o ${PREFIX}/docker-proxy github.com/docker/libnetwork/cmd/proxy +} + + diff --git a/components/engine/hack/dockerfile/install/runc.installer b/components/engine/hack/dockerfile/install/runc.installer new file mode 100755 index 0000000000..923ae706e1 --- /dev/null +++ b/components/engine/hack/dockerfile/install/runc.installer @@ -0,0 +1,17 @@ +#!/bin/sh + +# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly +RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5 + +install_runc() { + # Do not build with ambient capabilities support + RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}" + + echo "Install runc version $RUNC_COMMIT" + git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" + cd "$GOPATH/src/github.com/opencontainers/runc" + git checkout -q "$RUNC_COMMIT" + make BUILDTAGS="$RUNC_BUILDTAGS" $1 + mkdir -p ${PREFIX} + cp runc ${PREFIX}/docker-runc +} diff --git a/components/engine/hack/dockerfile/install/tini.installer b/components/engine/hack/dockerfile/install/tini.installer new file mode 100755 index 0000000000..d1c2deb62e --- /dev/null +++ b/components/engine/hack/dockerfile/install/tini.installer @@ -0,0 +1,14 @@ +#!/bin/sh + +TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 + +install_tini() { + echo "Install tini version $TINI_COMMIT" + git clone https://github.com/krallin/tini.git "$GOPATH/tini" + cd "$GOPATH/tini" + git checkout -q "$TINI_COMMIT" + cmake . + make tini-static + mkdir -p ${PREFIX} + cp tini-static ${PREFIX}/docker-init +} diff --git a/components/engine/hack/dockerfile/install/tomlv.installer b/components/engine/hack/dockerfile/install/tomlv.installer new file mode 100755 index 0000000000..c926454f22 --- /dev/null +++ b/components/engine/hack/dockerfile/install/tomlv.installer @@ -0,0 +1,12 @@ +#!/bin/sh + +# When updating TOMLV_COMMIT, consider updating github.com/BurntSushi/toml +# in vendor.conf accordingly +TOMLV_COMMIT=a368813c5e648fee92e5f6c30e3944ff9d5e8895 + +install_tomlv() { + echo "Install tomlv version $TOMLV_COMMIT" + git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" + cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT" + go build -v -buildmode=pie -o ${PREFIX}/tomlv github.com/BurntSushi/toml/cmd/tomlv +} diff --git a/components/engine/hack/dockerfile/install/vndr.installer b/components/engine/hack/dockerfile/install/vndr.installer new file mode 100755 index 0000000000..1d30eecc38 --- /dev/null +++ b/components/engine/hack/dockerfile/install/vndr.installer @@ -0,0 +1,11 @@ +#!/bin/sh + +VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384 + +install_vndr() { + echo "Install vndr version $VNDR_COMMIT" + git clone https://github.com/LK4D4/vndr.git "$GOPATH/src/github.com/LK4D4/vndr" + cd "$GOPATH/src/github.com/LK4D4/vndr" + git checkout -q "$VNDR_COMMIT" + go build -buildmode=pie -v -o ${PREFIX}/vndr . +} diff --git a/components/engine/hack/make/.go-autogen b/components/engine/hack/make/.go-autogen index 850c3ec9ac..ba001895d8 100644 --- a/components/engine/hack/make/.go-autogen +++ b/components/engine/hack/make/.go-autogen @@ -2,7 +2,9 @@ rm -rf autogen -source hack/dockerfile/binaries-commits +source hack/dockerfile/install/runc.installer +source hack/dockerfile/install/tini.installer +source hack/dockerfile/install/containerd.installer cat > dockerversion/version_autogen.go < Date: Wed, 21 Feb 2018 16:08:32 -0500 Subject: [PATCH 3/7] Bump swarmkit Signed-off-by: Brian Goff Upstream-commit: be713ec486090db312f9a83481a2c4663d473f71 Component: engine --- components/engine/vendor.conf | 2 +- .../swarmkit/manager/dispatcher/dispatcher.go | 111 +++++++----------- 2 files changed, 46 insertions(+), 67 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 42fbd52bd7..f00b59b3e1 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -120,7 +120,7 @@ github.com/dmcgowan/go-tar go1.10 github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577 # cluster -github.com/docker/swarmkit f74983e7c015a38a81c8642803a78b8322cf7eac +github.com/docker/swarmkit 49a9d7f6ba3c1925262641e694c18eb43575f74b github.com/gogo/protobuf v0.4 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index bf48818ea5..7b9e175438 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -125,12 +125,17 @@ type clusterUpdate struct { // Dispatcher is responsible for dispatching tasks and tracking agent health. type Dispatcher struct { - // mu is a lock to provide mutually exclusive access to dispatcher fields - // e.g. lastSeenManagers, networkBootstrapKeys, lastSeenRootCert etc. + // Mutex to synchronize access to dispatcher shared state e.g. nodes, + // lastSeenManagers, networkBootstrapKeys etc. + // TODO(anshul): This can potentially be removed and rpcRW used in its place. mu sync.Mutex - // shutdownWait is used by stop() to wait for existing operations to finish. - shutdownWait sync.WaitGroup - + // WaitGroup to handle the case when Stop() gets called before Run() + // has finished initializing the dispatcher. + wg sync.WaitGroup + // This RWMutex synchronizes RPC handlers and the dispatcher stop(). + // The RPC handlers use the read lock while stop() uses the write lock + // and acts as a barrier to shutdown. + rpcRW sync.RWMutex nodes *nodeStore store *store.MemoryStore lastSeenManagers []*api.WeightedPeer @@ -253,11 +258,8 @@ func (d *Dispatcher) Run(ctx context.Context) error { defer cancel() d.ctx, d.cancel = context.WithCancel(ctx) ctx = d.ctx - - // If Stop() is called, it should wait - // for Run() to complete. - d.shutdownWait.Add(1) - defer d.shutdownWait.Done() + d.wg.Add(1) + defer d.wg.Done() d.mu.Unlock() publishManagers := func(peers []*api.Peer) { @@ -320,15 +322,18 @@ func (d *Dispatcher) Stop() error { return errors.New("dispatcher is already stopped") } - // Cancel dispatcher context. - // This should also close the the streams in Tasks(), Assignments(). + log := log.G(d.ctx).WithField("method", "(*Dispatcher).Stop") + log.Info("dispatcher stopping") d.cancel() d.mu.Unlock() - // Wait for the RPCs that are in-progress to finish. - d.shutdownWait.Wait() - + // The active nodes list can be cleaned out only when all + // existing RPCs have finished. + // RPCs that start after rpcRW.Unlock() should find the context + // cancelled and should fail organically. + d.rpcRW.Lock() d.nodes.Clean() + d.rpcRW.Unlock() d.processUpdatesLock.Lock() // In case there are any waiters. There is no chance of any starting @@ -338,6 +343,14 @@ func (d *Dispatcher) Stop() error { d.processUpdatesLock.Unlock() d.clusterUpdateQueue.Close() + + // TODO(anshul): This use of Wait() could be unsafe. + // According to go's documentation on WaitGroup, + // Add() with a positive delta that occur when the counter is zero + // must happen before a Wait(). + // As is, dispatcher Stop() can race with Run(). + d.wg.Wait() + return nil } @@ -485,13 +498,13 @@ func nodeIPFromContext(ctx context.Context) (string, error) { // register is used for registration of node with particular dispatcher. func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, error) { + logLocal := log.G(ctx).WithField("method", "(*Dispatcher).register") + // prevent register until we're ready to accept it dctx, err := d.isRunningLocked() if err != nil { return "", err } - logLocal := log.G(ctx).WithField("method", "(*Dispatcher).register") - if err := d.nodes.CheckRateLimit(nodeID); err != nil { return "", err } @@ -539,15 +552,8 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a // UpdateTaskStatus updates status of task. Node should send such updates // on every status change of its tasks. func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStatusRequest) (*api.UpdateTaskStatusResponse, error) { - // shutdownWait.Add() followed by isRunning() to ensures that - // if this rpc sees the dispatcher running, - // it will already have called Add() on the shutdownWait wait, - // which ensures that Stop() will wait for this rpc to complete. - // Note that Stop() first does Dispatcher.ctx.cancel() followed by - // shutdownWait.Wait() to make sure new rpc's don't start before waiting - // for existing ones to finish. - d.shutdownWait.Add(1) - defer d.shutdownWait.Done() + d.rpcRW.RLock() + defer d.rpcRW.RUnlock() dctx, err := d.isRunningLocked() if err != nil { @@ -740,15 +746,8 @@ func (d *Dispatcher) processUpdates(ctx context.Context) { // of tasks which should be run on node, if task is not present in that list, // it should be terminated. func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServer) error { - // shutdownWait.Add() followed by isRunning() to ensures that - // if this rpc sees the dispatcher running, - // it will already have called Add() on the shutdownWait wait, - // which ensures that Stop() will wait for this rpc to complete. - // Note that Stop() first does Dispatcher.ctx.cancel() followed by - // shutdownWait.Wait() to make sure new rpc's don't start before waiting - // for existing ones to finish. - d.shutdownWait.Add(1) - defer d.shutdownWait.Done() + d.rpcRW.RLock() + defer d.rpcRW.RUnlock() dctx, err := d.isRunningLocked() if err != nil { @@ -873,15 +872,8 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe // Assignments is a stream of assignments for a node. Each message contains // either full list of tasks and secrets for the node, or an incremental update. func (d *Dispatcher) Assignments(r *api.AssignmentsRequest, stream api.Dispatcher_AssignmentsServer) error { - // shutdownWait.Add() followed by isRunning() to ensures that - // if this rpc sees the dispatcher running, - // it will already have called Add() on the shutdownWait wait, - // which ensures that Stop() will wait for this rpc to complete. - // Note that Stop() first does Dispatcher.ctx.cancel() followed by - // shutdownWait.Wait() to make sure new rpc's don't start before waiting - // for existing ones to finish. - d.shutdownWait.Add(1) - defer d.shutdownWait.Done() + d.rpcRW.RLock() + defer d.rpcRW.RUnlock() dctx, err := d.isRunningLocked() if err != nil { @@ -1140,20 +1132,13 @@ func (d *Dispatcher) markNodeNotReady(id string, state api.NodeStatus_State, mes // Node should send new heartbeat earlier than now + TTL, otherwise it will // be deregistered from dispatcher and its status will be updated to NodeStatus_DOWN func (d *Dispatcher) Heartbeat(ctx context.Context, r *api.HeartbeatRequest) (*api.HeartbeatResponse, error) { - // shutdownWait.Add() followed by isRunning() to ensures that - // if this rpc sees the dispatcher running, - // it will already have called Add() on the shutdownWait wait, - // which ensures that Stop() will wait for this rpc to complete. - // Note that Stop() first does Dispatcher.ctx.cancel() followed by - // shutdownWait.Wait() to make sure new rpc's don't start before waiting - // for existing ones to finish. - d.shutdownWait.Add(1) - defer d.shutdownWait.Done() + d.rpcRW.RLock() + defer d.rpcRW.RUnlock() - // isRunningLocked() is not needed since its OK if - // the dispatcher context is cancelled while this call is in progress - // since Stop() which cancels the dispatcher context will wait for - // Heartbeat() to complete. + // Its OK to call isRunning() here instead of isRunningLocked() + // because of the rpcRW readlock above. + // TODO(anshul) other uses of isRunningLocked() can probably + // also be removed. if !d.isRunning() { return nil, status.Errorf(codes.Aborted, "dispatcher is stopped") } @@ -1192,15 +1177,8 @@ func (d *Dispatcher) getRootCACert() []byte { // a special boolean field Disconnect which if true indicates that node should // reconnect to another Manager immediately. func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_SessionServer) error { - // shutdownWait.Add() followed by isRunning() to ensures that - // if this rpc sees the dispatcher running, - // it will already have called Add() on the shutdownWait wait, - // which ensures that Stop() will wait for this rpc to complete. - // Note that Stop() first does Dispatcher.ctx.cancel() followed by - // shutdownWait.Wait() to make sure new rpc's don't start before waiting - // for existing ones to finish. - d.shutdownWait.Add(1) - defer d.shutdownWait.Done() + d.rpcRW.RLock() + defer d.rpcRW.RUnlock() dctx, err := d.isRunningLocked() if err != nil { @@ -1208,6 +1186,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio } ctx := stream.Context() + nodeInfo, err := ca.RemoteNode(ctx) if err != nil { return err From 4811cada93c33ae38340bd2f69a32813ae8fea3e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Feb 2018 17:10:18 -0500 Subject: [PATCH 4/7] Move ImagePrune Signed-off-by: Daniel Nephin Upstream-commit: 9c25df0fa201279f78d555c91ae000cc4f9b3036 Component: engine --- components/engine/daemon/image_prune.go | 168 ++++++++++++++++++++++++ components/engine/daemon/prune.go | 158 +--------------------- 2 files changed, 169 insertions(+), 157 deletions(-) create mode 100644 components/engine/daemon/image_prune.go diff --git a/components/engine/daemon/image_prune.go b/components/engine/daemon/image_prune.go new file mode 100644 index 0000000000..8a1c2cd225 --- /dev/null +++ b/components/engine/daemon/image_prune.go @@ -0,0 +1,168 @@ +package daemon + +import ( + "sync/atomic" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/errdefs" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" +) + +var imagesAcceptedFilters = map[string]bool{ + "dangling": true, + "label": true, + "label!": true, + "until": true, +} + +// ImagesPrune removes unused images +func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { + if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) { + return nil, errPruneRunning + } + defer atomic.StoreInt32(&daemon.pruneRunning, 0) + + // make sure that only accepted filters have been received + err := pruneFilters.Validate(imagesAcceptedFilters) + if err != nil { + return nil, err + } + + rep := &types.ImagesPruneReport{} + + danglingOnly := true + if pruneFilters.Contains("dangling") { + if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") { + danglingOnly = false + } else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") { + return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")} + } + } + + until, err := getUntilFromPruneFilters(pruneFilters) + if err != nil { + return nil, err + } + + var allImages map[image.ID]*image.Image + if danglingOnly { + allImages = daemon.imageStore.Heads() + } else { + allImages = daemon.imageStore.Map() + } + + // Filter intermediary images and get their unique size + allLayers := make(map[layer.ChainID]layer.Layer) + for _, ls := range daemon.layerStores { + for k, v := range ls.Map() { + allLayers[k] = v + } + } + topImages := map[image.ID]*image.Image{} + for id, img := range allImages { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + dgst := digest.Digest(id) + if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { + continue + } + if !until.IsZero() && img.Created.After(until) { + continue + } + if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) { + continue + } + topImages[id] = img + } + } + + canceled := false +deleteImagesLoop: + for id := range topImages { + select { + case <-ctx.Done(): + // we still want to calculate freed size and return the data + canceled = true + break deleteImagesLoop + default: + } + + deletedImages := []types.ImageDeleteResponseItem{} + refs := daemon.referenceStore.References(id.Digest()) + if len(refs) > 0 { + shouldDelete := !danglingOnly + if !shouldDelete { + hasTag := false + for _, ref := range refs { + if _, ok := ref.(reference.NamedTagged); ok { + hasTag = true + break + } + } + + // Only delete if it's untagged (i.e. repo:) + shouldDelete = !hasTag + } + + if shouldDelete { + for _, ref := range refs { + imgDel, err := daemon.ImageDelete(ref.String(), false, true) + if imageDeleteFailed(ref.String(), err) { + continue + } + deletedImages = append(deletedImages, imgDel...) + } + } + } else { + hex := id.Digest().Hex() + imgDel, err := daemon.ImageDelete(hex, false, true) + if imageDeleteFailed(hex, err) { + continue + } + deletedImages = append(deletedImages, imgDel...) + } + + rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...) + } + + // Compute how much space was freed + for _, d := range rep.ImagesDeleted { + if d.Deleted != "" { + chid := layer.ChainID(d.Deleted) + if l, ok := allLayers[chid]; ok { + diffSize, err := l.DiffSize() + if err != nil { + logrus.Warnf("failed to get layer %s size: %v", chid, err) + continue + } + rep.SpaceReclaimed += uint64(diffSize) + } + } + } + + if canceled { + logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep) + } + + return rep, nil +} + +func imageDeleteFailed(ref string, err error) bool { + switch { + case err == nil: + return false + case errdefs.IsConflict(err): + return true + default: + logrus.Warnf("failed to prune image %s: %v", ref, err) + return true + } +} diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index eee9e106bd..f16e448ba8 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -6,18 +6,13 @@ import ( "sync/atomic" "time" - "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" timetypes "github.com/docker/docker/api/types/time" - "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/runconfig" "github.com/docker/docker/volume" "github.com/docker/libnetwork" - digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -36,12 +31,7 @@ var ( "label": true, "label!": true, } - imagesAcceptedFilters = map[string]bool{ - "dangling": true, - "label": true, - "label!": true, - "until": true, - } + networksAcceptedFilters = map[string]bool{ "label": true, "label!": true, @@ -159,152 +149,6 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg return rep, err } -// ImagesPrune removes unused images -func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { - if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) { - return nil, errPruneRunning - } - defer atomic.StoreInt32(&daemon.pruneRunning, 0) - - // make sure that only accepted filters have been received - err := pruneFilters.Validate(imagesAcceptedFilters) - if err != nil { - return nil, err - } - - rep := &types.ImagesPruneReport{} - - danglingOnly := true - if pruneFilters.Contains("dangling") { - if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") { - danglingOnly = false - } else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") { - return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")} - } - } - - until, err := getUntilFromPruneFilters(pruneFilters) - if err != nil { - return nil, err - } - - var allImages map[image.ID]*image.Image - if danglingOnly { - allImages = daemon.imageStore.Heads() - } else { - allImages = daemon.imageStore.Map() - } - - // Filter intermediary images and get their unique size - allLayers := make(map[layer.ChainID]layer.Layer) - for _, ls := range daemon.layerStores { - for k, v := range ls.Map() { - allLayers[k] = v - } - } - topImages := map[image.ID]*image.Image{} - for id, img := range allImages { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - dgst := digest.Digest(id) - if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { - continue - } - if !until.IsZero() && img.Created.After(until) { - continue - } - if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) { - continue - } - topImages[id] = img - } - } - - canceled := false -deleteImagesLoop: - for id := range topImages { - select { - case <-ctx.Done(): - // we still want to calculate freed size and return the data - canceled = true - break deleteImagesLoop - default: - } - - deletedImages := []types.ImageDeleteResponseItem{} - refs := daemon.referenceStore.References(id.Digest()) - if len(refs) > 0 { - shouldDelete := !danglingOnly - if !shouldDelete { - hasTag := false - for _, ref := range refs { - if _, ok := ref.(reference.NamedTagged); ok { - hasTag = true - break - } - } - - // Only delete if it's untagged (i.e. repo:) - shouldDelete = !hasTag - } - - if shouldDelete { - for _, ref := range refs { - imgDel, err := daemon.ImageDelete(ref.String(), false, true) - if imageDeleteFailed(ref.String(), err) { - continue - } - deletedImages = append(deletedImages, imgDel...) - } - } - } else { - hex := id.Digest().Hex() - imgDel, err := daemon.ImageDelete(hex, false, true) - if imageDeleteFailed(hex, err) { - continue - } - deletedImages = append(deletedImages, imgDel...) - } - - rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...) - } - - // Compute how much space was freed - for _, d := range rep.ImagesDeleted { - if d.Deleted != "" { - chid := layer.ChainID(d.Deleted) - if l, ok := allLayers[chid]; ok { - diffSize, err := l.DiffSize() - if err != nil { - logrus.Warnf("failed to get layer %s size: %v", chid, err) - continue - } - rep.SpaceReclaimed += uint64(diffSize) - } - } - } - - if canceled { - logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep) - } - - return rep, nil -} - -func imageDeleteFailed(ref string, err error) bool { - switch { - case err == nil: - return false - case errdefs.IsConflict(err): - return true - default: - logrus.Warnf("failed to prune image %s: %v", ref, err) - return true - } -} - // localNetworksPrune removes unused local networks func (daemon *Daemon) localNetworksPrune(ctx context.Context, pruneFilters filters.Args) *types.NetworksPruneReport { rep := &types.NetworksPruneReport{} From 192d8829e00a3e36ccdb2bc5de3856ff3abf23ea Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Feb 2018 17:16:57 -0500 Subject: [PATCH 5/7] GetLayerFolders Signed-off-by: Daniel Nephin Upstream-commit: f6639cb46d83cb9bb92a30578c10fdb372a58f04 Component: engine --- components/engine/daemon/image_prune.go | 2 +- components/engine/daemon/image_windows.go | 35 +++++++++++++++++++++++ components/engine/daemon/oci_windows.go | 26 ++--------------- 3 files changed, 39 insertions(+), 24 deletions(-) create mode 100644 components/engine/daemon/image_windows.go diff --git a/components/engine/daemon/image_prune.go b/components/engine/daemon/image_prune.go index 8a1c2cd225..193c3aa0ac 100644 --- a/components/engine/daemon/image_prune.go +++ b/components/engine/daemon/image_prune.go @@ -1,4 +1,4 @@ -package daemon +package daemon // import "github.com/docker/docker/daemon" import ( "sync/atomic" diff --git a/components/engine/daemon/image_windows.go b/components/engine/daemon/image_windows.go new file mode 100644 index 0000000000..d661400a38 --- /dev/null +++ b/components/engine/daemon/image_windows.go @@ -0,0 +1,35 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" +) + +// GetLayerFolders returns the layer folders from an image RootFS +func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { + folders := []string{} + max := len(img.RootFS.DiffIDs) + for index := 1; index <= max; index++ { + // FIXME: why does this mutate the RootFS? + img.RootFS.DiffIDs = img.RootFS.DiffIDs[:index] + if !system.IsOSSupported(img.OperatingSystem()) { + return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID()) + } + layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + if err != nil { + return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + } + // Reverse order, expecting parent first + folders = append([]string{layerPath}, folders...) + } + if rwLayer == nil { + return nil, errors.New("RWLayer is unexpectedly nil") + } + m, err := rwLayer.Metadata() + if err != nil { + return nil, errors.Wrap(err, "failed to get layer metadata") + } + return append(folders, m["dir"]), nil +} diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index 64c651c4af..e4b536f4e4 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -1,7 +1,6 @@ package daemon // import "github.com/docker/docker/daemon" import ( - "errors" "fmt" "io/ioutil" "path/filepath" @@ -10,11 +9,11 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/layer" "github.com/docker/docker/oci" "github.com/docker/docker/pkg/sysinfo" "github.com/docker/docker/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" ) @@ -139,29 +138,10 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } } s.Process.User.Username = c.Config.User - - // Get the layer path for each layer. - max := len(img.RootFS.DiffIDs) - for i := 1; i <= max; i++ { - img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i] - if !system.IsOSSupported(img.OperatingSystem()) { - return nil, fmt.Errorf("cannot get layerpath for ImageID %s: %s ", img.RootFS.ChainID(), system.ErrNotSupportedOperatingSystem) - } - layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) - if err != nil { - return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID(), err) - } - // Reverse order, expecting parent most first - s.Windows.LayerFolders = append([]string{layerPath}, s.Windows.LayerFolders...) - } - if c.RWLayer == nil { - return nil, errors.New("RWLayer of container " + c.ID + " is unexpectedly nil") - } - m, err := c.RWLayer.Metadata() + s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer) if err != nil { - return nil, fmt.Errorf("failed to get layer metadata - %s", err) + return nil, errors.Wrapf(err, "container %s", c.ID) } - s.Windows.LayerFolders = append(s.Windows.LayerFolders, m["dir"]) dnsSearch := daemon.getDNSSearchSettings(c) From 718daa9c98854822d1566c503d91f89aff1d603f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Feb 2018 17:18:28 -0500 Subject: [PATCH 6/7] Image events Signed-off-by: Daniel Nephin Upstream-commit: 0ac4ad0580c48749fb7d07f60a77a0014b1c3a99 Component: engine --- components/engine/daemon/events.go | 24 -------------------- components/engine/daemon/image_events.go | 29 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 components/engine/daemon/image_events.go diff --git a/components/engine/daemon/events.go b/components/engine/daemon/events.go index 2e594c0269..cf1634a198 100644 --- a/components/engine/daemon/events.go +++ b/components/engine/daemon/events.go @@ -44,30 +44,6 @@ func (daemon *Daemon) LogContainerEventWithAttributes(container *container.Conta daemon.EventsService.Log(action, events.ContainerEventType, actor) } -// LogImageEvent generates an event related to an image with only the default attributes. -func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { - daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := daemon.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - daemon.EventsService.Log(action, events.ImageEventType, actor) -} - // LogPluginEvent generates an event related to a plugin with only the default attributes. func (daemon *Daemon) LogPluginEvent(pluginID, refName, action string) { daemon.LogPluginEventWithAttributes(pluginID, refName, action, map[string]string{}) diff --git a/components/engine/daemon/image_events.go b/components/engine/daemon/image_events.go new file mode 100644 index 0000000000..0d364f7b58 --- /dev/null +++ b/components/engine/daemon/image_events.go @@ -0,0 +1,29 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "github.com/docker/docker/api/types/events" +) + +// LogImageEvent generates an event related to an image with only the default attributes. +func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { + daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) +} + +// LogImageEventWithAttributes generates an event related to an image with specific given attributes. +func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { + img, err := daemon.GetImage(imageID) + if err == nil && img.Config != nil { + // image has not been removed yet. + // it could be missing if the event is `delete`. + copyAttributes(attributes, img.Config.Labels) + } + if refName != "" { + attributes["name"] = refName + } + actor := events.Actor{ + ID: imageID, + Attributes: attributes, + } + + daemon.EventsService.Log(action, events.ImageEventType, actor) +} From 112db723e1ab59af671bb69565332ccb85515271 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Feb 2018 17:20:58 -0500 Subject: [PATCH 7/7] Image commit Signed-off-by: Daniel Nephin Upstream-commit: bad33bbd02b9f2754d3a8c64648d46baba509680 Component: engine --- components/engine/daemon/commit.go | 118 --------------------- components/engine/daemon/image_commit.go | 124 +++++++++++++++++++++++ 2 files changed, 124 insertions(+), 118 deletions(-) create mode 100644 components/engine/daemon/image_commit.go diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 8865d832f3..6af494da74 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -1,9 +1,7 @@ package daemon // import "github.com/docker/docker/daemon" import ( - "encoding/json" "fmt" - "io" "runtime" "strings" "time" @@ -12,10 +10,6 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder/dockerfile" "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/system" "github.com/pkg/errors" ) @@ -190,115 +184,3 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma containerActions.WithValues("commit").UpdateSince(start) return id.String(), nil } - -func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { - layerStore, ok := daemon.layerStores[c.ContainerOS] - if !ok { - return "", system.ErrNotSupportedOperatingSystem - } - rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel) - if err != nil { - return "", err - } - defer func() { - if rwTar != nil { - rwTar.Close() - } - }() - - var parent *image.Image - if c.ParentImageID == "" { - parent = new(image.Image) - parent.RootFS = image.NewRootFS() - } else { - parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID)) - if err != nil { - return "", err - } - } - - l, err := layerStore.Register(rwTar, parent.RootFS.ChainID()) - if err != nil { - return "", err - } - defer layer.ReleaseAndLog(layerStore, l) - - cc := image.ChildConfig{ - ContainerID: c.ContainerID, - Author: c.Author, - Comment: c.Comment, - ContainerConfig: c.ContainerConfig, - Config: c.Config, - DiffID: l.DiffID(), - } - config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS)) - if err != nil { - return "", err - } - - id, err := daemon.imageStore.Create(config) - if err != nil { - return "", err - } - - if c.ParentImageID != "" { - if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { - return "", err - } - } - return id, nil -} - -func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) { - rwlayer, err := layerStore.GetRWLayer(id) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - layerStore.ReleaseRWLayer(rwlayer) - } - }() - - // TODO: this mount call is not necessary as we assume that TarStream() should - // mount the layer if needed. But the Diff() function for windows requests that - // the layer should be mounted when calling it. So we reserve this mount call - // until windows driver can implement Diff() interface correctly. - _, err = rwlayer.Mount(mountLabel) - if err != nil { - return nil, err - } - - archive, err := rwlayer.TarStream() - if err != nil { - rwlayer.Unmount() - return nil, err - } - return ioutils.NewReadCloserWrapper(archive, func() error { - archive.Close() - err = rwlayer.Unmount() - layerStore.ReleaseRWLayer(rwlayer) - return err - }), - nil -} - -// CommitBuildStep is used by the builder to create an image for each step in -// the build. -// -// This method is different from CreateImageFromContainer: -// * it doesn't attempt to validate container state -// * it doesn't send a commit action to metrics -// * it doesn't log a container commit event -// -// This is a temporary shim. Should be removed when builder stops using commit. -func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { - container, err := daemon.GetContainer(c.ContainerID) - if err != nil { - return "", err - } - c.ContainerMountLabel = container.MountLabel - c.ContainerOS = container.OS - c.ParentImageID = string(container.ImageID) - return daemon.commitImage(c) -} diff --git a/components/engine/daemon/image_commit.go b/components/engine/daemon/image_commit.go new file mode 100644 index 0000000000..48f890b80d --- /dev/null +++ b/components/engine/daemon/image_commit.go @@ -0,0 +1,124 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "encoding/json" + "io" + + "github.com/docker/docker/api/types/backend" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/system" +) + +func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { + layerStore, ok := daemon.layerStores[c.ContainerOS] + if !ok { + return "", system.ErrNotSupportedOperatingSystem + } + rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel) + if err != nil { + return "", err + } + defer func() { + if rwTar != nil { + rwTar.Close() + } + }() + + var parent *image.Image + if c.ParentImageID == "" { + parent = new(image.Image) + parent.RootFS = image.NewRootFS() + } else { + parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID)) + if err != nil { + return "", err + } + } + + l, err := layerStore.Register(rwTar, parent.RootFS.ChainID()) + if err != nil { + return "", err + } + defer layer.ReleaseAndLog(layerStore, l) + + cc := image.ChildConfig{ + ContainerID: c.ContainerID, + Author: c.Author, + Comment: c.Comment, + ContainerConfig: c.ContainerConfig, + Config: c.Config, + DiffID: l.DiffID(), + } + config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS)) + if err != nil { + return "", err + } + + id, err := daemon.imageStore.Create(config) + if err != nil { + return "", err + } + + if c.ParentImageID != "" { + if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { + return "", err + } + } + return id, nil +} + +func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) { + rwlayer, err := layerStore.GetRWLayer(id) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + layerStore.ReleaseRWLayer(rwlayer) + } + }() + + // TODO: this mount call is not necessary as we assume that TarStream() should + // mount the layer if needed. But the Diff() function for windows requests that + // the layer should be mounted when calling it. So we reserve this mount call + // until windows driver can implement Diff() interface correctly. + _, err = rwlayer.Mount(mountLabel) + if err != nil { + return nil, err + } + + archive, err := rwlayer.TarStream() + if err != nil { + rwlayer.Unmount() + return nil, err + } + return ioutils.NewReadCloserWrapper(archive, func() error { + archive.Close() + err = rwlayer.Unmount() + layerStore.ReleaseRWLayer(rwlayer) + return err + }), + nil +} + +// CommitBuildStep is used by the builder to create an image for each step in +// the build. +// +// This method is different from CreateImageFromContainer: +// * it doesn't attempt to validate container state +// * it doesn't send a commit action to metrics +// * it doesn't log a container commit event +// +// This is a temporary shim. Should be removed when builder stops using commit. +func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { + container, err := daemon.GetContainer(c.ContainerID) + if err != nil { + return "", err + } + c.ContainerMountLabel = container.MountLabel + c.ContainerOS = container.OS + c.ParentImageID = string(container.ImageID) + return daemon.commitImage(c) +}