diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index fab79570c4..20fac46a33 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -32,86 +32,35 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM debian:stretch - +FROM golang:1.9.4 AS base +# FIXME(vdemeester) this is kept for other script depending on it to not fail right away +# Remove this once the other scripts uses something else to detect the version +ENV GO_VERSION 1.9.4 # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list -# Packaged dependencies -RUN apt-get update && apt-get install -y \ - apparmor \ - apt-utils \ - aufs-tools \ - automake \ - bash-completion \ - binutils-mingw-w64 \ - bsdmainutils \ - btrfs-tools \ - build-essential \ - cmake \ - createrepo \ - curl \ - dpkg-sig \ - gcc-mingw-w64 \ - git \ - iptables \ - jq \ - less \ - libapparmor-dev \ - libcap-dev \ - libdevmapper-dev \ - libnet-dev \ - libnl-3-dev \ - libprotobuf-c0-dev \ - libprotobuf-dev \ - libseccomp-dev \ - libsystemd-dev \ - libtool \ - libudev-dev \ - mercurial \ - net-tools \ - pigz \ - pkg-config \ - protobuf-compiler \ - protobuf-c-compiler \ - python-backports.ssl-match-hostname \ - python-dev \ - python-mock \ - python-pip \ - python-requests \ - python-setuptools \ - python-websocket \ - python-wheel \ - tar \ - thin-provisioning-tools \ - vim \ - vim-common \ - xfsprogs \ - zip \ - --no-install-recommends \ - && pip install awscli==1.10.15 - -# Install Go -# IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines -# will need updating, to avoid errors. Ping #docker-maintainers on IRC -# with a heads-up. -# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored -ENV GO_VERSION 1.9.4 -RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ - | tar -xzC /usr/local - -ENV PATH /go/bin:/usr/local/go/bin:$PATH -ENV GOPATH /go - +FROM base AS criu # Install CRIU for checkpoint/restore support ENV CRIU_VERSION 3.6 -RUN mkdir -p /usr/src/criu \ +# Install dependancy packages specific to criu +RUN apt-get update && apt-get install -y \ + libnet-dev \ + libprotobuf-c0-dev \ + libprotobuf-dev \ + libnl-3-dev \ + libcap-dev \ + protobuf-compiler \ + protobuf-c-compiler \ + python-protobuf \ + && mkdir -p /usr/src/criu \ && curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \ && cd /usr/src/criu \ && make \ - && make install-criu + && make PREFIX=/opt/criu install-criu + +FROM base AS registry # Install two versions of the registry. The first is an older version that # only supports schema1 manifests. The second is a newer version that supports # both. This allows integration-cli tests to cover push/pull with both schema1 @@ -129,6 +78,9 @@ RUN set -x \ go build -buildmode=pie -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry \ && rm -rf "$GOPATH" + + +FROM base AS notary # Install notary and notary-server ENV NOTARY_VERSION v0.5.0 RUN set -x \ @@ -141,18 +93,18 @@ RUN set -x \ go build -buildmode=pie -o /usr/local/bin/notary github.com/docker/notary/cmd/notary \ && rm -rf "$GOPATH" + + +FROM base AS docker-py # Get the "docker-py" source so we can run their integration tests ENV DOCKER_PY_COMMIT 8b246db271a85d6541dc458838627e89c683e42f -# To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ - && git checkout -q $DOCKER_PY_COMMIT \ - && pip install docker-pycreds==0.2.1 \ - && pip install -r test-requirements.txt + && git checkout -q $DOCKER_PY_COMMIT -# Install yamllint for validating swagger.yaml -RUN pip install yamllint==1.5.0 + +FROM base AS swagger # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb RUN set -x \ @@ -162,49 +114,147 @@ RUN set -x \ && go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \ && rm -rf "$GOPATH" -# Set user.email so crosbymichael's in-container merge commits go smoothly -RUN git config --global user.email 'docker-dummy@example.com' - -# Add an unprivileged user to be used for tests which need it -RUN groupadd -r docker -RUN useradd --create-home --gid docker unprivilegeduser - -VOLUME /var/lib/docker -WORKDIR /go/src/github.com/docker/docker -ENV DOCKER_BUILDTAGS apparmor seccomp selinux - -# Let us use a .bashrc file -RUN ln -sfv $PWD/.bashrc ~/.bashrc -# Add integration helps to bashrc -RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc +FROM base AS frozen-images +RUN apt-get update && apt-get install -y jq ca-certificates --no-install-recommends # Get useful and necessary Hub images so we can "docker load" locally instead of pulling -COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ -RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ +COPY contrib/download-frozen-image-v2.sh / +RUN /download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) -# Install tomlv, vndr, runc, containerd, tini, proxy dockercli -# Please edit hack/dockerfile/install/.installer to update them. -COPY hack/dockerfile/install hack/dockerfile/install -RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ - do hack/dockerfile/install/install.sh $i; \ - done -ENV PATH=/usr/local/cli:$PATH +# Just a little hack so we don't have to install these deps twice, once for runc and once for dockerd +FROM base AS runtime-dev +RUN apt-get update && apt-get install -y \ + libapparmor-dev \ + libseccomp-dev + +FROM base AS tomlv +ENV INSTALL_BINARY_NAME=tomlv +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS vndr +ENV INSTALL_BINARY_NAME=vndr +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS containerd +RUN apt-get update && apt-get install -y btrfs-tools +ENV INSTALL_BINARY_NAME=containerd +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS proxy +ENV INSTALL_BINARY_NAME=proxy +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS gometalinter +ENV INSTALL_BINARY_NAME=gometalinter +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS dockercli +ENV INSTALL_BINARY_NAME=dockercli +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM runtime-dev AS runc +ENV INSTALL_BINARY_NAME=runc +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS tini +RUN apt-get update && apt-get install -y cmake vim-common +COPY hack/dockerfile/install/install.sh ./install.sh +ENV INSTALL_BINARY_NAME=tini +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + + + +# TODO: Some of this is only really needed for testing, it would be nice to split this up +FROM runtime-dev AS dev +RUN groupadd -r docker +RUN useradd --create-home --gid docker unprivilegeduser # Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH RUN echo "source /usr/share/bash-completion/bash_completion" >> /etc/bash.bashrc RUN ln -s /usr/local/completion/bash/docker /etc/bash_completion.d/docker +RUN ldconfig +# This should only install packages that are specifically needed for the dev environment and nothing else +# Do you really need to add another package here? Can it be done in a different build stage? +RUN apt-get update && apt-get install -y \ + apparmor \ + aufs-tools \ + bash-completion \ + btrfs-tools \ + iptables \ + jq \ + libdevmapper-dev \ + libudev-dev \ + libsystemd-dev \ + binutils-mingw-w64 \ + g++-mingw-w64-x86-64 \ + net-tools \ + pigz \ + python-backports.ssl-match-hostname \ + python-dev \ + python-mock \ + python-pip \ + python-requests \ + python-setuptools \ + python-websocket \ + python-wheel \ + thin-provisioning-tools \ + vim \ + vim-common \ + xfsprogs \ + zip \ + bzip2 \ + xz-utils \ + --no-install-recommends +COPY --from=swagger /usr/local/bin/swagger* /usr/local/bin/ +COPY --from=frozen-images /docker-frozen-images /docker-frozen-images +COPY --from=gometalinter /opt/gometalinter/ /usr/local/bin/ +COPY --from=tomlv /opt/tomlv/ /usr/local/bin/ +COPY --from=vndr /opt/vndr/ /usr/local/bin/ +COPY --from=tini /opt/tini/ /usr/local/bin/ +COPY --from=runc /opt/runc/ /usr/local/bin/ +COPY --from=containerd /opt/containerd/ /usr/local/bin/ +COPY --from=proxy /opt/proxy/ /usr/local/bin/ +COPY --from=dockercli /opt/dockercli /usr/local/cli +COPY --from=registry /usr/local/bin/registry* /usr/local/bin/ +COPY --from=notary /usr/local/bin/notary* /usr/local/bin/ +COPY --from=criu /opt/criu/ /usr/local/ +COPY --from=docker-py /docker-py /docker-py +# TODO: This is for the docker-py tests, which shouldn't really be needed for +# this image, but currently CI is expecting to run this image. This should be +# split out into a separate image, including all the `python-*` deps installed +# above. +RUN cd /docker-py \ + && pip install docker-pycreds==0.2.1 \ + && pip install -r test-requirements.txt -# Wrap all commands in the "docker-in-docker" script to allow nested containers -ENTRYPOINT ["hack/dind"] - +ENV PATH=/usr/local/cli:$PATH +ENV DOCKER_BUILDTAGS apparmor seccomp selinux # Options for hack/validate/gometalinter ENV GOMETALINTER_OPTS="--deadline=2m" - +WORKDIR /go/src/github.com/docker/docker +VOLUME /var/lib/docker +# Wrap all commands in the "docker-in-docker" script to allow nested containers +ENTRYPOINT ["hack/dind"] # Upload docker source COPY . /go/src/github.com/docker/docker - diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index 068d80516f..394f593c2e 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -147,7 +147,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 0e03fb0a1a..58d52aadce 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -133,7 +133,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index ce13170e32..bd700c98e2 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -17,7 +17,8 @@ WORKDIR /go/src/github.com/docker/docker/ COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 07f4b01492..5af94acf5b 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -131,7 +131,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index dc55d57231..9d772dd073 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -125,7 +125,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/builder/builder.go b/components/engine/builder/builder.go index 95320c9a6c..6da0e49e10 100644 --- a/components/engine/builder/builder.go +++ b/components/engine/builder/builder.go @@ -53,7 +53,7 @@ type Backend interface { // ImageBackend are the interface methods required from an image component type ImageBackend interface { - GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ReleaseableLayer, error) + GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ROLayer, error) } // ExecBackend contains the interface methods required for executing containers @@ -100,10 +100,16 @@ type Image interface { OperatingSystem() string } -// ReleaseableLayer is an image layer that can be mounted and released -type ReleaseableLayer interface { +// ROLayer is a reference to image rootfs layer +type ROLayer interface { Release() error - Mount() (containerfs.ContainerFS, error) - Commit() (ReleaseableLayer, error) + NewRWLayer() (RWLayer, error) DiffID() layer.DiffID } + +// RWLayer is active layer that can be read/modified +type RWLayer interface { + Release() error + Root() containerfs.ContainerFS + Commit() (ROLayer, error) +} diff --git a/components/engine/builder/dockerfile/copy.go b/components/engine/builder/dockerfile/copy.go index f0b27b4c68..cb9f24d205 100644 --- a/components/engine/builder/dockerfile/copy.go +++ b/components/engine/builder/dockerfile/copy.go @@ -72,8 +72,12 @@ type copier struct { source builder.Source pathCache pathCache download sourceDownloader - tmpPaths []string platform string + // for cleanup. TODO: having copier.cleanup() is error prone and hard to + // follow. Code calling performCopy should manage the lifecycle of its params. + // Copier should take override source as input, not imageMount. + activeLayer builder.RWLayer + tmpPaths []string } func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier { @@ -155,6 +159,10 @@ func (o *copier) Cleanup() { os.RemoveAll(path) } o.tmpPaths = []string{} + if o.activeLayer != nil { + o.activeLayer.Release() + o.activeLayer = nil + } } // TODO: allowWildcards can probably be removed by refactoring this function further. @@ -166,9 +174,15 @@ func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, // done on image Source? if imageSource != nil { var err error - o.source, err = imageSource.Source() + rwLayer, err := imageSource.NewRWLayer() if err != nil { - return nil, errors.Wrapf(err, "failed to copy from %s", imageSource.ImageID()) + return nil, err + } + o.activeLayer = rwLayer + + o.source, err = remotecontext.NewLazySource(rwLayer.Root()) + if err != nil { + return nil, errors.Wrapf(err, "failed to create context for copy from %s", rwLayer.Root().Path()) } } diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index a96d579905..6d52e7e619 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -127,7 +127,7 @@ func TestFromScratch(t *testing.T) { func TestFromWithArg(t *testing.T) { tag, expected := ":sometag", "expectedthisid" - getImage := func(name string) (builder.Image, builder.ReleaseableLayer, error) { + getImage := func(name string) (builder.Image, builder.ROLayer, error) { assert.Equal(t, "alpine"+tag, name) return &mockImage{id: "expectedthisid"}, nil, nil } @@ -159,7 +159,7 @@ func TestFromWithArg(t *testing.T) { func TestFromWithUndefinedArg(t *testing.T) { tag, expected := "sometag", "expectedthisid" - getImage := func(name string) (builder.Image, builder.ReleaseableLayer, error) { + getImage := func(name string) (builder.Image, builder.ROLayer, error) { assert.Equal(t, "alpine", name) return &mockImage{id: "expectedthisid"}, nil, nil } @@ -433,7 +433,7 @@ func TestRunWithBuildArgs(t *testing.T) { return imageCache } b.imageProber = newImageProber(mockBackend, nil, false) - mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) { + mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ROLayer, error) { return &mockImage{ id: "abcdef", config: &container.Config{Cmd: origCmd}, diff --git a/components/engine/builder/dockerfile/imagecontext.go b/components/engine/builder/dockerfile/imagecontext.go index f83528e104..0d4af384af 100644 --- a/components/engine/builder/dockerfile/imagecontext.go +++ b/components/engine/builder/dockerfile/imagecontext.go @@ -5,7 +5,6 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/remotecontext" dockerimage "github.com/docker/docker/image" "github.com/docker/docker/pkg/system" "github.com/pkg/errors" @@ -13,7 +12,7 @@ import ( "golang.org/x/net/context" ) -type getAndMountFunc func(string, bool) (builder.Image, builder.ReleaseableLayer, error) +type getAndMountFunc func(string, bool) (builder.Image, builder.ROLayer, error) // imageSources mounts images and provides a cache for mounted images. It tracks // all images so they can be unmounted at the end of the build. @@ -24,7 +23,7 @@ type imageSources struct { } func newImageSources(ctx context.Context, options builderOptions) *imageSources { - getAndMount := func(idOrRef string, localOnly bool) (builder.Image, builder.ReleaseableLayer, error) { + getAndMount := func(idOrRef string, localOnly bool) (builder.Image, builder.ROLayer, error) { pullOption := backend.PullOptionNoPull if !localOnly { if options.Options.PullParent { @@ -92,32 +91,14 @@ func (m *imageSources) Add(im *imageMount) { type imageMount struct { image builder.Image source builder.Source - layer builder.ReleaseableLayer + layer builder.ROLayer } -func newImageMount(image builder.Image, layer builder.ReleaseableLayer) *imageMount { +func newImageMount(image builder.Image, layer builder.ROLayer) *imageMount { im := &imageMount{image: image, layer: layer} return im } -func (im *imageMount) Source() (builder.Source, error) { - if im.source == nil { - if im.layer == nil { - return nil, errors.Errorf("empty context") - } - mountPath, err := im.layer.Mount() - if err != nil { - return nil, errors.Wrapf(err, "failed to mount %s", im.image.ImageID()) - } - source, err := remotecontext.NewLazySource(mountPath) - if err != nil { - return nil, errors.Wrapf(err, "failed to create lazycontext for %s", mountPath) - } - im.source = source - } - return im.source, nil -} - func (im *imageMount) unmount() error { if im.layer == nil { return nil @@ -133,8 +114,8 @@ func (im *imageMount) Image() builder.Image { return im.image } -func (im *imageMount) Layer() builder.ReleaseableLayer { - return im.layer +func (im *imageMount) NewRWLayer() (builder.RWLayer, error) { + return im.layer.NewRWLayer() } func (im *imageMount) ImageID() string { diff --git a/components/engine/builder/dockerfile/instructions/commands.go b/components/engine/builder/dockerfile/instructions/commands.go index ef435b27b3..d4f55ceb43 100644 --- a/components/engine/builder/dockerfile/instructions/commands.go +++ b/components/engine/builder/dockerfile/instructions/commands.go @@ -9,7 +9,7 @@ import ( "github.com/docker/docker/api/types/strslice" ) -// KeyValuePair represent an arbitrary named value (useful in slice insted of map[string] string to preserve ordering) +// KeyValuePair represent an arbitrary named value (useful in slice instead of map[string] string to preserve ordering) type KeyValuePair struct { Key string Value string @@ -195,7 +195,7 @@ func (c *WorkdirCommand) Expand(expander SingleWordExpander) error { return nil } -// ShellDependantCmdLine represents a cmdline optionaly prepended with the shell +// ShellDependantCmdLine represents a cmdline optionally prepended with the shell type ShellDependantCmdLine struct { CmdLine strslice.StrSlice PrependShell bool diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index 4b4f638b1f..c8b34f8f65 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -17,6 +17,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/builder" "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" @@ -114,8 +115,8 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta return err } -func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error { - newLayer, err := imageMount.Layer().Commit() +func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error { + newLayer, err := layer.Commit() if err != nil { return err } @@ -124,7 +125,7 @@ func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runC // if there is an error before we can add the full mount with image b.imageSources.Add(newImageMount(nil, newLayer)) - parentImage, ok := imageMount.Image().(*image.Image) + parentImage, ok := parent.(*image.Image) if !ok { return errors.Errorf("unexpected image type") } @@ -177,7 +178,13 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error return errors.Wrapf(err, "failed to get destination image %q", state.imageID) } - destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.options.Platform) + rwLayer, err := imageMount.NewRWLayer() + if err != nil { + return err + } + defer rwLayer.Release() + + destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, b.options.Platform) if err != nil { return err } @@ -203,10 +210,10 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error return errors.Wrapf(err, "failed to copy files") } } - return b.exportImage(state, imageMount, runConfigWithCommentCmd) + return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) } -func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) { +func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) { // Twiddle the destination when it's a relative path - meaning, make it // relative to the WORKINGDIR dest, err := normalizeDest(workingDir, inst.dest, platform) @@ -214,12 +221,7 @@ func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMo return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName) } - destMount, err := imageMount.Source() - if err != nil { - return copyInfo{}, errors.Wrapf(err, "failed to mount copy source") - } - - return newCopyInfoFromSource(destMount, dest, ""), nil + return copyInfo{root: rwLayer.Root(), path: dest}, nil } // normalizeDest normalises the destination of a COPY/ADD command in a diff --git a/components/engine/builder/dockerfile/mockbackend_test.go b/components/engine/builder/dockerfile/mockbackend_test.go index 06adb928fd..77bc8b5715 100644 --- a/components/engine/builder/dockerfile/mockbackend_test.go +++ b/components/engine/builder/dockerfile/mockbackend_test.go @@ -20,7 +20,7 @@ import ( type MockBackend struct { containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) commitFunc func(backend.CommitConfig) (image.ID, error) - getImageFunc func(string) (builder.Image, builder.ReleaseableLayer, error) + getImageFunc func(string) (builder.Image, builder.ROLayer, error) makeImageCacheFunc func(cacheFrom []string) builder.ImageCache } @@ -66,7 +66,7 @@ func (m *MockBackend) CopyOnBuild(containerID string, destPath string, srcRoot s return nil } -func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) { +func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if m.getImageFunc != nil { return m.getImageFunc(refOrID) } @@ -124,14 +124,25 @@ func (l *mockLayer) Release() error { return nil } -func (l *mockLayer) Mount() (containerfs.ContainerFS, error) { - return containerfs.NewLocalContainerFS("mountPath"), nil -} - -func (l *mockLayer) Commit() (builder.ReleaseableLayer, error) { - return nil, nil +func (l *mockLayer) NewRWLayer() (builder.RWLayer, error) { + return &mockRWLayer{}, nil } func (l *mockLayer) DiffID() layer.DiffID { return layer.DiffID("abcdef") } + +type mockRWLayer struct { +} + +func (l *mockRWLayer) Release() error { + return nil +} + +func (l *mockRWLayer) Commit() (builder.ROLayer, error) { + return nil, nil +} + +func (l *mockRWLayer) Root() containerfs.ContainerFS { + return nil +} diff --git a/components/engine/client/client_test.go b/components/engine/client/client_test.go index db394e24b5..d6b8f67352 100644 --- a/components/engine/client/client_test.go +++ b/components/engine/client/client_test.go @@ -248,7 +248,7 @@ func TestNegotiateAPIVersion(t *testing.T) { } // TestNegotiateAPIVersionOverride asserts that we honor -// the environment variable DOCKER_API_VERSION when negotianing versions +// the environment variable DOCKER_API_VERSION when negotiating versions func TestNegotiateAPVersionOverride(t *testing.T) { expected := "9.99" defer env.PatchAll(t, map[string]string{"DOCKER_API_VERSION": expected})() diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index cba9a7bbfc..b8552973e8 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -253,6 +253,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { Root: cli.Config.Root, Name: name, Backend: d, + ImageBackend: d.ImageService(), PluginBackend: d.PluginManager(), NetworkSubnetsProvider: d, DefaultAdvertiseAddr: cli.Config.SwarmDefaultAdvertiseAddr, @@ -345,12 +346,12 @@ func newRouterOptions(config *config.Config, daemon *daemon.Daemon) (routerOptio return opts, errors.Wrap(err, "failed to create fscache") } - manager, err := dockerfile.NewBuildManager(daemon, sm, buildCache, daemon.IDMappings()) + manager, err := dockerfile.NewBuildManager(daemon.BuilderBackend(), sm, buildCache, daemon.IDMappings()) if err != nil { return opts, err } - bb, err := buildbackend.NewBackend(daemon, manager, buildCache) + bb, err := buildbackend.NewBackend(daemon.ImageService(), manager, buildCache) if err != nil { return opts, errors.Wrap(err, "failed to create buildmanager") } @@ -507,14 +508,14 @@ func initRouter(opts routerOptions) { // we need to add the checkpoint router before the container router or the DELETE gets masked checkpointrouter.NewRouter(opts.daemon, decoder), container.NewRouter(opts.daemon, decoder), - image.NewRouter(opts.daemon), + image.NewRouter(opts.daemon.ImageService()), systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache), volume.NewRouter(opts.daemon), build.NewRouter(opts.buildBackend, opts.daemon), sessionrouter.NewRouter(opts.sessionManager), swarmrouter.NewRouter(opts.cluster), pluginrouter.NewRouter(opts.daemon.PluginManager()), - distributionrouter.NewRouter(opts.daemon), + distributionrouter.NewRouter(opts.daemon.ImageService()), } if opts.daemon.NetworkControllerEnabled() { diff --git a/components/engine/container/container.go b/components/engine/container/container.go index 46e592ab45..461139b435 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -38,7 +38,7 @@ import ( "github.com/docker/docker/runconfig" "github.com/docker/docker/volume" "github.com/docker/go-connections/nat" - "github.com/docker/go-units" + units "github.com/docker/go-units" "github.com/docker/libnetwork" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/options" @@ -391,6 +391,8 @@ func (container *Container) StartLogger() (logger.Logger, error) { if err != nil { return nil, err } + + container.LogPath = info.LogPath } l, err := initDriver(info) @@ -974,11 +976,6 @@ func (container *Container) startLogging() error { copier.Run() container.LogDriver = l - // set LogPath field only for json-file logdriver - if jl, ok := l.(*jsonfilelog.JSONFileLogger); ok { - container.LogPath = jl.LogPath() - } - return nil } diff --git a/components/engine/container/container_unit_test.go b/components/engine/container/container_unit_test.go index f6ded8fa9d..863a47a1f2 100644 --- a/components/engine/container/container_unit_test.go +++ b/components/engine/container/container_unit_test.go @@ -1,12 +1,17 @@ package container // import "github.com/docker/docker/container" import ( + "fmt" + "io/ioutil" + "os" "path/filepath" "testing" "github.com/docker/docker/api/types/container" swarmtypes "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/daemon/logger/jsonfilelog" "github.com/docker/docker/pkg/signal" + "github.com/stretchr/testify/require" ) func TestContainerStopSignal(t *testing.T) { @@ -66,3 +71,56 @@ func TestContainerSecretReferenceDestTarget(t *testing.T) { t.Fatalf("expected secret dest %q; received %q", expected, d) } } + +func TestContainerLogPathSetForJSONFileLogger(t *testing.T) { + containerRoot, err := ioutil.TempDir("", "TestContainerLogPathSetForJSONFileLogger") + require.NoError(t, err) + defer os.RemoveAll(containerRoot) + + c := &Container{ + Config: &container.Config{}, + HostConfig: &container.HostConfig{ + LogConfig: container.LogConfig{ + Type: jsonfilelog.Name, + }, + }, + ID: "TestContainerLogPathSetForJSONFileLogger", + Root: containerRoot, + } + + logger, err := c.StartLogger() + require.NoError(t, err) + defer logger.Close() + + expectedLogPath, err := filepath.Abs(filepath.Join(containerRoot, fmt.Sprintf("%s-json.log", c.ID))) + require.NoError(t, err) + require.Equal(t, c.LogPath, expectedLogPath) +} + +func TestContainerLogPathSetForRingLogger(t *testing.T) { + containerRoot, err := ioutil.TempDir("", "TestContainerLogPathSetForRingLogger") + require.NoError(t, err) + defer os.RemoveAll(containerRoot) + + c := &Container{ + Config: &container.Config{}, + HostConfig: &container.HostConfig{ + LogConfig: container.LogConfig{ + Type: jsonfilelog.Name, + Config: map[string]string{ + "mode": string(container.LogModeNonBlock), + }, + }, + }, + ID: "TestContainerLogPathSetForRingLogger", + Root: containerRoot, + } + + logger, err := c.StartLogger() + require.NoError(t, err) + defer logger.Close() + + expectedLogPath, err := filepath.Abs(filepath.Join(containerRoot, fmt.Sprintf("%s-json.log", c.ID))) + require.NoError(t, err) + require.Equal(t, c.LogPath, expectedLogPath) +} diff --git a/components/engine/daemon/build.go b/components/engine/daemon/build.go deleted file mode 100644 index 0384c13f32..0000000000 --- a/components/engine/daemon/build.go +++ /dev/null @@ -1,229 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "io" - - "github.com/docker/distribution/reference" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/backend" - "github.com/docker/docker/builder" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" - "github.com/docker/docker/pkg/containerfs" - "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/pkg/system" - "github.com/docker/docker/registry" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" -) - -type releaseableLayer struct { - released bool - layerStore layer.Store - roLayer layer.Layer - rwLayer layer.RWLayer -} - -func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) { - var err error - var mountPath containerfs.ContainerFS - var chainID layer.ChainID - if rl.roLayer != nil { - chainID = rl.roLayer.ChainID() - } - - mountID := stringid.GenerateRandomID() - rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, nil) - if err != nil { - return nil, errors.Wrap(err, "failed to create rwlayer") - } - - mountPath, err = rl.rwLayer.Mount("") - if err != nil { - // Clean up the layer if we fail to mount it here. - metadata, err := rl.layerStore.ReleaseRWLayer(rl.rwLayer) - layer.LogReleaseMetadata(metadata) - if err != nil { - logrus.Errorf("Failed to release RWLayer: %s", err) - } - rl.rwLayer = nil - return nil, err - } - - return mountPath, nil -} - -func (rl *releaseableLayer) Commit() (builder.ReleaseableLayer, error) { - var chainID layer.ChainID - if rl.roLayer != nil { - chainID = rl.roLayer.ChainID() - } - - stream, err := rl.rwLayer.TarStream() - if err != nil { - return nil, err - } - defer stream.Close() - - newLayer, err := rl.layerStore.Register(stream, chainID) - if err != nil { - return nil, err - } - // TODO: An optimization would be to handle empty layers before returning - return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer}, nil -} - -func (rl *releaseableLayer) DiffID() layer.DiffID { - if rl.roLayer == nil { - return layer.DigestSHA256EmptyTar - } - return rl.roLayer.DiffID() -} - -func (rl *releaseableLayer) Release() error { - if rl.released { - return nil - } - if err := rl.releaseRWLayer(); err != nil { - // Best effort attempt at releasing read-only layer before returning original error. - rl.releaseROLayer() - return err - } - if err := rl.releaseROLayer(); err != nil { - return err - } - rl.released = true - return nil -} - -func (rl *releaseableLayer) releaseRWLayer() error { - if rl.rwLayer == nil { - return nil - } - if err := rl.rwLayer.Unmount(); err != nil { - logrus.Errorf("Failed to unmount RWLayer: %s", err) - return err - } - metadata, err := rl.layerStore.ReleaseRWLayer(rl.rwLayer) - layer.LogReleaseMetadata(metadata) - if err != nil { - logrus.Errorf("Failed to release RWLayer: %s", err) - } - rl.rwLayer = nil - return err -} - -func (rl *releaseableLayer) releaseROLayer() error { - if rl.roLayer == nil { - return nil - } - metadata, err := rl.layerStore.Release(rl.roLayer) - layer.LogReleaseMetadata(metadata) - if err != nil { - logrus.Errorf("Failed to release ROLayer: %s", err) - } - rl.roLayer = nil - return err -} - -func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (builder.ReleaseableLayer, error) { - if img == nil || img.RootFS.ChainID() == "" { - return &releaseableLayer{layerStore: layerStore}, nil - } - // Hold a reference to the image layer so that it can't be removed before - // it is released - roLayer, err := layerStore.Get(img.RootFS.ChainID()) - if err != nil { - return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID()) - } - return &releaseableLayer{layerStore: layerStore, roLayer: roLayer}, nil -} - -// TODO: could this use the regular daemon PullImage ? -func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { - ref, err := reference.ParseNormalizedNamed(name) - if err != nil { - return nil, err - } - ref = reference.TagNameOnly(ref) - - pullRegistryAuth := &types.AuthConfig{} - if len(authConfigs) > 0 { - // The request came with a full auth config, use it - repoInfo, err := daemon.RegistryService.ResolveRepository(ref) - if err != nil { - return nil, err - } - - resolvedConfig := registry.ResolveAuthConfig(authConfigs, repoInfo.Index) - pullRegistryAuth = &resolvedConfig - } - - if err := daemon.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil { - return nil, err - } - return daemon.GetImage(name) -} - -// GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID. -// Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent -// leaking of layers. -func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) { - if refOrID == "" { - if !system.IsOSSupported(opts.OS) { - return nil, nil, system.ErrNotSupportedOperatingSystem - } - layer, err := newReleasableLayerForImage(nil, daemon.layerStores[opts.OS]) - return nil, layer, err - } - - if opts.PullOption != backend.PullOptionForcePull { - image, err := daemon.GetImage(refOrID) - if err != nil && opts.PullOption == backend.PullOptionNoPull { - return nil, nil, err - } - // TODO: shouldn't we error out if error is different from "not found" ? - if image != nil { - if !system.IsOSSupported(image.OperatingSystem()) { - return nil, nil, system.ErrNotSupportedOperatingSystem - } - layer, err := newReleasableLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) - return image, layer, err - } - } - - image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.OS) - if err != nil { - return nil, nil, err - } - if !system.IsOSSupported(image.OperatingSystem()) { - return nil, nil, system.ErrNotSupportedOperatingSystem - } - layer, err := newReleasableLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) - return image, layer, err -} - -// CreateImage creates a new image by adding a config and ID to the image store. -// This is similar to LoadImage() except that it receives JSON encoded bytes of -// an image instead of a tar archive. -func (daemon *Daemon) CreateImage(config []byte, parent string) (builder.Image, error) { - id, err := daemon.imageStore.Create(config) - if err != nil { - return nil, errors.Wrapf(err, "failed to create image") - } - - if parent != "" { - if err := daemon.imageStore.SetParent(id, image.ID(parent)); err != nil { - return nil, errors.Wrapf(err, "failed to set parent %s", parent) - } - } - - return daemon.imageStore.Get(id) -} - -// IDMappings returns uid/gid mappings for the builder -func (daemon *Daemon) IDMappings() *idtools.IDMappings { - return daemon.idMappings -} diff --git a/components/engine/daemon/cluster/cluster.go b/components/engine/daemon/cluster/cluster.go index 7fdf633bd8..cdd47ec0e7 100644 --- a/components/engine/daemon/cluster/cluster.go +++ b/components/engine/daemon/cluster/cluster.go @@ -83,6 +83,7 @@ type Config struct { Root string Name string Backend executorpkg.Backend + ImageBackend executorpkg.ImageBackend PluginBackend plugin.Backend NetworkSubnetsProvider NetworkSubnetsProvider diff --git a/components/engine/daemon/cluster/executor/backend.go b/components/engine/daemon/cluster/executor/backend.go index a0b56a97bc..3a5ff479bd 100644 --- a/components/engine/daemon/cluster/executor/backend.go +++ b/components/engine/daemon/cluster/executor/backend.go @@ -31,7 +31,6 @@ type Backend interface { FindNetwork(idName string) (libnetwork.Network, error) SetupIngress(clustertypes.NetworkCreateRequest, string) (<-chan struct{}, error) ReleaseIngress() (<-chan struct{}, error) - PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error ContainerStop(name string, seconds *int) error @@ -58,9 +57,14 @@ type Backend interface { UnsubscribeFromEvents(listener chan interface{}) UpdateAttachment(string, string, string, *network.NetworkingConfig) error WaitForDetachment(context.Context, string, string, string, string) error - GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) - LookupImage(name string) (*types.ImageInspect, error) PluginManager() *plugin.Manager PluginGetter() *plugin.Store GetAttachmentStore() *networkSettings.AttachmentStore } + +// ImageBackend is used by an executor to perform image operations +type ImageBackend interface { + PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error + GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) + LookupImage(name string) (*types.ImageInspect, error) +} diff --git a/components/engine/daemon/cluster/executor/container/adapter.go b/components/engine/daemon/cluster/executor/container/adapter.go index 0520fa0bc1..7395036cc8 100644 --- a/components/engine/daemon/cluster/executor/container/adapter.go +++ b/components/engine/daemon/cluster/executor/container/adapter.go @@ -36,11 +36,12 @@ import ( // containerConfig. type containerAdapter struct { backend executorpkg.Backend + imageBackend executorpkg.ImageBackend container *containerConfig dependencies exec.DependencyGetter } -func newContainerAdapter(b executorpkg.Backend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*containerAdapter, error) { +func newContainerAdapter(b executorpkg.Backend, i executorpkg.ImageBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*containerAdapter, error) { ctnr, err := newContainerConfig(task, node) if err != nil { return nil, err @@ -49,6 +50,7 @@ func newContainerAdapter(b executorpkg.Backend, task *api.Task, node *api.NodeDe return &containerAdapter{ container: ctnr, backend: b, + imageBackend: i, dependencies: dependencies, }, nil } @@ -66,7 +68,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { named, err := reference.ParseNormalizedNamed(spec.Image) if err == nil { if _, ok := named.(reference.Canonical); ok { - _, err := c.backend.LookupImage(spec.Image) + _, err := c.imageBackend.LookupImage(spec.Image) if err == nil { return nil } @@ -92,7 +94,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { // TODO @jhowardmsft LCOW Support: This will need revisiting as // the stack is built up to include LCOW support for swarm. platform := runtime.GOOS - err := c.backend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) + err := c.imageBackend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) pw.CloseWithError(err) }() diff --git a/components/engine/daemon/cluster/executor/container/attachment.go b/components/engine/daemon/cluster/executor/container/attachment.go index 0c630745ba..4d6e5c2983 100644 --- a/components/engine/daemon/cluster/executor/container/attachment.go +++ b/components/engine/daemon/cluster/executor/container/attachment.go @@ -20,8 +20,8 @@ type networkAttacherController struct { closed chan struct{} } -func newNetworkAttacherController(b executorpkg.Backend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*networkAttacherController, error) { - adapter, err := newContainerAdapter(b, task, node, dependencies) +func newNetworkAttacherController(b executorpkg.Backend, i executorpkg.ImageBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*networkAttacherController, error) { + adapter, err := newContainerAdapter(b, i, task, node, dependencies) if err != nil { return nil, err } diff --git a/components/engine/daemon/cluster/executor/container/controller.go b/components/engine/daemon/cluster/executor/container/controller.go index 5b7732cf3f..32bb114fde 100644 --- a/components/engine/daemon/cluster/executor/container/controller.go +++ b/components/engine/daemon/cluster/executor/container/controller.go @@ -40,8 +40,8 @@ type controller struct { var _ exec.Controller = &controller{} // NewController returns a docker exec runner for the provided task. -func newController(b executorpkg.Backend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*controller, error) { - adapter, err := newContainerAdapter(b, task, node, dependencies) +func newController(b executorpkg.Backend, i executorpkg.ImageBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*controller, error) { + adapter, err := newContainerAdapter(b, i, task, node, dependencies) if err != nil { return nil, err } diff --git a/components/engine/daemon/cluster/executor/container/executor.go b/components/engine/daemon/cluster/executor/container/executor.go index 2c4f619cf0..713933fb6f 100644 --- a/components/engine/daemon/cluster/executor/container/executor.go +++ b/components/engine/daemon/cluster/executor/container/executor.go @@ -26,6 +26,7 @@ import ( type executor struct { backend executorpkg.Backend + imageBackend executorpkg.ImageBackend pluginBackend plugin.Backend dependencies exec.DependencyManager mutex sync.Mutex // This mutex protects the following node field @@ -33,10 +34,11 @@ type executor struct { } // NewExecutor returns an executor from the docker client. -func NewExecutor(b executorpkg.Backend, p plugin.Backend) exec.Executor { +func NewExecutor(b executorpkg.Backend, p plugin.Backend, i executorpkg.ImageBackend) exec.Executor { return &executor{ backend: b, pluginBackend: p, + imageBackend: i, dependencies: agent.NewDependencyManager(), } } @@ -200,7 +202,7 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) { e.mutex.Unlock() if t.Spec.GetAttachment() != nil { - return newNetworkAttacherController(e.backend, t, nodeDescription, dependencyGetter) + return newNetworkAttacherController(e.backend, e.imageBackend, t, nodeDescription, dependencyGetter) } var ctlr exec.Controller @@ -229,7 +231,7 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) { return ctlr, fmt.Errorf("unsupported runtime type: %q", runtimeKind) } case *api.TaskSpec_Container: - c, err := newController(e.backend, t, nodeDescription, dependencyGetter) + c, err := newController(e.backend, e.imageBackend, t, nodeDescription, dependencyGetter) if err != nil { return ctlr, err } diff --git a/components/engine/daemon/cluster/executor/container/health_test.go b/components/engine/daemon/cluster/executor/container/health_test.go index 59633dd127..2054286417 100644 --- a/components/engine/daemon/cluster/executor/container/health_test.go +++ b/components/engine/daemon/cluster/executor/container/health_test.go @@ -52,7 +52,7 @@ func TestHealthStates(t *testing.T) { EventsService: e, } - controller, err := newController(daemon, task, nil, nil) + controller, err := newController(daemon, nil, task, nil, nil) if err != nil { t.Fatalf("create controller fail %v", err) } diff --git a/components/engine/daemon/cluster/executor/container/validate_test.go b/components/engine/daemon/cluster/executor/container/validate_test.go index d843431f4f..9b6cc8ee22 100644 --- a/components/engine/daemon/cluster/executor/container/validate_test.go +++ b/components/engine/daemon/cluster/executor/container/validate_test.go @@ -12,7 +12,7 @@ import ( ) func newTestControllerWithMount(m api.Mount) (*controller, error) { - return newController(&daemon.Daemon{}, &api.Task{ + return newController(&daemon.Daemon{}, nil, &api.Task{ ID: stringid.GenerateRandomID(), ServiceID: stringid.GenerateRandomID(), Spec: api.TaskSpec{ diff --git a/components/engine/daemon/cluster/noderunner.go b/components/engine/daemon/cluster/noderunner.go index f832aca145..b538e844e7 100644 --- a/components/engine/daemon/cluster/noderunner.go +++ b/components/engine/daemon/cluster/noderunner.go @@ -120,12 +120,15 @@ func (n *nodeRunner) start(conf nodeStartConfig) error { JoinAddr: joinAddr, StateDir: n.cluster.root, JoinToken: conf.joinToken, - Executor: container.NewExecutor(n.cluster.config.Backend, n.cluster.config.PluginBackend), - HeartbeatTick: 1, - ElectionTick: 3, - UnlockKey: conf.lockKey, - AutoLockManagers: conf.autolock, - PluginGetter: n.cluster.config.Backend.PluginGetter(), + Executor: container.NewExecutor( + n.cluster.config.Backend, + n.cluster.config.PluginBackend, + n.cluster.config.ImageBackend), + HeartbeatTick: 1, + ElectionTick: 3, + UnlockKey: conf.lockKey, + AutoLockManagers: conf.autolock, + PluginGetter: n.cluster.config.Backend.PluginGetter(), } if conf.availability != "" { avail, ok := swarmapi.NodeSpec_Availability_value[strings.ToUpper(string(conf.availability))] diff --git a/components/engine/daemon/cluster/services.go b/components/engine/daemon/cluster/services.go index 5cc9566fa0..3923ced6c0 100644 --- a/components/engine/daemon/cluster/services.go +++ b/components/engine/daemon/cluster/services.go @@ -570,7 +570,7 @@ func (c *Cluster) imageWithDigestString(ctx context.Context, image string, authC return "", errors.Errorf("image reference not tagged: %s", image) } - repo, _, err := c.config.Backend.GetRepository(ctx, taggedRef, authConfig) + repo, _, err := c.config.ImageBackend.GetRepository(ctx, taggedRef, authConfig) if err != nil { return "", err } diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 6af494da74..0f6f440514 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -155,7 +155,7 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma return "", err } - id, err := daemon.commitImage(backend.CommitConfig{ + id, err := daemon.imageService.CommitImage(backend.CommitConfig{ Author: c.Author, Comment: c.Comment, Config: newConfig, @@ -171,7 +171,7 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma var imageRef string if c.Repo != "" { - imageRef, err = daemon.TagImage(string(id), c.Repo, c.Tag) + imageRef, err = daemon.imageService.TagImage(string(id), c.Repo, c.Tag) if err != nil { return "", err } diff --git a/components/engine/daemon/container.go b/components/engine/daemon/container.go index a16abf2144..b1ae3daa94 100644 --- a/components/engine/daemon/container.go +++ b/components/engine/daemon/container.go @@ -158,7 +158,7 @@ func (daemon *Daemon) newContainer(name string, operatingSystem string, config * base.ImageID = imgID base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName} base.Name = name - base.Driver = daemon.GraphDriverName(operatingSystem) + base.Driver = daemon.imageService.GraphDriverForOS(operatingSystem) base.OS = operatingSystem return base, err } diff --git a/components/engine/daemon/container_operations_windows.go b/components/engine/daemon/container_operations_windows.go index 0559b8ac3e..a4541b0634 100644 --- a/components/engine/daemon/container_operations_windows.go +++ b/components/engine/daemon/container_operations_windows.go @@ -64,12 +64,6 @@ func (daemon *Daemon) setupConfigDir(c *container.Container) (setupErr error) { return nil } -// getSize returns real size & virtual size -func (daemon *Daemon) getSize(containerID string) (int64, int64) { - // TODO Windows - return 0, 0 -} - func (daemon *Daemon) setupIpcDirs(container *container.Container) error { return nil } diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index 7e6b6519fe..e57452b188 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" @@ -42,7 +41,7 @@ func (daemon *Daemon) containerCreate(params types.ContainerCreateConfig, manage os := runtime.GOOS if params.Config.Image != "" { - img, err := daemon.GetImage(params.Config.Image) + img, err := daemon.imageService.GetImage(params.Config.Image) if err == nil { os = img.OS } @@ -92,7 +91,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( os := runtime.GOOS if params.Config.Image != "" { - img, err = daemon.GetImage(params.Config.Image) + img, err = daemon.imageService.GetImage(params.Config.Image) if err != nil { return nil, err } @@ -158,9 +157,11 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( } // Set RWLayer for container after mount labels have been set - if err := daemon.setRWLayer(container); err != nil { + rwLayer, err := daemon.imageService.CreateLayer(container, setupInitLayer(daemon.idMappings)) + if err != nil { return nil, errdefs.System(err) } + container.RWLayer = rwLayer rootIDs := daemon.idMappings.RootPair() if err := idtools.MkdirAndChown(container.Root, 0700, rootIDs); err != nil { @@ -254,33 +255,6 @@ func (daemon *Daemon) generateSecurityOpt(hostConfig *containertypes.HostConfig) return nil, nil } -func (daemon *Daemon) setRWLayer(container *container.Container) error { - var layerID layer.ChainID - if container.ImageID != "" { - img, err := daemon.imageStore.Get(container.ImageID) - if err != nil { - return err - } - layerID = img.RootFS.ChainID() - } - - rwLayerOpts := &layer.CreateRWLayerOpts{ - MountLabel: container.MountLabel, - InitFunc: setupInitLayer(daemon.idMappings), - StorageOpt: container.HostConfig.StorageOpt, - } - - // Indexing by OS is safe here as validation of OS has already been performed in create() (the only - // caller), and guaranteed non-nil - rwLayer, err := daemon.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) - if err != nil { - return err - } - container.RWLayer = rwLayer - - return nil -} - // VolumeCreate creates a volume with the specified name, driver, and opts // This is called directly from the Engine API func (daemon *Daemon) VolumeCreate(name, driverName string, opts, labels map[string]string) (*types.Volume, error) { diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 335e6442f7..8c36f042ea 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -21,11 +21,13 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/builder" "github.com/docker/docker/container" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/exec" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/network" "github.com/docker/docker/errdefs" @@ -34,7 +36,6 @@ import ( _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/stats" dmetadata "github.com/docker/docker/distribution/metadata" - "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/dockerversion" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -57,7 +58,6 @@ import ( "github.com/docker/libnetwork" "github.com/docker/libnetwork/cluster" nwconfig "github.com/docker/libnetwork/config" - "github.com/docker/libtrust" "github.com/pkg/errors" ) @@ -70,44 +70,38 @@ var ( // Daemon holds information about the Docker daemon. type Daemon struct { - ID string - repository string - containers container.Store - containersReplica container.ViewDB - execCommands *exec.Store - downloadManager *xfer.LayerDownloadManager - uploadManager *xfer.LayerUploadManager - trustKey libtrust.PrivateKey - idIndex *truncindex.TruncIndex - configStore *config.Config - statsCollector *stats.Collector - defaultLogConfig containertypes.LogConfig - RegistryService registry.Service - EventsService *events.Events - netController libnetwork.NetworkController - volumes *store.VolumeStore - discoveryWatcher discovery.Reloader - root string - seccompEnabled bool - apparmorEnabled bool - shutdown bool - idMappings *idtools.IDMappings - graphDrivers map[string]string // By operating system - referenceStore refstore.Store - imageStore image.Store - imageRoot string - layerStores map[string]layer.Store // By operating system - distributionMetadataStore dmetadata.Store - PluginStore *plugin.Store // todo: remove - pluginManager *plugin.Manager - linkIndex *linkIndex - containerd libcontainerd.Client - containerdRemote libcontainerd.Remote - defaultIsolation containertypes.Isolation // Default isolation mode on Windows - clusterProvider cluster.Provider - cluster Cluster - genericResources []swarm.GenericResource - metricsPluginListener net.Listener + ID string + repository string + containers container.Store + containersReplica container.ViewDB + execCommands *exec.Store + imageService *images.ImageService + idIndex *truncindex.TruncIndex + configStore *config.Config + statsCollector *stats.Collector + defaultLogConfig containertypes.LogConfig + RegistryService registry.Service + EventsService *events.Events + netController libnetwork.NetworkController + volumes *store.VolumeStore + discoveryWatcher discovery.Reloader + root string + seccompEnabled bool + apparmorEnabled bool + shutdown bool + idMappings *idtools.IDMappings + // TODO: move graphDrivers field to an InfoService + graphDrivers map[string]string // By operating system + + PluginStore *plugin.Store // todo: remove + pluginManager *plugin.Manager + linkIndex *linkIndex + containerd libcontainerd.Client + defaultIsolation containertypes.Isolation // Default isolation mode on Windows + clusterProvider cluster.Provider + cluster Cluster + genericResources []swarm.GenericResource + metricsPluginListener net.Listener machineMemory uint64 @@ -162,7 +156,7 @@ func (daemon *Daemon) restore() error { // Ignore the container if it does not support the current driver being used by the graph currentDriverForContainerOS := daemon.graphDrivers[container.OS] if (container.Driver == "" && currentDriverForContainerOS == "aufs") || container.Driver == currentDriverForContainerOS { - rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID) + rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS) if err != nil { logrus.Errorf("Failed to load container mount %v: %v", id, err) continue @@ -705,7 +699,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // be set through an environment variable, a daemon start parameter, or chosen through // initialization of the layerstore through driver priority order for example. d.graphDrivers = make(map[string]string) - d.layerStores = make(map[string]layer.Store) + layerStores := make(map[string]layer.Store) if runtime.GOOS == "windows" { d.graphDrivers[runtime.GOOS] = "windowsfilter" if system.LCOWSupported() { @@ -754,7 +748,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe } for operatingSystem, gd := range d.graphDrivers { - d.layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{ + layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{ Root: config.Root, MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), GraphDriver: gd, @@ -771,7 +765,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // As layerstore initialization may set the driver for os := range d.graphDrivers { - d.graphDrivers[os] = d.layerStores[os].DriverName() + d.graphDrivers[os] = layerStores[os].DriverName() } // Configure and validate the kernels security support. Note this is a Linux/FreeBSD @@ -780,22 +774,17 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } - logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads) - d.downloadManager = xfer.NewLayerDownloadManager(d.layerStores, *config.MaxConcurrentDownloads) - logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) - d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads) - - d.imageRoot = filepath.Join(config.Root, "image", d.graphDrivers[runtime.GOOS]) - ifs, err := image.NewFSStoreBackend(filepath.Join(d.imageRoot, "imagedb")) + imageRoot := filepath.Join(config.Root, "image", d.graphDrivers[runtime.GOOS]) + ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) if err != nil { return nil, err } lgrMap := make(map[string]image.LayerGetReleaser) - for os, ls := range d.layerStores { + for os, ls := range layerStores { lgrMap[os] = ls } - d.imageStore, err = image.NewImageStore(ifs, lgrMap) + imageStore, err := image.NewImageStore(ifs, lgrMap) if err != nil { return nil, err } @@ -817,8 +806,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } - eventsService := events.New() - // We have a single tag/reference store for the daemon globally. However, it's // stored under the graphdriver. On host platforms which only support a single // container OS, but multiple selectable graphdrivers, this means depending on which @@ -829,14 +816,13 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // operating systems, the list of graphdrivers available isn't user configurable. // For backwards compatibility, we just put it under the windowsfilter // directory regardless. - refStoreLocation := filepath.Join(d.imageRoot, `repositories.json`) + refStoreLocation := filepath.Join(imageRoot, `repositories.json`) rs, err := refstore.NewReferenceStore(refStoreLocation) if err != nil { return nil, fmt.Errorf("Couldn't create reference store repository: %s", err) } - d.referenceStore = rs - d.distributionMetadataStore, err = dmetadata.NewFSMetadataStore(filepath.Join(d.imageRoot, "distribution")) + distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution")) if err != nil { return nil, err } @@ -844,7 +830,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // No content-addressability migration on Windows as it never supported pre-CA if runtime.GOOS != "windows" { migrationStart := time.Now() - if err := v1.Migrate(config.Root, d.graphDrivers[runtime.GOOS], d.layerStores[runtime.GOOS], d.imageStore, rs, d.distributionMetadataStore); err != nil { + if err := v1.Migrate(config.Root, d.graphDrivers[runtime.GOOS], layerStores[runtime.GOOS], imageStore, rs, distributionMetadataStore); err != nil { logrus.Errorf("Graph migration failed: %q. Your old graph data was found to be too inconsistent for upgrading to content-addressable storage. Some of the old data was probably not upgraded. We recommend starting over with a clean storage directory if possible.", err) } logrus.Infof("Graph migration to content-addressability took %.2f seconds", time.Since(migrationStart).Seconds()) @@ -870,20 +856,34 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } d.execCommands = exec.NewStore() - d.trustKey = trustKey d.idIndex = truncindex.NewTruncIndex([]string{}) d.statsCollector = d.newStatsCollector(1 * time.Second) - d.EventsService = eventsService + d.EventsService = events.New() d.volumes = volStore d.root = config.Root d.idMappings = idMappings d.seccompEnabled = sysInfo.Seccomp d.apparmorEnabled = sysInfo.AppArmor - d.containerdRemote = containerdRemote d.linkIndex = newLinkIndex() + // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only + // used above to run migration. They could be initialized in ImageService + // if migration is called from daemon/images. layerStore might move as well. + d.imageService = images.NewImageService(images.ImageServiceConfig{ + ContainerStore: d.containers, + DistributionMetadataStore: distributionMetadataStore, + EventsService: d.EventsService, + ImageStore: imageStore, + LayerStores: layerStores, + MaxConcurrentDownloads: *config.MaxConcurrentDownloads, + MaxConcurrentUploads: *config.MaxConcurrentUploads, + ReferenceStore: rs, + RegistryService: registryService, + TrustKey: trustKey, + }) + go d.execCommandGC() d.containerd, err = containerdRemote.NewClient(ContainersNamespace, d) @@ -1005,7 +1005,7 @@ func (daemon *Daemon) Shutdown() error { logrus.Errorf("Stop container error: %v", err) return } - if mountid, err := daemon.layerStores[c.OS].GetMountID(c.ID); err == nil { + if mountid, err := daemon.imageService.GetLayerMountID(c.ID, c.OS); err == nil { daemon.cleanupMountsByID(mountid) } logrus.Debugf("container stopped %s", c.ID) @@ -1018,12 +1018,8 @@ func (daemon *Daemon) Shutdown() error { } } - for os, ls := range daemon.layerStores { - if ls != nil { - if err := ls.Cleanup(); err != nil { - logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) - } - } + if daemon.imageService != nil { + daemon.imageService.Cleanup() } // If we are part of a cluster, clean up cluster's stuff @@ -1064,7 +1060,7 @@ func (daemon *Daemon) Mount(container *container.Container) error { if runtime.GOOS != "windows" { daemon.Unmount(container) return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')", - daemon.GraphDriverName(container.OS), container.ID, container.BaseFS, dir) + daemon.imageService.GraphDriverForOS(container.OS), container.ID, container.BaseFS, dir) } } container.BaseFS = dir // TODO: combine these fields @@ -1108,11 +1104,6 @@ func (daemon *Daemon) Subnets() ([]net.IPNet, []net.IPNet) { return v4Subnets, v6Subnets } -// GraphDriverName returns the name of the graph driver used by the layer.Store -func (daemon *Daemon) GraphDriverName(os string) string { - return daemon.layerStores[os].DriverName() -} - // prepareTempDir prepares and returns the default directory to use // for temporary files. // If it doesn't exist, it is created. If it exists, its content is removed. @@ -1323,3 +1314,21 @@ func fixMemorySwappiness(resources *containertypes.Resources) { func (daemon *Daemon) GetAttachmentStore() *network.AttachmentStore { return &daemon.attachmentStore } + +// IDMappings returns uid/gid mappings for the builder +func (daemon *Daemon) IDMappings() *idtools.IDMappings { + return daemon.idMappings +} + +// ImageService returns the Daemon's ImageService +func (daemon *Daemon) ImageService() *images.ImageService { + return daemon.imageService +} + +// BuilderBackend returns the backend used by builder +func (daemon *Daemon) BuilderBackend() builder.Backend { + return struct { + *Daemon + *images.ImageService + }{daemon, daemon.imageService} +} diff --git a/components/engine/daemon/delete.go b/components/engine/daemon/delete.go index b0ea416f9f..1eff18968f 100644 --- a/components/engine/daemon/delete.go +++ b/components/engine/daemon/delete.go @@ -10,7 +10,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/container" "github.com/docker/docker/errdefs" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/system" "github.com/docker/docker/volume" volumestore "github.com/docker/docker/volume/store" @@ -121,12 +120,11 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo // When container creation fails and `RWLayer` has not been created yet, we // do not call `ReleaseRWLayer` if container.RWLayer != nil { - metadata, err := daemon.layerStores[container.OS].ReleaseRWLayer(container.RWLayer) - layer.LogReleaseMetadata(metadata) - if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { - e := errors.Wrapf(err, "driver %q failed to remove root filesystem for %s", daemon.GraphDriverName(container.OS), container.ID) - container.SetRemovalError(e) - return e + err := daemon.imageService.ReleaseLayer(container.RWLayer, container.OS) + if err != nil { + err = errors.Wrapf(err, "container %s", container.ID) + container.SetRemovalError(err) + return err } container.RWLayer = nil } diff --git a/components/engine/daemon/disk_usage.go b/components/engine/daemon/disk_usage.go index 2ccafbbbf1..2f6132eb39 100644 --- a/components/engine/daemon/disk_usage.go +++ b/components/engine/daemon/disk_usage.go @@ -8,34 +8,11 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/volume" - "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" ) -func (daemon *Daemon) getLayerRefs() map[layer.ChainID]int { - tmpImages := daemon.imageStore.Map() - layerRefs := map[layer.ChainID]int{} - for id, img := range tmpImages { - dgst := digest.Digest(id) - if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { - continue - } - - rootFS := *img.RootFS - rootFS.DiffIDs = nil - for _, id := range img.RootFS.DiffIDs { - rootFS.Append(id) - chid := rootFS.ChainID() - layerRefs[chid]++ - } - } - - return layerRefs -} - // SystemDiskUsage returns information about the daemon data disk usage func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, error) { if !atomic.CompareAndSwapInt32(&daemon.diskUsageRunning, 0, 1) { @@ -53,7 +30,7 @@ func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, er } // Get all top images with extra attributes - allImages, err := daemon.Images(filters.NewArgs(), false, true) + allImages, err := daemon.imageService.Images(filters.NewArgs(), false, true) if err != nil { return nil, fmt.Errorf("failed to retrieve image list: %v", err) } @@ -93,28 +70,9 @@ func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, er return nil, err } - // Get total layers size on disk - var allLayersSize int64 - layerRefs := daemon.getLayerRefs() - for _, ls := range daemon.layerStores { - allLayers := ls.Map() - for _, l := range allLayers { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - size, err := l.DiffSize() - if err == nil { - if _, ok := layerRefs[l.ChainID()]; ok { - allLayersSize += size - } else { - logrus.Warnf("found leaked image layer %v", l.ChainID()) - } - } else { - logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) - } - } - } + allLayersSize, err := daemon.imageService.LayerDiskUsage(ctx) + if err != nil { + return nil, err } return &types.DiskUsage{ diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 1f788ad9a8..52c23a3c28 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -51,13 +51,13 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R if !system.IsOSSupported(container.OS) { return nil, fmt.Errorf("cannot export %s: %s ", container.ID, system.ErrNotSupportedOperatingSystem) } - rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID) + rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS) if err != nil { return nil, err } defer func() { if err != nil { - daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer) + daemon.imageService.ReleaseLayer(rwlayer, container.OS) } }() @@ -78,7 +78,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R arch = ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() rwlayer.Unmount() - daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer) + daemon.imageService.ReleaseLayer(rwlayer, container.OS) return err }) daemon.LogContainerEvent(container, "export") diff --git a/components/engine/daemon/image.go b/components/engine/daemon/image.go deleted file mode 100644 index 53df7417d0..0000000000 --- a/components/engine/daemon/image.go +++ /dev/null @@ -1,73 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "fmt" - - "github.com/docker/distribution/reference" - "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" -) - -// errImageDoesNotExist is error returned when no image can be found for a reference. -type errImageDoesNotExist struct { - ref reference.Reference -} - -func (e errImageDoesNotExist) Error() string { - ref := e.ref - if named, ok := ref.(reference.Named); ok { - ref = reference.TagNameOnly(named) - } - return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref)) -} - -func (e errImageDoesNotExist) NotFound() {} - -// GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by -// refOrID. -func (daemon *Daemon) GetImageIDAndOS(refOrID string) (image.ID, string, error) { - ref, err := reference.ParseAnyReference(refOrID) - if err != nil { - return "", "", errdefs.InvalidParameter(err) - } - namedRef, ok := ref.(reference.Named) - if !ok { - digested, ok := ref.(reference.Digested) - if !ok { - return "", "", errImageDoesNotExist{ref} - } - id := image.IDFromDigest(digested.Digest()) - if img, err := daemon.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil - } - return "", "", errImageDoesNotExist{ref} - } - - if digest, err := daemon.referenceStore.Get(namedRef); err == nil { - // Search the image stores to get the operating system, defaulting to host OS. - id := image.IDFromDigest(digest) - if img, err := daemon.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil - } - } - - // Search based on ID - if id, err := daemon.imageStore.Search(refOrID); err == nil { - img, err := daemon.imageStore.Get(id) - if err != nil { - return "", "", errImageDoesNotExist{ref} - } - return id, img.OperatingSystem(), nil - } - - return "", "", errImageDoesNotExist{ref} -} - -// GetImage returns an image corresponding to the image referred to by refOrID. -func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) { - imgID, _, err := daemon.GetImageIDAndOS(refOrID) - if err != nil { - return nil, err - } - return daemon.imageStore.Get(imgID) -} diff --git a/components/engine/daemon/image_events.go b/components/engine/daemon/image_events.go deleted file mode 100644 index 0d364f7b58..0000000000 --- a/components/engine/daemon/image_events.go +++ /dev/null @@ -1,29 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "github.com/docker/docker/api/types/events" -) - -// LogImageEvent generates an event related to an image with only the default attributes. -func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { - daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := daemon.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - daemon.EventsService.Log(action, events.ImageEventType, actor) -} diff --git a/components/engine/daemon/cache.go b/components/engine/daemon/images/cache.go similarity index 60% rename from components/engine/daemon/cache.go rename to components/engine/daemon/images/cache.go index 2a42011eff..3b433106e8 100644 --- a/components/engine/daemon/cache.go +++ b/components/engine/daemon/images/cache.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "github.com/docker/docker/builder" @@ -7,15 +7,15 @@ import ( ) // MakeImageCache creates a stateful image cache. -func (daemon *Daemon) MakeImageCache(sourceRefs []string) builder.ImageCache { +func (i *ImageService) MakeImageCache(sourceRefs []string) builder.ImageCache { if len(sourceRefs) == 0 { - return cache.NewLocal(daemon.imageStore) + return cache.NewLocal(i.imageStore) } - cache := cache.New(daemon.imageStore) + cache := cache.New(i.imageStore) for _, ref := range sourceRefs { - img, err := daemon.GetImage(ref) + img, err := i.GetImage(ref) if err != nil { logrus.Warnf("Could not look up %s for cache resolution, skipping: %+v", ref, err) continue diff --git a/components/engine/daemon/images/image.go b/components/engine/daemon/images/image.go new file mode 100644 index 0000000000..79cc07c4fd --- /dev/null +++ b/components/engine/daemon/images/image.go @@ -0,0 +1,64 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "fmt" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/errdefs" + "github.com/docker/docker/image" +) + +// ErrImageDoesNotExist is error returned when no image can be found for a reference. +type ErrImageDoesNotExist struct { + ref reference.Reference +} + +func (e ErrImageDoesNotExist) Error() string { + ref := e.ref + if named, ok := ref.(reference.Named); ok { + ref = reference.TagNameOnly(named) + } + return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref)) +} + +// NotFound implements the NotFound interface +func (e ErrImageDoesNotExist) NotFound() {} + +// GetImage returns an image corresponding to the image referred to by refOrID. +func (i *ImageService) GetImage(refOrID string) (*image.Image, error) { + ref, err := reference.ParseAnyReference(refOrID) + if err != nil { + return nil, errdefs.InvalidParameter(err) + } + namedRef, ok := ref.(reference.Named) + if !ok { + digested, ok := ref.(reference.Digested) + if !ok { + return nil, ErrImageDoesNotExist{ref} + } + id := image.IDFromDigest(digested.Digest()) + if img, err := i.imageStore.Get(id); err == nil { + return img, nil + } + return nil, ErrImageDoesNotExist{ref} + } + + if digest, err := i.referenceStore.Get(namedRef); err == nil { + // Search the image stores to get the operating system, defaulting to host OS. + id := image.IDFromDigest(digest) + if img, err := i.imageStore.Get(id); err == nil { + return img, nil + } + } + + // Search based on ID + if id, err := i.imageStore.Search(refOrID); err == nil { + img, err := i.imageStore.Get(id) + if err != nil { + return nil, ErrImageDoesNotExist{ref} + } + return img, nil + } + + return nil, ErrImageDoesNotExist{ref} +} diff --git a/components/engine/daemon/images/image_builder.go b/components/engine/daemon/images/image_builder.go new file mode 100644 index 0000000000..b88e1bc0fa --- /dev/null +++ b/components/engine/daemon/images/image_builder.go @@ -0,0 +1,219 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "io" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/backend" + "github.com/docker/docker/builder" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/containerfs" + "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/system" + "github.com/docker/docker/registry" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +type roLayer struct { + released bool + layerStore layer.Store + roLayer layer.Layer +} + +func (l *roLayer) DiffID() layer.DiffID { + if l.roLayer == nil { + return layer.DigestSHA256EmptyTar + } + return l.roLayer.DiffID() +} + +func (l *roLayer) Release() error { + if l.released { + return nil + } + if l.roLayer != nil { + metadata, err := l.layerStore.Release(l.roLayer) + layer.LogReleaseMetadata(metadata) + if err != nil { + return errors.Wrap(err, "failed to release ROLayer") + } + } + l.roLayer = nil + l.released = true + return nil +} + +func (l *roLayer) NewRWLayer() (builder.RWLayer, error) { + var chainID layer.ChainID + if l.roLayer != nil { + chainID = l.roLayer.ChainID() + } + + mountID := stringid.GenerateRandomID() + newLayer, err := l.layerStore.CreateRWLayer(mountID, chainID, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to create rwlayer") + } + + rwLayer := &rwLayer{layerStore: l.layerStore, rwLayer: newLayer} + + fs, err := newLayer.Mount("") + if err != nil { + rwLayer.Release() + return nil, err + } + + rwLayer.fs = fs + + return rwLayer, nil +} + +type rwLayer struct { + released bool + layerStore layer.Store + rwLayer layer.RWLayer + fs containerfs.ContainerFS +} + +func (l *rwLayer) Root() containerfs.ContainerFS { + return l.fs +} + +func (l *rwLayer) Commit() (builder.ROLayer, error) { + stream, err := l.rwLayer.TarStream() + if err != nil { + return nil, err + } + defer stream.Close() + + var chainID layer.ChainID + if parent := l.rwLayer.Parent(); parent != nil { + chainID = parent.ChainID() + } + + newLayer, err := l.layerStore.Register(stream, chainID) + if err != nil { + return nil, err + } + // TODO: An optimization would be to handle empty layers before returning + return &roLayer{layerStore: l.layerStore, roLayer: newLayer}, nil +} + +func (l *rwLayer) Release() error { + if l.released { + return nil + } + + if l.fs != nil { + if err := l.rwLayer.Unmount(); err != nil { + return errors.Wrap(err, "failed to unmount RWLayer") + } + l.fs = nil + } + + metadata, err := l.layerStore.ReleaseRWLayer(l.rwLayer) + layer.LogReleaseMetadata(metadata) + if err != nil { + return errors.Wrap(err, "failed to release RWLayer") + } + l.released = true + return nil +} + +func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLayer, error) { + if img == nil || img.RootFS.ChainID() == "" { + return &roLayer{layerStore: layerStore}, nil + } + // Hold a reference to the image layer so that it can't be removed before + // it is released + layer, err := layerStore.Get(img.RootFS.ChainID()) + if err != nil { + return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID()) + } + return &roLayer{layerStore: layerStore, roLayer: layer}, nil +} + +// TODO: could this use the regular daemon PullImage ? +func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { + ref, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, err + } + ref = reference.TagNameOnly(ref) + + pullRegistryAuth := &types.AuthConfig{} + if len(authConfigs) > 0 { + // The request came with a full auth config, use it + repoInfo, err := i.registryService.ResolveRepository(ref) + if err != nil { + return nil, err + } + + resolvedConfig := registry.ResolveAuthConfig(authConfigs, repoInfo.Index) + pullRegistryAuth = &resolvedConfig + } + + if err := i.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil { + return nil, err + } + return i.GetImage(name) +} + +// GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID. +// Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent +// leaking of layers. +func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { + if refOrID == "" { + if !system.IsOSSupported(opts.OS) { + return nil, nil, system.ErrNotSupportedOperatingSystem + } + layer, err := newROLayerForImage(nil, i.layerStores[opts.OS]) + return nil, layer, err + } + + if opts.PullOption != backend.PullOptionForcePull { + image, err := i.GetImage(refOrID) + if err != nil && opts.PullOption == backend.PullOptionNoPull { + return nil, nil, err + } + // TODO: shouldn't we error out if error is different from "not found" ? + if image != nil { + if !system.IsOSSupported(image.OperatingSystem()) { + return nil, nil, system.ErrNotSupportedOperatingSystem + } + layer, err := newROLayerForImage(image, i.layerStores[image.OperatingSystem()]) + return image, layer, err + } + } + + image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.OS) + if err != nil { + return nil, nil, err + } + if !system.IsOSSupported(image.OperatingSystem()) { + return nil, nil, system.ErrNotSupportedOperatingSystem + } + layer, err := newROLayerForImage(image, i.layerStores[image.OperatingSystem()]) + return image, layer, err +} + +// CreateImage creates a new image by adding a config and ID to the image store. +// This is similar to LoadImage() except that it receives JSON encoded bytes of +// an image instead of a tar archive. +func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) { + id, err := i.imageStore.Create(config) + if err != nil { + return nil, errors.Wrapf(err, "failed to create image") + } + + if parent != "" { + if err := i.imageStore.SetParent(id, image.ID(parent)); err != nil { + return nil, errors.Wrapf(err, "failed to set parent %s", parent) + } + } + + return i.imageStore.Get(id) +} diff --git a/components/engine/daemon/image_commit.go b/components/engine/daemon/images/image_commit.go similarity index 78% rename from components/engine/daemon/image_commit.go rename to components/engine/daemon/images/image_commit.go index 48f890b80d..4caba9f27b 100644 --- a/components/engine/daemon/image_commit.go +++ b/components/engine/daemon/images/image_commit.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -9,10 +9,12 @@ import ( "github.com/docker/docker/layer" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" ) -func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { - layerStore, ok := daemon.layerStores[c.ContainerOS] +// CommitImage creates a new image from a commit config +func (i *ImageService) CommitImage(c backend.CommitConfig) (image.ID, error) { + layerStore, ok := i.layerStores[c.ContainerOS] if !ok { return "", system.ErrNotSupportedOperatingSystem } @@ -31,7 +33,7 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { parent = new(image.Image) parent.RootFS = image.NewRootFS() } else { - parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID)) + parent, err = i.imageStore.Get(image.ID(c.ParentImageID)) if err != nil { return "", err } @@ -56,13 +58,13 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { return "", err } - id, err := daemon.imageStore.Create(config) + id, err := i.imageStore.Create(config) if err != nil { return "", err } if c.ParentImageID != "" { - if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { + if err := i.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { return "", err } } @@ -112,13 +114,14 @@ func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.R // * it doesn't log a container commit event // // This is a temporary shim. Should be removed when builder stops using commit. -func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { - container, err := daemon.GetContainer(c.ContainerID) - if err != nil { - return "", err +func (i *ImageService) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { + container := i.containers.Get(c.ContainerID) + if container == nil { + // TODO: use typed error + return "", errors.Errorf("container not found: %s", c.ContainerID) } c.ContainerMountLabel = container.MountLabel c.ContainerOS = container.OS c.ParentImageID = string(container.ImageID) - return daemon.commitImage(c) + return i.CommitImage(c) } diff --git a/components/engine/daemon/image_delete.go b/components/engine/daemon/images/image_delete.go similarity index 79% rename from components/engine/daemon/image_delete.go rename to components/engine/daemon/images/image_delete.go index c591390b4f..60896e487a 100644 --- a/components/engine/daemon/image_delete.go +++ b/components/engine/daemon/images/image_delete.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "fmt" @@ -60,22 +60,24 @@ const ( // meaning any delete conflicts will cause the image to not be deleted and the // conflict will not be reported. // -// FIXME: remove ImageDelete's dependency on Daemon, then move to the graph -// package. This would require that we no longer need the daemon to determine -// whether images are being used by a stopped or running container. -func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { +func (i *ImageService) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { start := time.Now() records := []types.ImageDeleteResponseItem{} - imgID, operatingSystem, err := daemon.GetImageIDAndOS(imageRef) + img, err := i.GetImage(imageRef) if err != nil { return nil, err } - if !system.IsOSSupported(operatingSystem) { + if !system.IsOSSupported(img.OperatingSystem()) { return nil, errors.Errorf("unable to delete image: %q", system.ErrNotSupportedOperatingSystem) } - repoRefs := daemon.referenceStore.References(imgID.Digest()) + imgID := img.ID() + repoRefs := i.referenceStore.References(imgID.Digest()) + + using := func(c *container.Container) bool { + return c.ImageID == imgID + } var removedRepositoryRef bool if !isImageIDPrefix(imgID.String(), imageRef) { @@ -84,7 +86,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I // true, there are multiple repository references to this // image, or there are no containers using the given reference. if !force && isSingleReference(repoRefs) { - if container := daemon.getContainerUsingImage(imgID); container != nil { + if container := i.containers.First(using); container != nil { // If we removed the repository reference then // this image would remain "dangling" and since // we really want to avoid that the client must @@ -99,17 +101,17 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I return nil, err } - parsedRef, err = daemon.removeImageRef(parsedRef) + parsedRef, err = i.removeImageRef(parsedRef) if err != nil { return nil, err } untaggedRecord := types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(parsedRef)} - daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") + i.LogImageEvent(imgID.String(), imgID.String(), "untag") records = append(records, untaggedRecord) - repoRefs = daemon.referenceStore.References(imgID.Digest()) + repoRefs = i.referenceStore.References(imgID.Digest()) // If a tag reference was removed and the only remaining // references to the same repository are digest references, @@ -127,7 +129,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I remainingRefs := []reference.Named{} for _, repoRef := range repoRefs { if _, repoRefIsCanonical := repoRef.(reference.Canonical); repoRefIsCanonical && parsedRef.Name() == repoRef.Name() { - if _, err := daemon.removeImageRef(repoRef); err != nil { + if _, err := i.removeImageRef(repoRef); err != nil { return records, err } @@ -157,25 +159,25 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I if !force { c |= conflictSoft &^ conflictActiveReference } - if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil { + if conflict := i.checkImageDeleteConflict(imgID, c); conflict != nil { return nil, conflict } for _, repoRef := range repoRefs { - parsedRef, err := daemon.removeImageRef(repoRef) + parsedRef, err := i.removeImageRef(repoRef) if err != nil { return nil, err } untaggedRecord := types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(parsedRef)} - daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") + i.LogImageEvent(imgID.String(), imgID.String(), "untag") records = append(records, untaggedRecord) } } } - if err := daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef); err != nil { + if err := i.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef); err != nil { return nil, err } @@ -223,26 +225,18 @@ func isImageIDPrefix(imageID, possiblePrefix string) bool { return false } -// getContainerUsingImage returns a container that was created using the given -// imageID. Returns nil if there is no such container. -func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *container.Container { - return daemon.containers.First(func(c *container.Container) bool { - return c.ImageID == imageID - }) -} - // removeImageRef attempts to parse and remove the given image reference from // this daemon's store of repository tag/digest references. The given // repositoryRef must not be an image ID but a repository name followed by an // optional tag or digest reference. If tag or digest is omitted, the default // tag is used. Returns the resolved image reference and an error. -func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) { +func (i *ImageService) removeImageRef(ref reference.Named) (reference.Named, error) { ref = reference.TagNameOnly(ref) // Ignore the boolean value returned, as far as we're concerned, this // is an idempotent operation and it's okay if the reference didn't // exist in the first place. - _, err := daemon.referenceStore.Delete(ref) + _, err := i.referenceStore.Delete(ref) return ref, err } @@ -252,18 +246,18 @@ func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, erro // on the first encountered error. Removed references are logged to this // daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the // given list of records. -func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { - imageRefs := daemon.referenceStore.References(imgID.Digest()) +func (i *ImageService) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { + imageRefs := i.referenceStore.References(imgID.Digest()) for _, imageRef := range imageRefs { - parsedRef, err := daemon.removeImageRef(imageRef) + parsedRef, err := i.removeImageRef(imageRef) if err != nil { return err } untaggedRecord := types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(parsedRef)} - daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") + i.LogImageEvent(imgID.String(), imgID.String(), "untag") *records = append(*records, untaggedRecord) } @@ -303,15 +297,15 @@ func (idc *imageDeleteConflict) Conflict() {} // conflict is encountered, it will be returned immediately without deleting // the image. If quiet is true, any encountered conflicts will be ignored and // the function will return nil immediately without deleting the image. -func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { +func (i *ImageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { // First, determine if this image has any conflicts. Ignore soft conflicts // if force is true. c := conflictHard if !force { c |= conflictSoft } - if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil { - if quiet && (!daemon.imageIsDangling(imgID) || conflict.used) { + if conflict := i.checkImageDeleteConflict(imgID, c); conflict != nil { + if quiet && (!i.imageIsDangling(imgID) || conflict.used) { // Ignore conflicts UNLESS the image is "dangling" or not being used in // which case we want the user to know. return nil @@ -322,23 +316,23 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe return conflict } - parent, err := daemon.imageStore.GetParent(imgID) + parent, err := i.imageStore.GetParent(imgID) if err != nil { // There may be no parent parent = "" } // Delete all repository tag/digest references to this image. - if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil { + if err := i.removeAllReferencesToImageID(imgID, records); err != nil { return err } - removedLayers, err := daemon.imageStore.Delete(imgID) + removedLayers, err := i.imageStore.Delete(imgID) if err != nil { return err } - daemon.LogImageEvent(imgID.String(), imgID.String(), "delete") + i.LogImageEvent(imgID.String(), imgID.String(), "delete") *records = append(*records, types.ImageDeleteResponseItem{Deleted: imgID.String()}) for _, removedLayer := range removedLayers { *records = append(*records, types.ImageDeleteResponseItem{Deleted: removedLayer.ChainID.String()}) @@ -353,7 +347,7 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe // either running or stopped). // Do not force prunings, but do so quietly (stopping on any encountered // conflicts). - return daemon.imageDeleteHelper(parent, records, false, true, true) + return i.imageDeleteHelper(parent, records, false, true, true) } // checkImageDeleteConflict determines whether there are any conflicts @@ -362,9 +356,9 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe // using the image. A soft conflict is any tags/digest referencing the given // image or any stopped container using the image. If ignoreSoftConflicts is // true, this function will not check for soft conflict conditions. -func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { +func (i *ImageService) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { // Check if the image has any descendant images. - if mask&conflictDependentChild != 0 && len(daemon.imageStore.Children(imgID)) > 0 { + if mask&conflictDependentChild != 0 && len(i.imageStore.Children(imgID)) > 0 { return &imageDeleteConflict{ hard: true, imgID: imgID, @@ -377,7 +371,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType running := func(c *container.Container) bool { return c.IsRunning() && c.ImageID == imgID } - if container := daemon.containers.First(running); container != nil { + if container := i.containers.First(running); container != nil { return &imageDeleteConflict{ imgID: imgID, hard: true, @@ -388,7 +382,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType } // Check if any repository tags/digest reference this image. - if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID.Digest())) > 0 { + if mask&conflictActiveReference != 0 && len(i.referenceStore.References(imgID.Digest())) > 0 { return &imageDeleteConflict{ imgID: imgID, message: "image is referenced in multiple repositories", @@ -400,7 +394,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType stopped := func(c *container.Container) bool { return !c.IsRunning() && c.ImageID == imgID } - if container := daemon.containers.First(stopped); container != nil { + if container := i.containers.First(stopped); container != nil { return &imageDeleteConflict{ imgID: imgID, used: true, @@ -415,6 +409,6 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType // imageIsDangling returns whether the given image is "dangling" which means // that there are no repository references to the given image and it has no // child images. -func (daemon *Daemon) imageIsDangling(imgID image.ID) bool { - return !(len(daemon.referenceStore.References(imgID.Digest())) > 0 || len(daemon.imageStore.Children(imgID)) > 0) +func (i *ImageService) imageIsDangling(imgID image.ID) bool { + return !(len(i.referenceStore.References(imgID.Digest())) > 0 || len(i.imageStore.Children(imgID)) > 0) } diff --git a/components/engine/daemon/images/image_events.go b/components/engine/daemon/images/image_events.go new file mode 100644 index 0000000000..d0b3064d70 --- /dev/null +++ b/components/engine/daemon/images/image_events.go @@ -0,0 +1,39 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "github.com/docker/docker/api/types/events" +) + +// LogImageEvent generates an event related to an image with only the default attributes. +func (i *ImageService) LogImageEvent(imageID, refName, action string) { + i.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) +} + +// LogImageEventWithAttributes generates an event related to an image with specific given attributes. +func (i *ImageService) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { + img, err := i.GetImage(imageID) + if err == nil && img.Config != nil { + // image has not been removed yet. + // it could be missing if the event is `delete`. + copyAttributes(attributes, img.Config.Labels) + } + if refName != "" { + attributes["name"] = refName + } + actor := events.Actor{ + ID: imageID, + Attributes: attributes, + } + + i.eventsService.Log(action, events.ImageEventType, actor) +} + +// copyAttributes guarantees that labels are not mutated by event triggers. +func copyAttributes(attributes, labels map[string]string) { + if labels == nil { + return + } + for k, v := range labels { + attributes[k] = v + } +} diff --git a/components/engine/daemon/image_exporter.go b/components/engine/daemon/images/image_exporter.go similarity index 60% rename from components/engine/daemon/image_exporter.go rename to components/engine/daemon/images/image_exporter.go index d746236af9..58105dcb71 100644 --- a/components/engine/daemon/image_exporter.go +++ b/components/engine/daemon/images/image_exporter.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -11,15 +11,15 @@ import ( // stream. All images with the given tag and all versions containing // the same tag are exported. names is the set of tags to export, and // outStream is the writer which the images are written to. -func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { - imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStores, daemon.referenceStore, daemon) +func (i *ImageService) ExportImage(names []string, outStream io.Writer) error { + imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Save(names, outStream) } // LoadImage uploads a set of images into the repository. This is the // complement of ImageExport. The input stream is an uncompressed tar // ball containing images and metadata. -func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { - imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStores, daemon.referenceStore, daemon) +func (i *ImageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { + imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Load(inTar, outStream, quiet) } diff --git a/components/engine/daemon/image_history.go b/components/engine/daemon/images/image_history.go similarity index 76% rename from components/engine/daemon/image_history.go rename to components/engine/daemon/images/image_history.go index 58dd59d3a7..2b92292631 100644 --- a/components/engine/daemon/image_history.go +++ b/components/engine/daemon/images/image_history.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "fmt" @@ -11,9 +11,9 @@ import ( // ImageHistory returns a slice of ImageHistory structures for the specified image // name by walking the image lineage. -func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { +func (i *ImageService) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { start := time.Now() - img, err := daemon.GetImage(name) + img, err := i.GetImage(name) if err != nil { return nil, err } @@ -33,12 +33,12 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e } rootFS.Append(img.RootFS.DiffIDs[layerCounter]) - l, err := daemon.layerStores[img.OperatingSystem()].Get(rootFS.ChainID()) + l, err := i.layerStores[img.OperatingSystem()].Get(rootFS.ChainID()) if err != nil { return nil, err } layerSize, err = l.DiffSize() - layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l) + layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e h.ID = id.String() var tags []string - for _, r := range daemon.referenceStore.References(id.Digest()) { + for _, r := range i.referenceStore.References(id.Digest()) { if _, ok := r.(reference.NamedTagged); ok { tags = append(tags, reference.FamiliarString(r)) } @@ -74,7 +74,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e if id == "" { break } - histImg, err = daemon.GetImage(id.String()) + histImg, err = i.GetImage(id.String()) if err != nil { break } diff --git a/components/engine/daemon/import.go b/components/engine/daemon/images/image_import.go similarity index 85% rename from components/engine/daemon/import.go rename to components/engine/daemon/images/image_import.go index 01b7906888..8d54e0704f 100644 --- a/components/engine/daemon/import.go +++ b/components/engine/daemon/images/image_import.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -27,7 +27,7 @@ import ( // inConfig (if src is "-"), or from a URI specified in src. Progress output is // written to outStream. Repository and tag names can optionally be given in // the repo and tag arguments, respectively. -func (daemon *Daemon) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { +func (i *ImageService) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { var ( rc io.ReadCloser resp *http.Response @@ -91,11 +91,11 @@ func (daemon *Daemon) ImportImage(src string, repository, os string, tag string, if err != nil { return err } - l, err := daemon.layerStores[os].Register(inflatedLayerData, "") + l, err := i.layerStores[os].Register(inflatedLayerData, "") if err != nil { return err } - defer layer.ReleaseAndLog(daemon.layerStores[os], l) + defer layer.ReleaseAndLog(i.layerStores[os], l) created := time.Now().UTC() imgConfig, err := json.Marshal(&image.Image{ @@ -120,19 +120,19 @@ func (daemon *Daemon) ImportImage(src string, repository, os string, tag string, return err } - id, err := daemon.imageStore.Create(imgConfig) + id, err := i.imageStore.Create(imgConfig) if err != nil { return err } // FIXME: connect with commit code and call refstore directly if newRef != nil { - if err := daemon.TagImageWithReference(id, newRef); err != nil { + if err := i.TagImageWithReference(id, newRef); err != nil { return err } } - daemon.LogImageEvent(id.String(), id.String(), "import") + i.LogImageEvent(id.String(), id.String(), "import") outStream.Write(streamformatter.FormatStatus("", id.String())) return nil } diff --git a/components/engine/daemon/image_inspect.go b/components/engine/daemon/images/image_inspect.go similarity index 81% rename from components/engine/daemon/image_inspect.go rename to components/engine/daemon/images/image_inspect.go index 731057cf34..16c4c9b2dc 100644 --- a/components/engine/daemon/image_inspect.go +++ b/components/engine/daemon/images/image_inspect.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "time" @@ -13,15 +13,15 @@ import ( // LookupImage looks up an image by name and returns it as an ImageInspect // structure. -func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { - img, err := daemon.GetImage(name) +func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) { + img, err := i.GetImage(name) if err != nil { return nil, errors.Wrapf(err, "no such image: %s", name) } if !system.IsOSSupported(img.OperatingSystem()) { return nil, system.ErrNotSupportedOperatingSystem } - refs := daemon.referenceStore.References(img.ID().Digest()) + refs := i.referenceStore.References(img.ID().Digest()) repoTags := []string{} repoDigests := []string{} for _, ref := range refs { @@ -37,11 +37,11 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { var layerMetadata map[string]string layerID := img.RootFS.ChainID() if layerID != "" { - l, err := daemon.layerStores[img.OperatingSystem()].Get(layerID) + l, err := i.layerStores[img.OperatingSystem()].Get(layerID) if err != nil { return nil, err } - defer layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l) + defer layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l) size, err = l.Size() if err != nil { return nil, err @@ -58,7 +58,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { comment = img.History[len(img.History)-1].Comment } - lastUpdated, err := daemon.imageStore.GetLastUpdated(img.ID()) + lastUpdated, err := i.imageStore.GetLastUpdated(img.ID()) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { }, } - imageInspect.GraphDriver.Name = daemon.GraphDriverName(img.OperatingSystem()) + imageInspect.GraphDriver.Name = i.layerStores[img.OperatingSystem()].DriverName() imageInspect.GraphDriver.Data = layerMetadata return imageInspect, nil diff --git a/components/engine/daemon/image_prune.go b/components/engine/daemon/images/image_prune.go similarity index 65% rename from components/engine/daemon/image_prune.go rename to components/engine/daemon/images/image_prune.go index 193c3aa0ac..9427536759 100644 --- a/components/engine/daemon/image_prune.go +++ b/components/engine/daemon/images/image_prune.go @@ -1,11 +1,14 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( + "fmt" "sync/atomic" + "time" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + timetypes "github.com/docker/docker/api/types/time" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -21,12 +24,16 @@ var imagesAcceptedFilters = map[string]bool{ "until": true, } +// errPruneRunning is returned when a prune request is received while +// one is in progress +var errPruneRunning = fmt.Errorf("a prune operation is already running") + // ImagesPrune removes unused images -func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { - if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) { +func (i *ImageService) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { + if !atomic.CompareAndSwapInt32(&i.pruneRunning, 0, 1) { return nil, errPruneRunning } - defer atomic.StoreInt32(&daemon.pruneRunning, 0) + defer atomic.StoreInt32(&i.pruneRunning, 0) // make sure that only accepted filters have been received err := pruneFilters.Validate(imagesAcceptedFilters) @@ -52,14 +59,14 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args var allImages map[image.ID]*image.Image if danglingOnly { - allImages = daemon.imageStore.Heads() + allImages = i.imageStore.Heads() } else { - allImages = daemon.imageStore.Map() + allImages = i.imageStore.Map() } // Filter intermediary images and get their unique size allLayers := make(map[layer.ChainID]layer.Layer) - for _, ls := range daemon.layerStores { + for _, ls := range i.layerStores { for k, v := range ls.Map() { allLayers[k] = v } @@ -71,7 +78,7 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args return nil, ctx.Err() default: dgst := digest.Digest(id) - if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { + if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { continue } if !until.IsZero() && img.Created.After(until) { @@ -96,7 +103,7 @@ deleteImagesLoop: } deletedImages := []types.ImageDeleteResponseItem{} - refs := daemon.referenceStore.References(id.Digest()) + refs := i.referenceStore.References(id.Digest()) if len(refs) > 0 { shouldDelete := !danglingOnly if !shouldDelete { @@ -114,7 +121,7 @@ deleteImagesLoop: if shouldDelete { for _, ref := range refs { - imgDel, err := daemon.ImageDelete(ref.String(), false, true) + imgDel, err := i.ImageDelete(ref.String(), false, true) if imageDeleteFailed(ref.String(), err) { continue } @@ -123,7 +130,7 @@ deleteImagesLoop: } } else { hex := id.Digest().Hex() - imgDel, err := daemon.ImageDelete(hex, false, true) + imgDel, err := i.ImageDelete(hex, false, true) if imageDeleteFailed(hex, err) { continue } @@ -166,3 +173,38 @@ func imageDeleteFailed(ref string, err error) bool { return true } } + +func matchLabels(pruneFilters filters.Args, labels map[string]string) bool { + if !pruneFilters.MatchKVList("label", labels) { + return false + } + // By default MatchKVList will return true if field (like 'label!') does not exist + // So we have to add additional Contains("label!") check + if pruneFilters.Contains("label!") { + if pruneFilters.MatchKVList("label!", labels) { + return false + } + } + return true +} + +func getUntilFromPruneFilters(pruneFilters filters.Args) (time.Time, error) { + until := time.Time{} + if !pruneFilters.Contains("until") { + return until, nil + } + untilFilters := pruneFilters.Get("until") + if len(untilFilters) > 1 { + return until, fmt.Errorf("more than one until filter specified") + } + ts, err := timetypes.GetTimestamp(untilFilters[0], time.Now()) + if err != nil { + return until, err + } + seconds, nanoseconds, err := timetypes.ParseTimestamps(ts, 0) + if err != nil { + return until, err + } + until = time.Unix(seconds, nanoseconds) + return until, nil +} diff --git a/components/engine/daemon/image_pull.go b/components/engine/daemon/images/image_pull.go similarity index 71% rename from components/engine/daemon/image_pull.go rename to components/engine/daemon/images/image_pull.go index cd3ef90b92..41056af678 100644 --- a/components/engine/daemon/image_pull.go +++ b/components/engine/daemon/images/image_pull.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -19,7 +19,7 @@ import ( // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (daemon *Daemon) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Special case: "pull -a" may send an image name with a // trailing :. This is ugly, but let's not break API // compatibility. @@ -44,10 +44,10 @@ func (daemon *Daemon) PullImage(ctx context.Context, image, tag, os string, meta } } - return daemon.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) + return i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) } -func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) @@ -71,13 +71,13 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference. MetaHeaders: metaHeaders, AuthConfig: authConfig, ProgressOutput: progress.ChanOutput(progressChan), - RegistryService: daemon.RegistryService, - ImageEventLogger: daemon.LogImageEvent, - MetadataStore: daemon.distributionMetadataStore, - ImageStore: distribution.NewImageConfigStoreFromStore(daemon.imageStore), - ReferenceStore: daemon.referenceStore, + RegistryService: i.registryService, + ImageEventLogger: i.LogImageEvent, + MetadataStore: i.distributionMetadataStore, + ImageStore: distribution.NewImageConfigStoreFromStore(i.imageStore), + ReferenceStore: i.referenceStore, }, - DownloadManager: daemon.downloadManager, + DownloadManager: i.downloadManager, Schema2Types: distribution.ImageTypes, OS: os, } @@ -89,9 +89,9 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference. } // GetRepository returns a repository from the registry. -func (daemon *Daemon) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { +func (i *ImageService) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { // get repository info - repoInfo, err := daemon.RegistryService.ResolveRepository(ref) + repoInfo, err := i.registryService.ResolveRepository(ref) if err != nil { return nil, false, err } @@ -101,7 +101,7 @@ func (daemon *Daemon) GetRepository(ctx context.Context, ref reference.Named, au } // get endpoints - endpoints, err := daemon.RegistryService.LookupPullEndpoints(reference.Domain(repoInfo.Name)) + endpoints, err := i.registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name)) if err != nil { return nil, false, err } diff --git a/components/engine/daemon/image_push.go b/components/engine/daemon/images/image_push.go similarity index 67% rename from components/engine/daemon/image_push.go rename to components/engine/daemon/images/image_push.go index b5bdb74fb6..af606a07b5 100644 --- a/components/engine/daemon/image_push.go +++ b/components/engine/daemon/images/image_push.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -13,7 +13,7 @@ import ( ) // PushImage initiates a push operation on the repository named localName. -func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { ref, err := reference.ParseNormalizedNamed(image) if err != nil { return err @@ -44,16 +44,16 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead MetaHeaders: metaHeaders, AuthConfig: authConfig, ProgressOutput: progress.ChanOutput(progressChan), - RegistryService: daemon.RegistryService, - ImageEventLogger: daemon.LogImageEvent, - MetadataStore: daemon.distributionMetadataStore, - ImageStore: distribution.NewImageConfigStoreFromStore(daemon.imageStore), - ReferenceStore: daemon.referenceStore, + RegistryService: i.registryService, + ImageEventLogger: i.LogImageEvent, + MetadataStore: i.distributionMetadataStore, + ImageStore: distribution.NewImageConfigStoreFromStore(i.imageStore), + ReferenceStore: i.referenceStore, }, ConfigMediaType: schema2.MediaTypeImageConfig, - LayerStores: distribution.NewLayerProvidersFromStores(daemon.layerStores), - TrustKey: daemon.trustKey, - UploadManager: daemon.uploadManager, + LayerStores: distribution.NewLayerProvidersFromStores(i.layerStores), + TrustKey: i.trustKey, + UploadManager: i.uploadManager, } err = distribution.Push(ctx, ref, imagePushConfig) diff --git a/components/engine/daemon/search.go b/components/engine/daemon/images/image_search.go similarity index 85% rename from components/engine/daemon/search.go rename to components/engine/daemon/images/image_search.go index 6d137d3d65..a901571439 100644 --- a/components/engine/daemon/search.go +++ b/components/engine/daemon/images/image_search.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "strconv" @@ -19,7 +19,10 @@ var acceptedSearchFilterTags = map[string]bool{ // SearchRegistryForImages queries the registry for images matching // term. authConfig is used to login. -func (daemon *Daemon) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, +// +// TODO: this could be implemented in a registry service instead of the image +// service. +func (i *ImageService) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { @@ -60,7 +63,7 @@ func (daemon *Daemon) SearchRegistryForImages(ctx context.Context, filtersArgs s } } - unfilteredResult, err := daemon.RegistryService.Search(ctx, term, limit, authConfig, dockerversion.DockerUserAgent(ctx), headers) + unfilteredResult, err := i.registryService.Search(ctx, term, limit, authConfig, dockerversion.DockerUserAgent(ctx), headers) if err != nil { return nil, err } diff --git a/components/engine/daemon/search_test.go b/components/engine/daemon/images/image_search_test.go similarity index 97% rename from components/engine/daemon/search_test.go rename to components/engine/daemon/images/image_search_test.go index 0237b103b8..74795c5865 100644 --- a/components/engine/daemon/search_test.go +++ b/components/engine/daemon/images/image_search_test.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "errors" @@ -76,8 +76,8 @@ func TestSearchRegistryForImagesErrors(t *testing.T) { }, } for index, e := range errorCases { - daemon := &Daemon{ - RegistryService: &FakeService{ + daemon := &ImageService{ + registryService: &FakeService{ shouldReturnError: e.shouldReturnError, }, } @@ -322,8 +322,8 @@ func TestSearchRegistryForImages(t *testing.T) { }, } for index, s := range successCases { - daemon := &Daemon{ - RegistryService: &FakeService{ + daemon := &ImageService{ + registryService: &FakeService{ term: term, results: s.registryResults, }, diff --git a/components/engine/daemon/image_tag.go b/components/engine/daemon/images/image_tag.go similarity index 53% rename from components/engine/daemon/image_tag.go rename to components/engine/daemon/images/image_tag.go index 56b325f66d..4693611c3a 100644 --- a/components/engine/daemon/image_tag.go +++ b/components/engine/daemon/images/image_tag.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "github.com/docker/distribution/reference" @@ -7,8 +7,8 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). -func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error) { - imageID, _, err := daemon.GetImageIDAndOS(imageName) +func (i *ImageService) TagImage(imageName, repository, tag string) (string, error) { + img, err := i.GetImage(imageName) if err != nil { return "", err } @@ -23,19 +23,19 @@ func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error } } - err = daemon.TagImageWithReference(imageID, newTag) + err = i.TagImageWithReference(img.ID(), newTag) return reference.FamiliarString(newTag), err } // TagImageWithReference adds the given reference to the image ID provided. -func (daemon *Daemon) TagImageWithReference(imageID image.ID, newTag reference.Named) error { - if err := daemon.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { +func (i *ImageService) TagImageWithReference(imageID image.ID, newTag reference.Named) error { + if err := i.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { return err } - if err := daemon.imageStore.SetLastUpdated(imageID); err != nil { + if err := i.imageStore.SetLastUpdated(imageID); err != nil { return err } - daemon.LogImageEvent(imageID.String(), reference.FamiliarString(newTag), "tag") + i.LogImageEvent(imageID.String(), reference.FamiliarString(newTag), "tag") return nil } diff --git a/components/engine/daemon/getsize_unix.go b/components/engine/daemon/images/image_unix.go similarity index 66% rename from components/engine/daemon/getsize_unix.go rename to components/engine/daemon/images/image_unix.go index 292108307c..3f577271a2 100644 --- a/components/engine/daemon/getsize_unix.go +++ b/components/engine/daemon/images/image_unix.go @@ -1,6 +1,6 @@ // +build linux freebsd -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "runtime" @@ -8,8 +8,8 @@ import ( "github.com/sirupsen/logrus" ) -// getSize returns the real size & virtual size of the container. -func (daemon *Daemon) getSize(containerID string) (int64, int64) { +// GetContainerLayerSize returns the real size & virtual size of the container. +func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64) { var ( sizeRw, sizeRootfs int64 err error @@ -17,17 +17,17 @@ func (daemon *Daemon) getSize(containerID string) (int64, int64) { // Safe to index by runtime.GOOS as Unix hosts don't support multiple // container operating systems. - rwlayer, err := daemon.layerStores[runtime.GOOS].GetRWLayer(containerID) + rwlayer, err := i.layerStores[runtime.GOOS].GetRWLayer(containerID) if err != nil { logrus.Errorf("Failed to compute size of container rootfs %v: %v", containerID, err) return sizeRw, sizeRootfs } - defer daemon.layerStores[runtime.GOOS].ReleaseRWLayer(rwlayer) + defer i.layerStores[runtime.GOOS].ReleaseRWLayer(rwlayer) sizeRw, err = rwlayer.Size() if err != nil { logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", - daemon.GraphDriverName(runtime.GOOS), containerID, err) + i.layerStores[runtime.GOOS].DriverName(), containerID, err) // FIXME: GetSize should return an error. Not changing it now in case // there is a side-effect. sizeRw = -1 diff --git a/components/engine/daemon/image_windows.go b/components/engine/daemon/images/image_windows.go similarity index 67% rename from components/engine/daemon/image_windows.go rename to components/engine/daemon/images/image_windows.go index d661400a38..6f4be49736 100644 --- a/components/engine/daemon/image_windows.go +++ b/components/engine/daemon/images/image_windows.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images import ( "github.com/docker/docker/image" @@ -7,8 +7,14 @@ import ( "github.com/pkg/errors" ) +// GetContainerLayerSize returns real size & virtual size +func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64) { + // TODO Windows + return 0, 0 +} + // GetLayerFolders returns the layer folders from an image RootFS -func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { +func (i *ImageService) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { folders := []string{} max := len(img.RootFS.DiffIDs) for index := 1; index <= max; index++ { @@ -17,9 +23,9 @@ func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ( if !system.IsOSSupported(img.OperatingSystem()) { return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID()) } - layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + layerPath, err := layer.GetLayerPath(i.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) if err != nil { - return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", i.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) } // Reverse order, expecting parent first folders = append([]string{layerPath}, folders...) diff --git a/components/engine/daemon/images.go b/components/engine/daemon/images/images.go similarity index 86% rename from components/engine/daemon/images.go rename to components/engine/daemon/images/images.go index 7e2a9418c7..46056f15b5 100644 --- a/components/engine/daemon/images.go +++ b/components/engine/daemon/images/images.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -34,8 +34,8 @@ func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] } func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } // Map returns a map of all images in the ImageStore -func (daemon *Daemon) Map() map[image.ID]*image.Image { - return daemon.imageStore.Map() +func (i *ImageService) Map() map[image.ID]*image.Image { + return i.imageStore.Map() } // Images returns a filtered list of images. filterArgs is a JSON-encoded set @@ -43,7 +43,7 @@ func (daemon *Daemon) Map() map[image.ID]*image.Image { // filter is a shell glob string applied to repository names. The argument // named all controls whether all images in the graph are filtered, or just // the heads. -func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { +func (i *ImageService) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { var ( allImages map[image.ID]*image.Image err error @@ -62,14 +62,14 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } } if danglingOnly { - allImages = daemon.imageStore.Heads() + allImages = i.imageStore.Heads() } else { - allImages = daemon.imageStore.Map() + allImages = i.imageStore.Map() } var beforeFilter, sinceFilter *image.Image err = imageFilters.WalkValues("before", func(value string) error { - beforeFilter, err = daemon.GetImage(value) + beforeFilter, err = i.GetImage(value) return err }) if err != nil { @@ -77,7 +77,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } err = imageFilters.WalkValues("since", func(value string) error { - sinceFilter, err = daemon.GetImage(value) + sinceFilter, err = i.GetImage(value) return err }) if err != nil { @@ -124,7 +124,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs layerID := img.RootFS.ChainID() var size int64 if layerID != "" { - l, err := daemon.layerStores[img.OperatingSystem()].Get(layerID) + l, err := i.layerStores[img.OperatingSystem()].Get(layerID) if err != nil { // The layer may have been deleted between the call to `Map()` or // `Heads()` and the call to `Get()`, so we just ignore this error @@ -135,7 +135,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } size, err = l.Size() - layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l) + layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l) if err != nil { return nil, err } @@ -143,7 +143,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs newImage := newImage(img, size) - for _, ref := range daemon.referenceStore.References(id.Digest()) { + for _, ref := range i.referenceStore.References(id.Digest()) { if imageFilters.Contains("reference") { var found bool var matchErr error @@ -165,7 +165,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } } if newImage.RepoDigests == nil && newImage.RepoTags == nil { - if all || len(daemon.imageStore.Children(id)) == 0 { + if all || len(i.imageStore.Children(id)) == 0 { if imageFilters.Contains("dangling") && !danglingOnly { //dangling=false case, so dangling image is not needed @@ -186,8 +186,8 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs if withExtraAttrs { // lazily init variables if imagesMap == nil { - allContainers = daemon.List() - allLayers = daemon.layerStores[img.OperatingSystem()].Map() + allContainers = i.containers.List() + allLayers = i.layerStores[img.OperatingSystem()].Map() imagesMap = make(map[*image.Image]*types.ImageSummary) layerRefs = make(map[layer.ChainID]int) } @@ -249,20 +249,20 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs // This new image contains only the layers from it's parent + 1 extra layer which contains the diff of all the layers in between. // The existing image(s) is not destroyed. // If no parent is specified, a new image with the diff of all the specified image's layers merged into a new layer that has no parents. -func (daemon *Daemon) SquashImage(id, parent string) (string, error) { +func (i *ImageService) SquashImage(id, parent string) (string, error) { var ( img *image.Image err error ) - if img, err = daemon.imageStore.Get(image.ID(id)); err != nil { + if img, err = i.imageStore.Get(image.ID(id)); err != nil { return "", err } var parentImg *image.Image var parentChainID layer.ChainID if len(parent) != 0 { - parentImg, err = daemon.imageStore.Get(image.ID(parent)) + parentImg, err = i.imageStore.Get(image.ID(parent)) if err != nil { return "", errors.Wrap(err, "error getting specified parent layer") } @@ -272,11 +272,11 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) { parentImg = &image.Image{RootFS: rootFS} } - l, err := daemon.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID()) + l, err := i.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID()) if err != nil { return "", errors.Wrap(err, "error getting image layer") } - defer daemon.layerStores[img.OperatingSystem()].Release(l) + defer i.layerStores[img.OperatingSystem()].Release(l) ts, err := l.TarStreamFrom(parentChainID) if err != nil { @@ -284,11 +284,11 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) { } defer ts.Close() - newL, err := daemon.layerStores[img.OperatingSystem()].Register(ts, parentChainID) + newL, err := i.layerStores[img.OperatingSystem()].Register(ts, parentChainID) if err != nil { return "", errors.Wrap(err, "error registering layer") } - defer daemon.layerStores[img.OperatingSystem()].Release(newL) + defer i.layerStores[img.OperatingSystem()].Release(newL) newImage := *img newImage.RootFS = nil @@ -323,7 +323,7 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) { return "", errors.Wrap(err, "error marshalling image config") } - newImgID, err := daemon.imageStore.Create(b) + newImgID, err := i.imageStore.Create(b) if err != nil { return "", errors.Wrap(err, "error creating new image after squash") } diff --git a/components/engine/daemon/images/locals.go b/components/engine/daemon/images/locals.go new file mode 100644 index 0000000000..a57ea2da60 --- /dev/null +++ b/components/engine/daemon/images/locals.go @@ -0,0 +1,32 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "fmt" + + metrics "github.com/docker/go-metrics" +) + +type invalidFilter struct { + filter string + value interface{} +} + +func (e invalidFilter) Error() string { + msg := "Invalid filter '" + e.filter + if e.value != nil { + msg += fmt.Sprintf("=%s", e.value) + } + return msg + "'" +} + +func (e invalidFilter) InvalidParameter() {} + +var imageActions metrics.LabeledTimer + +func init() { + ns := metrics.NewNamespace("engine", "daemon", nil) + imageActions = ns.NewLabeledTimer("image_actions", "The number of seconds it takes to process each image action", "action") + // TODO: is it OK to register a namespace with the same name? Or does this + // need to be exported from somewhere? + metrics.Register(ns) +} diff --git a/components/engine/daemon/images/service.go b/components/engine/daemon/images/service.go new file mode 100644 index 0000000000..70a8bf4455 --- /dev/null +++ b/components/engine/daemon/images/service.go @@ -0,0 +1,229 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "context" + "os" + + "github.com/docker/docker/container" + daemonevents "github.com/docker/docker/daemon/events" + "github.com/docker/docker/distribution/metadata" + "github.com/docker/docker/distribution/xfer" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + dockerreference "github.com/docker/docker/reference" + "github.com/docker/docker/registry" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type containerStore interface { + // used by image delete + First(container.StoreFilter) *container.Container + // used by image prune, and image list + List() []*container.Container + // TODO: remove, only used for CommitBuildStep + Get(string) *container.Container +} + +// ImageServiceConfig is the configuration used to create a new ImageService +type ImageServiceConfig struct { + ContainerStore containerStore + DistributionMetadataStore metadata.Store + EventsService *daemonevents.Events + ImageStore image.Store + LayerStores map[string]layer.Store + MaxConcurrentDownloads int + MaxConcurrentUploads int + ReferenceStore dockerreference.Store + RegistryService registry.Service + TrustKey libtrust.PrivateKey +} + +// NewImageService returns a new ImageService from a configuration +func NewImageService(config ImageServiceConfig) *ImageService { + logrus.Debugf("Max Concurrent Downloads: %d", config.MaxConcurrentDownloads) + logrus.Debugf("Max Concurrent Uploads: %d", config.MaxConcurrentUploads) + return &ImageService{ + containers: config.ContainerStore, + distributionMetadataStore: config.DistributionMetadataStore, + downloadManager: xfer.NewLayerDownloadManager(config.LayerStores, config.MaxConcurrentDownloads), + eventsService: config.EventsService, + imageStore: config.ImageStore, + layerStores: config.LayerStores, + referenceStore: config.ReferenceStore, + registryService: config.RegistryService, + trustKey: config.TrustKey, + uploadManager: xfer.NewLayerUploadManager(config.MaxConcurrentUploads), + } +} + +// ImageService provides a backend for image management +type ImageService struct { + containers containerStore + distributionMetadataStore metadata.Store + downloadManager *xfer.LayerDownloadManager + eventsService *daemonevents.Events + imageStore image.Store + layerStores map[string]layer.Store // By operating system + pruneRunning int32 + referenceStore dockerreference.Store + registryService registry.Service + trustKey libtrust.PrivateKey + uploadManager *xfer.LayerUploadManager +} + +// CountImages returns the number of images stored by ImageService +// called from info.go +func (i *ImageService) CountImages() int { + return len(i.imageStore.Map()) +} + +// Children returns the children image.IDs for a parent image. +// called from list.go to filter containers +// TODO: refactor to expose an ancestry for image.ID? +func (i *ImageService) Children(id image.ID) []image.ID { + return i.imageStore.Children(id) +} + +// CreateLayer creates a filesystem layer for a container. +// called from create.go +// TODO: accept an opt struct instead of container? +func (i *ImageService) CreateLayer(container *container.Container, initFunc layer.MountInit) (layer.RWLayer, error) { + var layerID layer.ChainID + if container.ImageID != "" { + img, err := i.imageStore.Get(container.ImageID) + if err != nil { + return nil, err + } + layerID = img.RootFS.ChainID() + } + + rwLayerOpts := &layer.CreateRWLayerOpts{ + MountLabel: container.MountLabel, + InitFunc: initFunc, + StorageOpt: container.HostConfig.StorageOpt, + } + + // Indexing by OS is safe here as validation of OS has already been performed in create() (the only + // caller), and guaranteed non-nil + return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) +} + +// GetLayerByID returns a layer by ID and operating system +// called from daemon.go Daemon.restore(), and Daemon.containerExport() +func (i *ImageService) GetLayerByID(cid string, os string) (layer.RWLayer, error) { + return i.layerStores[os].GetRWLayer(cid) +} + +// LayerStoreStatus returns the status for each layer store +// called from info.go +func (i *ImageService) LayerStoreStatus() map[string][][2]string { + result := make(map[string][][2]string) + for os, store := range i.layerStores { + result[os] = store.DriverStatus() + } + return result +} + +// GetLayerMountID returns the mount ID for a layer +// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup) +// TODO: needs to be refactored to Unmount (see callers), or removed and replaced +// with GetLayerByID +func (i *ImageService) GetLayerMountID(cid string, os string) (string, error) { + return i.layerStores[os].GetMountID(cid) +} + +// Cleanup resources before the process is shutdown. +// called from daemon.go Daemon.Shutdown() +func (i *ImageService) Cleanup() { + for os, ls := range i.layerStores { + if ls != nil { + if err := ls.Cleanup(); err != nil { + logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) + } + } + } +} + +// GraphDriverForOS returns the name of the graph drvier +// moved from Daemon.GraphDriverName, used by: +// - newContainer +// - to report an error in Daemon.Mount(container) +func (i *ImageService) GraphDriverForOS(os string) string { + return i.layerStores[os].DriverName() +} + +// ReleaseLayer releases a layer allowing it to be removed +// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport() +func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer, containerOS string) error { + metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer) + layer.LogReleaseMetadata(metadata) + if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { + return errors.Wrapf(err, "driver %q failed to remove root filesystem", + i.layerStores[containerOS].DriverName()) + } + return nil +} + +// LayerDiskUsage returns the number of bytes used by layer stores +// called from disk_usage.go +func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) { + var allLayersSize int64 + layerRefs := i.getLayerRefs() + for _, ls := range i.layerStores { + allLayers := ls.Map() + for _, l := range allLayers { + select { + case <-ctx.Done(): + return allLayersSize, ctx.Err() + default: + size, err := l.DiffSize() + if err == nil { + if _, ok := layerRefs[l.ChainID()]; ok { + allLayersSize += size + } else { + logrus.Warnf("found leaked image layer %v", l.ChainID()) + } + } else { + logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) + } + } + } + } + return allLayersSize, nil +} + +func (i *ImageService) getLayerRefs() map[layer.ChainID]int { + tmpImages := i.imageStore.Map() + layerRefs := map[layer.ChainID]int{} + for id, img := range tmpImages { + dgst := digest.Digest(id) + if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { + continue + } + + rootFS := *img.RootFS + rootFS.DiffIDs = nil + for _, id := range img.RootFS.DiffIDs { + rootFS.Append(id) + chid := rootFS.ChainID() + layerRefs[chid]++ + } + } + + return layerRefs +} + +// UpdateConfig values +// +// called from reload.go +func (i *ImageService) UpdateConfig(maxDownloads, maxUploads *int) { + if i.downloadManager != nil && maxDownloads != nil { + i.downloadManager.SetConcurrency(*maxDownloads) + } + if i.uploadManager != nil && maxUploads != nil { + i.uploadManager.SetConcurrency(*maxUploads) + } +} diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index c73734ded7..6f5ae69896 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -80,8 +80,9 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { var ds [][2]string drivers := "" + statuses := daemon.imageService.LayerStoreStatus() for os, gd := range daemon.graphDrivers { - ds = append(ds, daemon.layerStores[os].DriverStatus()...) + ds = append(ds, statuses[os]...) drivers += gd if len(daemon.graphDrivers) > 1 { drivers += fmt.Sprintf(" (%s) ", os) @@ -95,7 +96,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { ContainersRunning: cRunning, ContainersPaused: cPaused, ContainersStopped: cStopped, - Images: len(daemon.imageStore.Map()), + Images: daemon.imageService.CountImages(), Driver: drivers, DriverStatus: ds, Plugins: daemon.showPluginsInfo(), diff --git a/components/engine/daemon/inspect.go b/components/engine/daemon/inspect.go index 164e1aa2ae..f8dceb71ab 100644 --- a/components/engine/daemon/inspect.go +++ b/components/engine/daemon/inspect.go @@ -79,7 +79,7 @@ func (daemon *Daemon) ContainerInspectCurrent(name string, size bool) (*types.Co container.Unlock() if size { - sizeRw, sizeRootFs := daemon.getSize(base.ID) + sizeRw, sizeRootFs := daemon.imageService.GetContainerLayerSize(base.ID) base.SizeRw = &sizeRw base.SizeRootFs = &sizeRootFs } diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index e9c50071a0..84cacc344a 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/volume" @@ -239,7 +240,7 @@ func (daemon *Daemon) reducePsContainer(container *container.Snapshot, ctx *list // release lock because size calculation is slow if ctx.Size { - sizeRw, sizeRootFs := daemon.getSize(newC.ID) + sizeRw, sizeRootFs := daemon.imageService.GetContainerLayerSize(newC.ID) newC.SizeRw = sizeRw newC.SizeRootFs = sizeRootFs } @@ -323,17 +324,17 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis if psFilters.Contains("ancestor") { ancestorFilter = true psFilters.WalkValues("ancestor", func(ancestor string) error { - id, _, err := daemon.GetImageIDAndOS(ancestor) + img, err := daemon.imageService.GetImage(ancestor) if err != nil { logrus.Warnf("Error while looking up for image %v", ancestor) return nil } - if imagesFilter[id] { + if imagesFilter[img.ID()] { // Already seen this ancestor, skip it return nil } // Then walk down the graph and put the imageIds in imagesFilter - populateImageFilterByParents(imagesFilter, id, daemon.imageStore.Children) + populateImageFilterByParents(imagesFilter, img.ID(), daemon.imageService.Children) return nil }) } @@ -591,11 +592,11 @@ func (daemon *Daemon) refreshImage(s *container.Snapshot, ctx *listContext) (*ty c := s.Container image := s.Image // keep the original ref if still valid (hasn't changed) if image != s.ImageID { - id, _, err := daemon.GetImageIDAndOS(image) - if _, isDNE := err.(errImageDoesNotExist); err != nil && !isDNE { + img, err := daemon.imageService.GetImage(image) + if _, isDNE := err.(images.ErrImageDoesNotExist); err != nil && !isDNE { return nil, err } - if err != nil || id.String() != s.ImageID { + if err != nil || img.ImageID() != s.ImageID { // ref changed, we need to use original ID image = s.ImageID } diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 92484697d5..7d637f8573 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -150,11 +150,6 @@ func ValidateLogOpt(cfg map[string]string) error { return nil } -// LogPath returns the location the given json logger logs to. -func (l *JSONFileLogger) LogPath() string { - return l.writer.LogPath() -} - // Close closes underlying file and signals all readers to stop. func (l *JSONFileLogger) Close() error { l.mu.Lock() diff --git a/components/engine/daemon/logger/loggerutils/logfile.go b/components/engine/daemon/logger/loggerutils/logfile.go index a9d9d633db..b533726c42 100644 --- a/components/engine/daemon/logger/loggerutils/logfile.go +++ b/components/engine/daemon/logger/loggerutils/logfile.go @@ -130,13 +130,6 @@ func rotate(name string, maxFiles int) error { return nil } -// LogPath returns the location the given writer logs to. -func (w *LogFile) LogPath() string { - w.mu.Lock() - defer w.mu.Unlock() - return w.f.Name() -} - // MaxFiles return maximum number of files func (w *LogFile) MaxFiles() int { return w.maxFiles diff --git a/components/engine/daemon/metrics.go b/components/engine/daemon/metrics.go index b283526d9a..02a36603c9 100644 --- a/components/engine/daemon/metrics.go +++ b/components/engine/daemon/metrics.go @@ -14,7 +14,6 @@ const metricsPluginType = "MetricsCollector" var ( containerActions metrics.LabeledTimer - imageActions metrics.LabeledTimer networkActions metrics.LabeledTimer engineInfo metrics.LabeledGauge engineCpus metrics.Gauge @@ -52,7 +51,6 @@ func init() { engineMemory = ns.NewGauge("engine_memory", "The number of bytes of memory that the host system of the engine has", metrics.Bytes) healthChecksCounter = ns.NewCounter("health_checks", "The total number of health checks") healthChecksFailedCounter = ns.NewCounter("health_checks_failed", "The total number of failed health checks") - imageActions = ns.NewLabeledTimer("image_actions", "The number of seconds it takes to process each image action", "action") stateCtr = newStateCounter(ns.NewDesc("container_states", "The count of containers in various states", metrics.Unit("containers"), "state")) ns.Add(stateCtr) diff --git a/components/engine/daemon/monitor.go b/components/engine/daemon/monitor.go index ce897d9397..5e740dd4fe 100644 --- a/components/engine/daemon/monitor.go +++ b/components/engine/daemon/monitor.go @@ -111,10 +111,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc } daemon.setStateCounter(c) - if err := c.CheckpointTo(daemon.containersReplica); err != nil { - return err - } - return daemon.postRunProcessing(c, ei) + return c.CheckpointTo(daemon.containersReplica) } if execConfig := c.ExecCommands.Get(ei.ProcessID); execConfig != nil { diff --git a/components/engine/daemon/monitor_linux.go b/components/engine/daemon/monitor_linux.go deleted file mode 100644 index cb2aa72dbb..0000000000 --- a/components/engine/daemon/monitor_linux.go +++ /dev/null @@ -1,11 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "github.com/docker/docker/container" - "github.com/docker/docker/libcontainerd" -) - -// postRunProcessing perfoms any processing needed on the container after it has stopped. -func (daemon *Daemon) postRunProcessing(_ *container.Container, _ libcontainerd.EventInfo) error { - return nil -} diff --git a/components/engine/daemon/monitor_windows.go b/components/engine/daemon/monitor_windows.go deleted file mode 100644 index 5db81347c6..0000000000 --- a/components/engine/daemon/monitor_windows.go +++ /dev/null @@ -1,53 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "context" - - "github.com/docker/docker/container" - "github.com/docker/docker/libcontainerd" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -// postRunProcessing starts a servicing container if required -func (daemon *Daemon) postRunProcessing(c *container.Container, ei libcontainerd.EventInfo) error { - if ei.ExitCode == 0 && ei.UpdatePending { - spec, err := daemon.createSpec(c) - if err != nil { - return err - } - // Turn on servicing - spec.Windows.Servicing = true - - copts, err := daemon.getLibcontainerdCreateOptions(c) - if err != nil { - return err - } - - // Create a new servicing container, which will start, complete the - // update, and merge back the results if it succeeded, all as part of - // the below function call. - ctx := context.Background() - svcID := c.ID + "_servicing" - logger := logrus.WithField("container", svcID) - if err := daemon.containerd.Create(ctx, svcID, spec, copts); err != nil { - c.SetExitCode(-1) - return errors.Wrap(err, "post-run update servicing failed") - } - _, err = daemon.containerd.Start(ctx, svcID, "", false, nil) - if err != nil { - logger.WithError(err).Warn("failed to run servicing container") - if err := daemon.containerd.Delete(ctx, svcID); err != nil { - logger.WithError(err).Warn("failed to delete servicing container") - } - } else { - if _, _, err := daemon.containerd.DeleteTask(ctx, svcID); err != nil { - logger.WithError(err).Warn("failed to delete servicing container task") - } - if err := daemon.containerd.Delete(ctx, svcID); err != nil { - logger.WithError(err).Warn("failed to delete servicing container") - } - } - } - return nil -} diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index e4b536f4e4..e2e10f9999 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -24,7 +24,7 @@ const ( ) func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { - img, err := daemon.GetImage(string(c.ImageID)) + img, err := daemon.imageService.GetImage(string(c.ImageID)) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } } s.Process.User.Username = c.Config.User - s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer) + s.Windows.LayerFolders, err = daemon.imageService.GetLayerFolders(img, c.RWLayer) if err != nil { return nil, errors.Wrapf(err, "container %s", c.ID) } @@ -326,9 +326,6 @@ func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.S s.Windows.CredentialSpec = cs } - // Assume we are not starting a container for a servicing operation - s.Windows.Servicing = false - return nil } diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index f16e448ba8..9f42e49a2b 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -75,7 +75,7 @@ func (daemon *Daemon) ContainersPrune(ctx context.Context, pruneFilters filters. if !matchLabels(pruneFilters, c.Config.Labels) { continue } - cSize, _ := daemon.getSize(c.ID) + cSize, _ := daemon.imageService.GetContainerLayerSize(c.ID) // TODO: sets RmLink to true? err := daemon.ContainerRm(c.ID, &types.ContainerRmConfig{}) if err != nil { diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 2ded7d5bd9..210864ff87 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -90,12 +90,6 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentDownloads = &maxConcurrentDownloads } logrus.Debugf("Reset Max Concurrent Downloads: %d", *daemon.configStore.MaxConcurrentDownloads) - if daemon.downloadManager != nil { - daemon.downloadManager.SetConcurrency(*daemon.configStore.MaxConcurrentDownloads) - } - - // prepare reload event attributes with updatable configurations - attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads) // If no value is set for max-concurrent-upload we assume it is the default value // We always "reset" as the cost is lightweight and easy to maintain. @@ -106,10 +100,10 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentUploads = &maxConcurrentUploads } logrus.Debugf("Reset Max Concurrent Uploads: %d", *daemon.configStore.MaxConcurrentUploads) - if daemon.uploadManager != nil { - daemon.uploadManager.SetConcurrency(*daemon.configStore.MaxConcurrentUploads) - } + daemon.imageService.UpdateConfig(conf.MaxConcurrentDownloads, conf.MaxConcurrentUploads) + // prepare reload event attributes with updatable configurations + attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads) // prepare reload event attributes with updatable configurations attributes["max-concurrent-uploads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentUploads) } diff --git a/components/engine/daemon/reload_test.go b/components/engine/daemon/reload_test.go index 83e86fc5a0..a2500b2bb2 100644 --- a/components/engine/daemon/reload_test.go +++ b/components/engine/daemon/reload_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/docker/docker/daemon/config" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/pkg/discovery" _ "github.com/docker/docker/pkg/discovery/memory" "github.com/docker/docker/registry" @@ -15,11 +16,13 @@ import ( ) func TestDaemonReloadLabels(t *testing.T) { - daemon := &Daemon{} - daemon.configStore = &config.Config{ - CommonConfig: config.CommonConfig{ - Labels: []string{"foo:bar"}, + daemon := &Daemon{ + configStore: &config.Config{ + CommonConfig: config.CommonConfig{ + Labels: []string{"foo:bar"}, + }, }, + imageService: images.NewImageService(images.ImageServiceConfig{}), } valuesSets := make(map[string]interface{}) @@ -43,7 +46,8 @@ func TestDaemonReloadLabels(t *testing.T) { func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { daemon := &Daemon{ - configStore: &config.Config{}, + configStore: &config.Config{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error @@ -97,7 +101,9 @@ func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { } func TestDaemonReloadMirrors(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } var err error daemon.RegistryService, err = registry.NewService(registry.ServiceOptions{ InsecureRegistries: []string{}, @@ -194,7 +200,9 @@ func TestDaemonReloadMirrors(t *testing.T) { } func TestDaemonReloadInsecureRegistries(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } var err error // initialize daemon with existing insecure registries: "127.0.0.0/8", "10.10.1.11:5000", "10.10.1.22:5000" daemon.RegistryService, err = registry.NewService(registry.ServiceOptions{ @@ -284,7 +292,9 @@ func TestDaemonReloadInsecureRegistries(t *testing.T) { } func TestDaemonReloadNotAffectOthers(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ Labels: []string{"foo:bar"}, @@ -316,7 +326,9 @@ func TestDaemonReloadNotAffectOthers(t *testing.T) { } func TestDaemonDiscoveryReload(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ ClusterStore: "memory://127.0.0.1", @@ -393,7 +405,9 @@ func TestDaemonDiscoveryReload(t *testing.T) { } func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } daemon.configStore = &config.Config{} valuesSet := make(map[string]interface{}) @@ -438,7 +452,9 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { } func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ ClusterStore: "memory://127.0.0.1", @@ -482,7 +498,9 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { } func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: images.NewImageService(images.ImageServiceConfig{}), + } daemon.configStore = &config.Config{} valuesSet := make(map[string]interface{}) diff --git a/components/engine/daemon/start.go b/components/engine/daemon/start.go index 2750df20b8..c00bd9ceb2 100644 --- a/components/engine/daemon/start.go +++ b/components/engine/daemon/start.go @@ -223,7 +223,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) { if err := daemon.conditionalUnmountOnCleanup(container); err != nil { // FIXME: remove once reference counting for graphdrivers has been refactored // Ensure that all the mounts are gone - if mountid, err := daemon.layerStores[container.OS].GetMountID(container.ID); err == nil { + if mountid, err := daemon.imageService.GetLayerMountID(container.ID, container.OS); err == nil { daemon.cleanupMountsByID(mountid) } } diff --git a/components/engine/hack/test/e2e-run.sh b/components/engine/hack/test/e2e-run.sh index 448f120895..b80f7fc312 100755 --- a/components/engine/hack/test/e2e-run.sh +++ b/components/engine/hack/test/e2e-run.sh @@ -1,41 +1,71 @@ #!/usr/bin/env bash -set -e +set -e -u -o pipefail -TESTFLAGS=${TESTFLAGS:-""} -# Currently only DockerSuite and DockerNetworkSuite have been adapted for E2E testing -TESTFLAGS_LEGACY=${TESTFLAGS_LEGACY:-""} -TIMEOUT=${TIMEOUT:-60m} +ARCH=$(uname -m) +if [ "$ARCH" == "x86_64" ]; then + ARCH="amd64" +fi -SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" +export DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:-${ARCH}} -export DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:-amd64} +# Set defaults +: ${TESTFLAGS:=} +: ${TESTDEBUG:=} + +integration_api_dirs=${TEST_INTEGRATION_DIR:-"$( + find ./integration -type d | + grep -vE '(^./integration($|/internal)|/testdata)')"} run_test_integration() { - run_test_integration_suites - run_test_integration_legacy_suites + [[ "$TESTFLAGS" != *-check.f* ]] && run_test_integration_suites + run_test_integration_legacy_suites } run_test_integration_suites() { - local flags="-test.timeout=${TIMEOUT} $TESTFLAGS" - for dir in /tests/integration/*; do - if ! ( - cd $dir - echo "Running $PWD" - ./test.main $flags - ); then exit 1; fi - done + local flags="-test.v -test.timeout=${TIMEOUT:=10m} $TESTFLAGS" + for dir in $integration_api_dirs; do + if ! ( + cd $dir + echo "Running $PWD" + test_env ./test.main $flags + ); then exit 1; fi + done } run_test_integration_legacy_suites() { - ( - flags="-check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS_LEGACY" - cd /tests/integration-cli - echo "Running $PWD" - ./test.main $flags - ) + ( + flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS" + cd test/integration-cli + echo "Running $PWD" + test_env ./test.main $flags + ) } -bash $SCRIPTDIR/ensure-emptyfs.sh +# use "env -i" to tightly control the environment variables that bleed into the tests +test_env() { + ( + set -e +u + [ -n "$TESTDEBUG" ] && set -x + env -i \ + DOCKER_API_VERSION="$DOCKER_API_VERSION" \ + DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \ + DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \ + DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \ + DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \ + DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \ + DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \ + DOCKER_HOST="$DOCKER_HOST" \ + DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \ + DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \ + DOCKERFILE="$DOCKERFILE" \ + GOPATH="$GOPATH" \ + GOTRACEBACK=all \ + HOME="$ABS_DEST/fake-HOME" \ + PATH="$PATH" \ + TEMP="$TEMP" \ + TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \ + "$@" + ) +} -echo "Run integration tests" run_test_integration diff --git a/components/engine/integration-cli/docker_api_swarm_test.go b/components/engine/integration-cli/docker_api_swarm_test.go index fd0b4e6278..b5edfd2c04 100644 --- a/components/engine/integration-cli/docker_api_swarm_test.go +++ b/components/engine/integration-cli/docker_api_swarm_test.go @@ -910,6 +910,8 @@ func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) { // Test case for 30178 func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) { + // Issue #36386 can be a independent one, which is worth further investigation. + c.Skip("Root cause of Issue #36386 is needed") d := s.AddDaemon(c, true, true) out, err := d.Cmd("network", "create", "-d", "overlay", "lb") diff --git a/components/engine/integration-cli/docker_api_volumes_test.go b/components/engine/integration-cli/docker_api_volumes_test.go deleted file mode 100644 index 65a9652092..0000000000 --- a/components/engine/integration-cli/docker_api_volumes_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/docker/docker/api/types/filters" - volumetypes "github.com/docker/docker/api/types/volume" - "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/checker" - "github.com/go-check/check" - "golang.org/x/net/context" -) - -func (s *DockerSuite) TestVolumesAPIList(c *check.C) { - prefix, _ := getPrefixAndSlashFromDaemonPlatform() - cid, _ := dockerCmd(c, "run", "-d", "-v", prefix+"/foo", "busybox") - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - container, err := cli.ContainerInspect(context.Background(), strings.TrimSpace(cid)) - c.Assert(err, checker.IsNil) - vname := container.Mounts[0].Name - - volumes, err := cli.VolumeList(context.Background(), filters.Args{}) - c.Assert(err, checker.IsNil) - - found := false - for _, vol := range volumes.Volumes { - if vol.Name == vname { - found = true - break - } - } - c.Assert(found, checker.Equals, true) -} - -func (s *DockerSuite) TestVolumesAPICreate(c *check.C) { - config := volumetypes.VolumesCreateBody{ - Name: "test", - } - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - vol, err := cli.VolumeCreate(context.Background(), config) - c.Assert(err, check.IsNil) - - c.Assert(filepath.Base(filepath.Dir(vol.Mountpoint)), checker.Equals, config.Name) -} - -func (s *DockerSuite) TestVolumesAPIRemove(c *check.C) { - prefix, _ := getPrefixAndSlashFromDaemonPlatform() - cid, _ := dockerCmd(c, "run", "-d", "-v", prefix+"/foo", "--name=test", "busybox") - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - container, err := cli.ContainerInspect(context.Background(), strings.TrimSpace(cid)) - c.Assert(err, checker.IsNil) - vname := container.Mounts[0].Name - - err = cli.VolumeRemove(context.Background(), vname, false) - c.Assert(err.Error(), checker.Contains, "volume is in use") - - dockerCmd(c, "rm", "-f", "test") - err = cli.VolumeRemove(context.Background(), vname, false) - c.Assert(err, checker.IsNil) -} - -func (s *DockerSuite) TestVolumesAPIInspect(c *check.C) { - config := volumetypes.VolumesCreateBody{ - Name: "test", - } - - // sampling current time minus a minute so to now have false positive in case of delays - now := time.Now().Truncate(time.Minute) - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - _, err = cli.VolumeCreate(context.Background(), config) - c.Assert(err, check.IsNil) - - vol, err := cli.VolumeInspect(context.Background(), config.Name) - c.Assert(err, checker.IsNil) - c.Assert(vol.Name, checker.Equals, config.Name) - - // comparing CreatedAt field time for the new volume to now. Removing a minute from both to avoid false positive - testCreatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(vol.CreatedAt)) - c.Assert(err, check.IsNil) - testCreatedAt = testCreatedAt.Truncate(time.Minute) - if !testCreatedAt.Equal(now) { - c.Assert(fmt.Errorf("Time Volume is CreatedAt not equal to current time"), check.NotNil) - } -} diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 5dd1d8b355..fc2dafe411 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -1439,6 +1439,7 @@ func (s *DockerSuite) TestBuildRelativeCopy(c *check.C) { )) } +// FIXME(vdemeester) should be unit test func (s *DockerSuite) TestBuildBlankName(c *check.C) { name := "testbuildblankname" testCases := []struct { @@ -2066,6 +2067,7 @@ func (s *DockerSuite) TestBuildNoContext(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli e2e func (s *DockerSuite) TestBuildDockerfileStdin(c *check.C) { name := "stdindockerfile" tmpDir, err := ioutil.TempDir("", "fake-context") @@ -2085,6 +2087,7 @@ CMD ["cat", "/foo"]`), c.Assert(strings.TrimSpace(string(res)), checker.Equals, `[cat /foo]`) } +// FIXME(vdemeester) migrate to docker/cli tests (unit or e2e) func (s *DockerSuite) TestBuildDockerfileStdinConflict(c *check.C) { name := "stdindockerfiletarcontext" icmd.RunCmd(icmd.Cmd{ @@ -2401,6 +2404,7 @@ func (s *DockerSuite) TestBuildDockerignoringDockerfile(c *check.C) { build.WithFile("Dockerfile", dockerfile), build.WithFile(".dockerignore", "Dockerfile\n"), )) + // FIXME(vdemeester) why twice ? buildImageSuccessfully(c, name, build.WithBuildContext(c, build.WithFile("Dockerfile", dockerfile), build.WithFile(".dockerignore", "./Dockerfile\n"), @@ -2420,6 +2424,7 @@ func (s *DockerSuite) TestBuildDockerignoringRenamedDockerfile(c *check.C) { build.WithFile("MyDockerfile", dockerfile), build.WithFile(".dockerignore", "MyDockerfile\n"), )) + // FIXME(vdemeester) why twice ? buildImageSuccessfully(c, name, cli.WithFlags("-f", "MyDockerfile"), build.WithBuildContext(c, build.WithFile("Dockerfile", "Should not use me"), build.WithFile("MyDockerfile", dockerfile), @@ -3045,6 +3050,7 @@ func (s *DockerSuite) TestBuildAddTarXzGz(c *check.C) { buildImageSuccessfully(c, name, build.WithExternalBuildContext(ctx)) } +// FIXME(vdemeester) most of the from git tests could be moved to `docker/cli` e2e tests func (s *DockerSuite) TestBuildFromGit(c *check.C) { name := "testbuildfromgit" git := fakegit.New(c, "repo", map[string]string{ @@ -3422,6 +3428,7 @@ func (s *DockerSuite) TestBuildLabelsCache(c *check.C) { } +// FIXME(vdemeester) port to docker/cli e2e tests (api tests should test suppressOutput option though) func (s *DockerSuite) TestBuildNotVerboseSuccess(c *check.C) { // This test makes sure that -q works correctly when build is successful: // stdout has only the image ID (long image ID) and stderr is empty. @@ -3472,6 +3479,7 @@ func (s *DockerSuite) TestBuildNotVerboseSuccess(c *check.C) { } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildNotVerboseFailureWithNonExistImage(c *check.C) { // This test makes sure that -q works correctly when build fails by // comparing between the stderr output in quiet mode and in stdout @@ -3492,6 +3500,7 @@ func (s *DockerSuite) TestBuildNotVerboseFailureWithNonExistImage(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildNotVerboseFailure(c *check.C) { // This test makes sure that -q works correctly when build fails by // comparing between the stderr output in quiet mode and in stdout @@ -3519,6 +3528,7 @@ func (s *DockerSuite) TestBuildNotVerboseFailure(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildNotVerboseFailureRemote(c *check.C) { // This test ensures that when given a wrong URL, stderr in quiet mode and // stderr in verbose mode are identical. @@ -3548,6 +3558,7 @@ func (s *DockerSuite) TestBuildNotVerboseFailureRemote(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildStderr(c *check.C) { // This test just makes sure that no non-error output goes // to stderr @@ -3688,67 +3699,6 @@ CMD cat /foo/file`), } -// FIXME(vdemeester) part of this should be unit test, other part should be clearer -func (s *DockerSuite) TestBuildRenamedDockerfile(c *check.C) { - ctx := fakecontext.New(c, "", fakecontext.WithFiles(map[string]string{ - "Dockerfile": "FROM busybox\nRUN echo from Dockerfile", - "files/Dockerfile": "FROM busybox\nRUN echo from files/Dockerfile", - "files/dFile": "FROM busybox\nRUN echo from files/dFile", - "dFile": "FROM busybox\nRUN echo from dFile", - "files/dFile2": "FROM busybox\nRUN echo from files/dFile2", - })) - defer ctx.Close() - - cli.Docker(cli.Args("build", "-t", "test1", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join("files", "Dockerfile"), "-t", "test2", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from files/Dockerfile", - }) - - cli.Docker(cli.Args("build", fmt.Sprintf("--file=%s", filepath.Join("files", "dFile")), "-t", "test3", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from files/dFile", - }) - - cli.Docker(cli.Args("build", "--file=dFile", "-t", "test4", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from dFile", - }) - - dirWithNoDockerfile, err := ioutil.TempDir(os.TempDir(), "test5") - c.Assert(err, check.IsNil) - nonDockerfileFile := filepath.Join(dirWithNoDockerfile, "notDockerfile") - if _, err = os.Create(nonDockerfileFile); err != nil { - c.Fatal(err) - } - cli.Docker(cli.Args("build", fmt.Sprintf("--file=%s", nonDockerfileFile), "-t", "test5", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - ExitCode: 1, - Err: fmt.Sprintf("unable to prepare context: the Dockerfile (%s) must be within the build context", nonDockerfileFile), - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join("..", "Dockerfile"), "-t", "test6", ".."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - Out: "from Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join(ctx.Dir, "files", "Dockerfile"), "-t", "test7", ".."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - Out: "from files/Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join("..", "Dockerfile"), "-t", "test8", "."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - ExitCode: 1, - Err: "must be within the build context", - }) - - tmpDir := os.TempDir() - cli.Docker(cli.Args("build", "-t", "test9", ctx.Dir), cli.InDir(tmpDir)).Assert(c, icmd.Expected{ - Out: "from Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", "dFile2", "-t", "test10", "."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - Out: "from files/dFile2", - }) -} - func (s *DockerSuite) TestBuildFromMixedcaseDockerfile(c *check.C) { testRequires(c, UnixCli) // Dockerfile overwrites dockerfile on windows testRequires(c, DaemonIsLinux) @@ -3772,6 +3722,7 @@ func (s *DockerSuite) TestBuildFromMixedcaseDockerfile(c *check.C) { }) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildFromURLWithF(c *check.C) { server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"baz": `FROM busybox RUN echo from baz @@ -3798,6 +3749,7 @@ RUN find /tmp/`})) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildFromStdinWithF(c *check.C) { testRequires(c, DaemonIsLinux) // TODO Windows: This test is flaky; no idea why ctx := fakecontext.New(c, "", fakecontext.WithDockerfile(`FROM busybox @@ -3840,61 +3792,6 @@ func (s *DockerSuite) TestBuildFromOfficialNames(c *check.C) { } } -func (s *DockerSuite) TestBuildDockerfileOutsideContext(c *check.C) { - testRequires(c, UnixCli, DaemonIsLinux) // uses os.Symlink: not implemented in windows at the time of writing (go-1.4.2) - - name := "testbuilddockerfileoutsidecontext" - tmpdir, err := ioutil.TempDir("", name) - c.Assert(err, check.IsNil) - defer os.RemoveAll(tmpdir) - ctx := filepath.Join(tmpdir, "context") - if err := os.MkdirAll(ctx, 0755); err != nil { - c.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(ctx, "Dockerfile"), []byte("FROM scratch\nENV X Y"), 0644); err != nil { - c.Fatal(err) - } - wd, err := os.Getwd() - if err != nil { - c.Fatal(err) - } - defer os.Chdir(wd) - if err := os.Chdir(ctx); err != nil { - c.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(tmpdir, "outsideDockerfile"), []byte("FROM scratch\nENV x y"), 0644); err != nil { - c.Fatal(err) - } - if err := os.Symlink(filepath.Join("..", "outsideDockerfile"), filepath.Join(ctx, "dockerfile1")); err != nil { - c.Fatal(err) - } - if err := os.Symlink(filepath.Join(tmpdir, "outsideDockerfile"), filepath.Join(ctx, "dockerfile2")); err != nil { - c.Fatal(err) - } - - for _, dockerfilePath := range []string{ - filepath.Join("..", "outsideDockerfile"), - filepath.Join(ctx, "dockerfile1"), - filepath.Join(ctx, "dockerfile2"), - } { - result := dockerCmdWithResult("build", "-t", name, "--no-cache", "-f", dockerfilePath, ".") - result.Assert(c, icmd.Expected{ - Err: "must be within the build context", - ExitCode: 1, - }) - deleteImages(name) - } - - os.Chdir(tmpdir) - - // Path to Dockerfile should be resolved relative to working directory, not relative to context. - // There is a Dockerfile in the context, but since there is no Dockerfile in the current directory, the following should fail - out, _, err := dockerCmdWithError("build", "-t", name, "--no-cache", "-f", "Dockerfile", ctx) - if err == nil { - c.Fatalf("Expected error. Out: %s", out) - } -} - // FIXME(vdemeester) should be a unit test func (s *DockerSuite) TestBuildSpaces(c *check.C) { // Test to make sure that leading/trailing spaces on a command @@ -4186,6 +4083,7 @@ func (s *DockerTrustSuite) TestTrustedBuildUntrustedTag(c *check.C) { }) } +// FIXME(vdemeester) should migrate to docker/cli e2e tests func (s *DockerTrustSuite) TestBuildContextDirIsSymlink(c *check.C) { testRequires(c, DaemonIsLinux) tempDir, err := ioutil.TempDir("", "test-build-dir-is-symlink-") @@ -5130,6 +5028,7 @@ func (s *DockerSuite) TestBuildCacheRootSource(c *check.C) { } // #19375 +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildFailsGitNotCallable(c *check.C) { buildImage("gitnotcallable", cli.WithEnvironmentVariables("PATH="), build.WithContextPath("github.com/docker/v1.10-migrator.git")).Assert(c, icmd.Expected{ @@ -6447,6 +6346,7 @@ CMD echo foo c.Assert(strings.TrimSpace(out), checker.Equals, `["/bin/sh","-c","echo foo"]`) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildIidFile(c *check.C) { tmpDir, err := ioutil.TempDir("", "TestBuildIidFile") if err != nil { @@ -6471,6 +6371,7 @@ ENV BAR BAZ`), c.Assert(d.String(), checker.Equals, getIDByName(c, name)) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { tmpDir, err := ioutil.TempDir("", "TestBuildIidFileCleanupOnFail") if err != nil { @@ -6493,6 +6394,7 @@ func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { c.Assert(os.IsNotExist(err), check.Equals, true) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildIidFileSquash(c *check.C) { testRequires(c, ExperimentalDaemon) tmpDir, err := ioutil.TempDir("", "TestBuildIidFileSquash") diff --git a/components/engine/integration-cli/docker_cli_config_inspect_test.go b/components/engine/integration-cli/docker_cli_config_inspect_test.go deleted file mode 100644 index ba4e80f070..0000000000 --- a/components/engine/integration-cli/docker_cli_config_inspect_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// +build !windows - -package main - -import ( - "encoding/json" - - "github.com/docker/docker/api/types/swarm" - "github.com/docker/docker/integration-cli/checker" - "github.com/go-check/check" -) - -func (s *DockerSwarmSuite) TestConfigInspect(c *check.C) { - d := s.AddDaemon(c, true, true) - - testName := "test_config" - id := d.CreateConfig(c, swarm.ConfigSpec{ - Annotations: swarm.Annotations{ - Name: testName, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("configs: %s", id)) - - config := d.GetConfig(c, id) - c.Assert(config.Spec.Name, checker.Equals, testName) - - out, err := d.Cmd("config", "inspect", testName) - c.Assert(err, checker.IsNil, check.Commentf(out)) - - var configs []swarm.Config - c.Assert(json.Unmarshal([]byte(out), &configs), checker.IsNil) - c.Assert(configs, checker.HasLen, 1) -} - -func (s *DockerSwarmSuite) TestConfigInspectMultiple(c *check.C) { - d := s.AddDaemon(c, true, true) - - testNames := []string{ - "test0", - "test1", - } - for _, n := range testNames { - id := d.CreateConfig(c, swarm.ConfigSpec{ - Annotations: swarm.Annotations{ - Name: n, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("configs: %s", id)) - - config := d.GetConfig(c, id) - c.Assert(config.Spec.Name, checker.Equals, n) - - } - - args := []string{ - "config", - "inspect", - } - args = append(args, testNames...) - out, err := d.Cmd(args...) - c.Assert(err, checker.IsNil, check.Commentf(out)) - - var configs []swarm.Config - c.Assert(json.Unmarshal([]byte(out), &configs), checker.IsNil) - c.Assert(configs, checker.HasLen, 2) -} diff --git a/components/engine/integration-cli/docker_cli_cp_from_container_test.go b/components/engine/integration-cli/docker_cli_cp_from_container_test.go index 0a282f5c09..499be54522 100644 --- a/components/engine/integration-cli/docker_cli_cp_from_container_test.go +++ b/components/engine/integration-cli/docker_cli_cp_from_container_test.go @@ -8,8 +8,6 @@ import ( "github.com/go-check/check" ) -// docker cp CONTAINER:PATH LOCALPATH - // Try all of the test cases from the archive package which implements the // internals of `docker cp` and ensure that the behavior matches when actually // copying to and from containers. @@ -20,67 +18,9 @@ import ( // 3. DST parent directory must exist. // 4. If DST exists as a file, it must not end with a trailing separator. -// First get these easy error cases out of the way. - -// Test for error when SRC does not exist. -func (s *DockerSuite) TestCpFromErrSrcNotExists(c *check.C) { - containerID := makeTestContainer(c, testContainerOptions{}) - - tmpDir := getTestDir(c, "test-cp-from-err-src-not-exists") - defer os.RemoveAll(tmpDir) - - err := runDockerCp(c, containerCpPath(containerID, "file1"), tmpDir, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotExist(err), checker.True, check.Commentf("expected IsNotExist error, but got %T: %s", err, err)) -} - -// Test for error when SRC ends in a trailing -// path separator but it exists as a file. -func (s *DockerSuite) TestCpFromErrSrcNotDir(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-from-err-src-not-dir") - defer os.RemoveAll(tmpDir) - - err := runDockerCp(c, containerCpPathTrailingSep(containerID, "file1"), tmpDir, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) -} - -// Test for error when DST ends in a trailing -// path separator but exists as a file. -func (s *DockerSuite) TestCpFromErrDstNotDir(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-from-err-dst-not-dir") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - // Try with a file source. - srcPath := containerCpPath(containerID, "/file1") - dstPath := cpPathTrailingSep(tmpDir, "file1") - - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) - - // Try with a directory source. - srcPath = containerCpPath(containerID, "/dir1") - - err = runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) -} - // Check that copying from a container to a local symlink copies to the symlink // target and does not overwrite the local symlink itself. +// TODO: move to docker/cli and/or integration/container/copy_test.go func (s *DockerSuite) TestCpFromSymlinkDestination(c *check.C) { testRequires(c, DaemonIsLinux) containerID := makeTestContainer(c, testContainerOptions{addContent: true}) diff --git a/components/engine/integration-cli/docker_cli_cp_to_container_test.go b/components/engine/integration-cli/docker_cli_cp_to_container_test.go index 24c1fe2288..77567a3b95 100644 --- a/components/engine/integration-cli/docker_cli_cp_to_container_test.go +++ b/components/engine/integration-cli/docker_cli_cp_to_container_test.go @@ -2,15 +2,11 @@ package main import ( "os" - "runtime" - "strings" "github.com/docker/docker/integration-cli/checker" "github.com/go-check/check" ) -// docker cp LOCALPATH CONTAINER:PATH - // Try all of the test cases from the archive package which implements the // internals of `docker cp` and ensure that the behavior matches when actually // copying to and from containers. @@ -21,124 +17,6 @@ import ( // 3. DST parent directory must exist. // 4. If DST exists as a file, it must not end with a trailing separator. -// First get these easy error cases out of the way. - -// Test for error when SRC does not exist. -func (s *DockerSuite) TestCpToErrSrcNotExists(c *check.C) { - containerID := makeTestContainer(c, testContainerOptions{}) - - tmpDir := getTestDir(c, "test-cp-to-err-src-not-exists") - defer os.RemoveAll(tmpDir) - - srcPath := cpPath(tmpDir, "file1") - dstPath := containerCpPath(containerID, "file1") - _, srcStatErr := os.Stat(srcPath) - c.Assert(os.IsNotExist(srcStatErr), checker.True) - - err := runDockerCp(c, srcPath, dstPath, nil) - if runtime.GOOS == "windows" { - // Go 1.9+ on Windows returns a different error for `os.Stat()`, see - // https://github.com/golang/go/commit/6144c7270e5812d9de8fb97456ee4e5ae657fcbb#diff-f63e1a4b4377b2fe0b05011db3df9599 - // - // Go 1.8: CreateFile C:\not-exist: The system cannot find the file specified. - // Go 1.9: GetFileAttributesEx C:\not-exist: The system cannot find the file specified. - // - // Due to the CLI using a different version than the daemon, comparing the - // error message won't work, so just hard-code the common part here. - // - // TODO this should probably be a test in the CLI repository instead - c.Assert(strings.ToLower(err.Error()), checker.Contains, "cannot find the file specified") - c.Assert(strings.ToLower(err.Error()), checker.Contains, strings.ToLower(tmpDir)) - } else { - c.Assert(strings.ToLower(err.Error()), checker.Contains, strings.ToLower(srcStatErr.Error())) - } -} - -// Test for error when SRC ends in a trailing -// path separator but it exists as a file. -func (s *DockerSuite) TestCpToErrSrcNotDir(c *check.C) { - containerID := makeTestContainer(c, testContainerOptions{}) - - tmpDir := getTestDir(c, "test-cp-to-err-src-not-dir") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - srcPath := cpPathTrailingSep(tmpDir, "file1") - dstPath := containerCpPath(containerID, "testDir") - - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) -} - -// Test for error when SRC is a valid file or directory, -// but the DST parent directory does not exist. -func (s *DockerSuite) TestCpToErrDstParentNotExists(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-to-err-dst-parent-not-exists") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - // Try with a file source. - srcPath := cpPath(tmpDir, "file1") - dstPath := containerCpPath(containerID, "/notExists", "file1") - - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotExist(err), checker.True, check.Commentf("expected IsNotExist error, but got %T: %s", err, err)) - - // Try with a directory source. - srcPath = cpPath(tmpDir, "dir1") - - err = runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotExist(err), checker.True, check.Commentf("expected IsNotExist error, but got %T: %s", err, err)) -} - -// Test for error when DST ends in a trailing path separator but exists as a -// file. Also test that we cannot overwrite an existing directory with a -// non-directory and cannot overwrite an existing -func (s *DockerSuite) TestCpToErrDstNotDir(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-to-err-dst-not-dir") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - // Try with a file source. - srcPath := cpPath(tmpDir, "dir1/file1-1") - dstPath := containerCpPathTrailingSep(containerID, "file1") - - // The client should encounter an error trying to stat the destination - // and then be unable to copy since the destination is asserted to be a - // directory but does not exist. - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpDirNotExist(err), checker.True, check.Commentf("expected DirNotExist error, but got %T: %s", err, err)) - - // Try with a directory source. - srcPath = cpPath(tmpDir, "dir1") - - // The client should encounter an error trying to stat the destination and - // then decide to extract to the parent directory instead with a rebased - // name in the source archive, but this directory would overwrite the - // existing file with the same name. - err = runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCannotOverwriteNonDirWithDir(err), checker.True, check.Commentf("expected CannotOverwriteNonDirWithDir error, but got %T: %s", err, err)) -} - // Check that copying from a local path to a symlink in a container copies to // the symlink target and does not overwrite the container symlink itself. func (s *DockerSuite) TestCpToSymlinkDestination(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_cp_utils_test.go b/components/engine/integration-cli/docker_cli_cp_utils_test.go index 402a87ea90..79a016f0c6 100644 --- a/components/engine/integration-cli/docker_cli_cp_utils_test.go +++ b/components/engine/integration-cli/docker_cli_cp_utils_test.go @@ -228,18 +228,10 @@ func getTestDir(c *check.C, label string) (tmpDir string) { return } -func isCpNotExist(err error) bool { - return strings.Contains(strings.ToLower(err.Error()), "could not find the file") -} - func isCpDirNotExist(err error) bool { return strings.Contains(err.Error(), archive.ErrDirNotExists.Error()) } -func isCpNotDir(err error) bool { - return strings.Contains(err.Error(), archive.ErrNotDirectory.Error()) || strings.Contains(err.Error(), "filename, directory name, or volume label syntax is incorrect") -} - func isCpCannotCopyDir(err error) bool { return strings.Contains(err.Error(), archive.ErrCannotCopyDir.Error()) } @@ -248,10 +240,6 @@ func isCpCannotCopyReadOnly(err error) bool { return strings.Contains(err.Error(), "marked read-only") } -func isCannotOverwriteNonDirWithDir(err error) bool { - return strings.Contains(err.Error(), "cannot overwrite non-directory") -} - func fileContentEquals(c *check.C, filename, contents string) (err error) { c.Logf("checking that file %q contains %q\n", filename, contents) diff --git a/components/engine/integration-cli/docker_cli_network_unix_test.go b/components/engine/integration-cli/docker_cli_network_unix_test.go index 90e4f6c1f2..4e9edba5a5 100644 --- a/components/engine/integration-cli/docker_cli_network_unix_test.go +++ b/components/engine/integration-cli/docker_cli_network_unix_test.go @@ -1541,10 +1541,10 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectDisconnectAlias(c *check.C) { dockerCmd(c, "network", "create", "-d", "bridge", "net1") dockerCmd(c, "network", "create", "-d", "bridge", "net2") - cid, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo", "busybox", "top") + cid, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // ping first container and its alias @@ -1581,7 +1581,7 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectDisconnectAlias(c *check.C) { c.Assert(err, check.IsNil) // verify the alias option is rejected when running on predefined network - out, _, err := dockerCmdWithError("run", "--rm", "--name=any", "--net-alias=any", "busybox", "top") + out, _, err := dockerCmdWithError("run", "--rm", "--name=any", "--net-alias=any", "busybox:glibc", "top") c.Assert(err, checker.NotNil, check.Commentf("out: %s", out)) c.Assert(out, checker.Contains, runconfig.ErrUnsupportedNetworkAndAlias.Error()) @@ -1595,10 +1595,10 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectivity(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace) dockerCmd(c, "network", "create", "-d", "bridge", "br.net1") - dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c1.net1", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c1.net1", "busybox:glibc", "top") c.Assert(waitRun("c1.net1"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c2.net1", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c2.net1", "busybox:glibc", "top") c.Assert(waitRun("c2.net1"), check.IsNil) // ping first container by its unqualified name @@ -1643,9 +1643,9 @@ func (s *DockerSuite) TestDockerNetworkInternalMode(c *check.C) { nr := getNetworkResource(c, "internal") c.Assert(nr.Internal, checker.True) - dockerCmd(c, "run", "-d", "--net=internal", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=internal", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=internal", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=internal", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) out, _, err := dockerCmdWithError("exec", "first", "ping", "-W", "4", "-c", "1", "www.google.com") c.Assert(err, check.NotNil) diff --git a/components/engine/integration-cli/docker_cli_rm_test.go b/components/engine/integration-cli/docker_cli_rm_test.go index d281704a7b..5942b8286d 100644 --- a/components/engine/integration-cli/docker_cli_rm_test.go +++ b/components/engine/integration-cli/docker_cli_rm_test.go @@ -1,56 +1,11 @@ package main import ( - "io/ioutil" - "os" - "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/cli/build" "github.com/go-check/check" ) -func (s *DockerSuite) TestRmContainerWithRemovedVolume(c *check.C) { - testRequires(c, SameHostDaemon) - - prefix, slash := getPrefixAndSlashFromDaemonPlatform() - - tempDir, err := ioutil.TempDir("", "test-rm-container-with-removed-volume-") - if err != nil { - c.Fatalf("failed to create temporary directory: %s", tempDir) - } - defer os.RemoveAll(tempDir) - - dockerCmd(c, "run", "--name", "losemyvolumes", "-v", tempDir+":"+prefix+slash+"test", "busybox", "true") - - err = os.RemoveAll(tempDir) - c.Assert(err, check.IsNil) - - dockerCmd(c, "rm", "-v", "losemyvolumes") -} - -func (s *DockerSuite) TestRmContainerWithVolume(c *check.C) { - prefix, slash := getPrefixAndSlashFromDaemonPlatform() - - dockerCmd(c, "run", "--name", "foo", "-v", prefix+slash+"srv", "busybox", "true") - - dockerCmd(c, "rm", "-v", "foo") -} - -func (s *DockerSuite) TestRmContainerRunning(c *check.C) { - createRunningContainer(c, "foo") - - res, _, err := dockerCmdWithError("rm", "foo") - c.Assert(err, checker.NotNil, check.Commentf("Expected error, can't rm a running container")) - c.Assert(res, checker.Contains, "cannot remove a running container") -} - -func (s *DockerSuite) TestRmContainerForceRemoveRunning(c *check.C) { - createRunningContainer(c, "foo") - - // Stop then remove with -f - dockerCmd(c, "rm", "-f", "foo") -} - func (s *DockerSuite) TestRmContainerOrphaning(c *check.C) { dockerfile1 := `FROM busybox:latest ENTRYPOINT ["true"]` @@ -75,13 +30,3 @@ func (s *DockerSuite) TestRmContainerOrphaning(c *check.C) { c.Assert(out, checker.Contains, img1, check.Commentf("Orphaned container (could not find %q in docker images): %s", img1, out)) } - -func (s *DockerSuite) TestRmInvalidContainer(c *check.C) { - out, _, err := dockerCmdWithError("rm", "unknown") - c.Assert(err, checker.NotNil, check.Commentf("Expected error on rm unknown container, got none")) - c.Assert(out, checker.Contains, "No such container") -} - -func createRunningContainer(c *check.C, name string) { - runSleepingContainer(c, "-dt", "--name", name) -} diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 776f0e5ba5..871fee7d0a 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -294,7 +294,7 @@ func (s *DockerSuite) TestUserDefinedNetworkAlias(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm) dockerCmd(c, "network", "create", "-d", "bridge", "net1") - cid1, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo1", "--net-alias=foo2", "busybox", "top") + cid1, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo1", "--net-alias=foo2", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) // Check if default short-id alias is added automatically @@ -302,7 +302,7 @@ func (s *DockerSuite) TestUserDefinedNetworkAlias(c *check.C) { aliases := inspectField(c, id, "NetworkSettings.Networks.net1.Aliases") c.Assert(aliases, checker.Contains, stringid.TruncateID(id)) - cid2, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox", "top") + cid2, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // Check if default short-id alias is added automatically @@ -4237,35 +4237,6 @@ func (s *DockerSuite) TestRunCredentialSpecWellFormed(c *check.C) { dockerCmd(c, "run", `--security-opt=credentialspec=file://valid.json`, "busybox", "true") } -// Windows specific test to ensure that a servicing app container is started -// if necessary once a container exits. It does this by forcing a no-op -// servicing event and verifying the event from Hyper-V-Compute -func (s *DockerSuite) TestRunServicingContainer(c *check.C) { - testRequires(c, DaemonIsWindows, SameHostDaemon) - - // This functionality does not exist in post-RS3 builds. - // Note we get the version number from the full build string, as Windows - // reports Windows 8 version 6.2 build 9200 from non-manifested binaries. - // Ref: https://msdn.microsoft.com/en-us/library/windows/desktop/ms724451(v=vs.85).aspx - v, err := kernel.GetKernelVersion() - c.Assert(err, checker.IsNil) - build, _ := strconv.Atoi(strings.Split(strings.SplitN(v.String(), " ", 3)[2][1:], ".")[0]) - if build > 16299 { - c.Skip("Disabled on post-RS3 builds") - } - - out := cli.DockerCmd(c, "run", "-d", testEnv.PlatformDefaults.BaseImage, "cmd", "/c", "mkdir c:\\programdata\\Microsoft\\Windows\\ContainerUpdates\\000_000_d99f45d0-ffc8-4af7-bd9c-ea6a62e035c9_200 && sc control cexecsvc 255").Combined() - containerID := strings.TrimSpace(out) - cli.WaitExited(c, containerID, 60*time.Second) - - result := icmd.RunCommand("powershell", "echo", `(Get-WinEvent -ProviderName "Microsoft-Windows-Hyper-V-Compute" -FilterXPath 'Event[System[EventID=2010]]' -MaxEvents 1).Message`) - result.Assert(c, icmd.Success) - out2 := result.Combined() - c.Assert(out2, checker.Contains, `"Servicing":true`, check.Commentf("Servicing container does not appear to have been started: %s", out2)) - c.Assert(out2, checker.Contains, `Windows Container (Servicing)`, check.Commentf("Didn't find 'Windows Container (Servicing): %s", out2)) - c.Assert(out2, checker.Contains, containerID+"_servicing", check.Commentf("Didn't find '%s_servicing': %s", containerID+"_servicing", out2)) -} - func (s *DockerSuite) TestRunDuplicateMount(c *check.C) { testRequires(c, SameHostDaemon, DaemonIsLinux, NotUserNamespace) diff --git a/components/engine/integration-cli/docker_cli_secret_create_test.go b/components/engine/integration-cli/docker_cli_secret_create_test.go index 839c3922ac..a807e4e7e7 100644 --- a/components/engine/integration-cli/docker_cli_secret_create_test.go +++ b/components/engine/integration-cli/docker_cli_secret_create_test.go @@ -12,45 +12,6 @@ import ( "github.com/go-check/check" ) -func (s *DockerSwarmSuite) TestSecretCreate(c *check.C) { - d := s.AddDaemon(c, true, true) - - testName := "test_secret" - id := d.CreateSecret(c, swarm.SecretSpec{ - Annotations: swarm.Annotations{ - Name: testName, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) - - secret := d.GetSecret(c, id) - c.Assert(secret.Spec.Name, checker.Equals, testName) -} - -func (s *DockerSwarmSuite) TestSecretCreateWithLabels(c *check.C) { - d := s.AddDaemon(c, true, true) - - testName := "test_secret" - id := d.CreateSecret(c, swarm.SecretSpec{ - Annotations: swarm.Annotations{ - Name: testName, - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) - - secret := d.GetSecret(c, id) - c.Assert(secret.Spec.Name, checker.Equals, testName) - c.Assert(len(secret.Spec.Labels), checker.Equals, 2) - c.Assert(secret.Spec.Labels["key1"], checker.Equals, "value1") - c.Assert(secret.Spec.Labels["key2"], checker.Equals, "value2") -} - // Test case for 28884 func (s *DockerSwarmSuite) TestSecretCreateResolve(c *check.C) { d := s.AddDaemon(c, true, true) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 4a5ec9a566..8add18e1ec 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -345,13 +345,13 @@ func (s *DockerSwarmSuite) TestSwarmContainerEndpointOptions(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") - _, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox", "top") + _, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox:glibc", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) - _, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox", "top") + _, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox:glibc", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) - _, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox", "top") + _, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox:glibc", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) // ping first container and its alias, also ping third and anonymous container by its alias diff --git a/components/engine/integration-cli/docker_experimental_network_test.go b/components/engine/integration-cli/docker_experimental_network_test.go index fb20331631..9d6be69607 100644 --- a/components/engine/integration-cli/docker_experimental_network_test.go +++ b/components/engine/integration-cli/docker_experimental_network_test.go @@ -143,8 +143,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkMacvlanMultiSubnet(c *check.C) { // Ensure the network was created assertNwIsAvailable(c, "dualstackbridge") // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.100.0/24 and 2001:db8:abc2::/64 - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=first", "--ip", "172.28.100.20", "--ip6", "2001:db8:abc2::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=second", "--ip", "172.28.100.21", "--ip6", "2001:db8:abc2::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=first", "--ip", "172.28.100.20", "--ip6", "2001:db8:abc2::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=second", "--ip", "172.28.100.21", "--ip6", "2001:db8:abc2::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackbridge ip := inspectField(c, "first", "NetworkSettings.Networks.dualstackbridge.IPAddress") @@ -160,8 +160,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkMacvlanMultiSubnet(c *check.C) { c.Assert(err, check.IsNil) // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.102.0/24 and 2001:db8:abc4::/64 - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=third", "--ip", "172.28.102.20", "--ip6", "2001:db8:abc4::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=fourth", "--ip", "172.28.102.21", "--ip6", "2001:db8:abc4::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=third", "--ip", "172.28.102.20", "--ip6", "2001:db8:abc4::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=fourth", "--ip", "172.28.102.21", "--ip6", "2001:db8:abc4::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackbridge ip = inspectField(c, "third", "NetworkSettings.Networks.dualstackbridge.IPAddress") @@ -198,8 +198,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL2MultiSubnet(c *check.C) { // Ensure the network was created assertNwIsAvailable(c, "dualstackl2") // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.200.0/24 and 2001:db8:abc8::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=first", "--ip", "172.28.200.20", "--ip6", "2001:db8:abc8::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=second", "--ip", "172.28.200.21", "--ip6", "2001:db8:abc8::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=first", "--ip", "172.28.200.20", "--ip6", "2001:db8:abc8::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=second", "--ip", "172.28.200.21", "--ip6", "2001:db8:abc8::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl2 ip := inspectField(c, "first", "NetworkSettings.Networks.dualstackl2.IPAddress") @@ -214,8 +214,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL2MultiSubnet(c *check.C) { c.Assert(err, check.IsNil) // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.202.0/24 and 2001:db8:abc6::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=third", "--ip", "172.28.202.20", "--ip6", "2001:db8:abc6::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=fourth", "--ip", "172.28.202.21", "--ip6", "2001:db8:abc6::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=third", "--ip", "172.28.202.20", "--ip6", "2001:db8:abc6::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=fourth", "--ip", "172.28.202.21", "--ip6", "2001:db8:abc6::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl2 ip = inspectField(c, "third", "NetworkSettings.Networks.dualstackl2.IPAddress") @@ -253,8 +253,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL3MultiSubnet(c *check.C) { assertNwIsAvailable(c, "dualstackl3") // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.10.0/24 and 2001:db8:abc9::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=first", "--ip", "172.28.10.20", "--ip6", "2001:db8:abc9::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=second", "--ip", "172.28.10.21", "--ip6", "2001:db8:abc9::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=first", "--ip", "172.28.10.20", "--ip6", "2001:db8:abc9::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=second", "--ip", "172.28.10.21", "--ip6", "2001:db8:abc9::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl3 ip := inspectField(c, "first", "NetworkSettings.Networks.dualstackl3.IPAddress") @@ -269,8 +269,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL3MultiSubnet(c *check.C) { c.Assert(err, check.IsNil) // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.12.0/24 and 2001:db8:abc7::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=third", "--ip", "172.28.12.20", "--ip6", "2001:db8:abc7::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=fourth", "--ip", "172.28.12.21", "--ip6", "2001:db8:abc7::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=third", "--ip", "172.28.12.20", "--ip6", "2001:db8:abc7::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=fourth", "--ip", "172.28.12.21", "--ip6", "2001:db8:abc7::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl3 ip = inspectField(c, "third", "NetworkSettings.Networks.dualstackl3.IPAddress") @@ -356,9 +356,9 @@ func (s *DockerSuite) TestDockerNetworkMacVlanBridgeNilParent(c *check.C) { assertNwIsAvailable(c, "dm-nil-parent") // start two containers on the same subnet - dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // intra-network communications should succeed @@ -375,9 +375,9 @@ func (s *DockerSuite) TestDockerNetworkMacVlanBridgeInternalMode(c *check.C) { c.Assert(nr.Internal, checker.True) // start two containers on the same subnet - cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=first", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=second", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // access outside of the network should fail @@ -395,9 +395,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL2NilParent(c *check.C) { assertNwIsAvailable(c, "di-nil-parent") // start two containers on the same subnet - dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // intra-network communications should succeed @@ -414,9 +414,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL2InternalMode(c *check.C) { c.Assert(nr.Internal, checker.True) // start two containers on the same subnet - cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=first", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=second", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // access outside of the network should fail @@ -434,9 +434,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL3NilParent(c *check.C) { assertNwIsAvailable(c, "di-nil-parent-l3") // start two containers on separate subnets - dockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-nil-parent-l3", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-nil-parent-l3", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-nil-parent-l3", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-nil-parent-l3", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // intra-network communications should succeed @@ -454,9 +454,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL3InternalMode(c *check.C) { c.Assert(nr.Internal, checker.True) // start two containers on separate subnets - cli.DockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-internal-l3", "--name=first", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-internal-l3", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - cli.DockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-internal-l3", "--name=second", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-internal-l3", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // access outside of the network should fail @@ -496,9 +496,9 @@ func (s *DockerSuite) TestDockerNetworkMacVlanSubinterface(c *check.C) { assertNwIsAvailable(c, netName) // start containers on 802.1q tagged '-o parent' sub-interface - dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // verify containers can communicate _, _, err := dockerCmdWithError("exec", "second", "ping", "-c", "1", "first") diff --git a/components/engine/integration-cli/fixtures/load/frozen.go b/components/engine/integration-cli/fixtures/load/frozen.go index 7e104fe76c..5701a216a2 100644 --- a/components/engine/integration-cli/fixtures/load/frozen.go +++ b/components/engine/integration-cli/fixtures/load/frozen.go @@ -37,9 +37,6 @@ func FrozenImagesLinux(client client.APIClient, images ...string) error { if img == "hello-world:frozen" { srcName = "hello-world:latest" } - if img == "busybox:1.27-glibc" { - img = "busybox:latest" - } loadImages = append(loadImages, struct{ srcName, destName string }{ srcName: srcName, destName: img, diff --git a/components/engine/integration/build/build_test.go b/components/engine/integration/build/build_test.go index 3f7b1dcf34..124f1107fb 100644 --- a/components/engine/integration/build/build_test.go +++ b/components/engine/integration/build/build_test.go @@ -301,6 +301,46 @@ COPY bar /` require.NotContains(t, out.String(), "Using cache") } +// docker/for-linux#135 +// #35641 +func TestBuildMultiStageLayerLeak(t *testing.T) { + ctx := context.TODO() + defer setupTest(t)() + + // all commands need to match until COPY + dockerfile := `FROM busybox +WORKDIR /foo +COPY foo . +FROM busybox +WORKDIR /foo +COPY bar . +RUN [ -f bar ] +RUN [ ! -f foo ] +` + + source := fakecontext.New(t, "", + fakecontext.WithFile("foo", "0"), + fakecontext.WithFile("bar", "1"), + fakecontext.WithDockerfile(dockerfile)) + defer source.Close() + + apiclient := testEnv.APIClient() + resp, err := apiclient.ImageBuild(ctx, + source.AsTarReader(t), + types.ImageBuildOptions{ + Remove: true, + ForceRemove: true, + }) + + out := bytes.NewBuffer(nil) + require.NoError(t, err) + _, err = io.Copy(out, resp.Body) + resp.Body.Close() + require.NoError(t, err) + + assert.Contains(t, out.String(), "Successfully built") +} + func writeTarRecord(t *testing.T, w *tar.Writer, fn, contents string) { err := w.WriteHeader(&tar.Header{ Name: fn, diff --git a/components/engine/integration/config/config_test.go b/components/engine/integration/config/config_test.go index c152be59bf..4e31b205ee 100644 --- a/components/engine/integration/config/config_test.go +++ b/components/engine/integration/config/config_test.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "encoding/json" "sort" "testing" "time" @@ -327,3 +328,27 @@ func waitAndAssert(t *testing.T, timeout time.Duration, f func(*testing.T) bool) time.Sleep(100 * time.Millisecond) } } + +func TestConfigInspect(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux") + + defer setupTest(t)() + d := swarm.NewSwarm(t, testEnv) + defer d.Stop(t) + client, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) + require.NoError(t, err) + + ctx := context.Background() + + testName := t.Name() + configID := createConfig(ctx, t, client, testName, []byte("TESTINGDATA"), nil) + + insp, body, err := client.ConfigInspectWithRaw(ctx, configID) + require.NoError(t, err) + assert.Equal(t, insp.Spec.Name, testName) + + var config swarmtypes.Config + err = json.Unmarshal(body, &config) + require.NoError(t, err) + assert.Equal(t, config, insp) +} diff --git a/components/engine/integration/container/copy_test.go b/components/engine/integration/container/copy_test.go new file mode 100644 index 0000000000..43dc31f2f2 --- /dev/null +++ b/components/engine/integration/container/copy_test.go @@ -0,0 +1,65 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "context" + "fmt" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/internal/testutil" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/require" +) + +func TestCopyFromContainerPathDoesNotExist(t *testing.T) { + defer setupTest(t)() + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + _, _, err := apiclient.CopyFromContainer(ctx, cid, "/dne") + require.True(t, client.IsErrNotFound(err)) + expected := fmt.Sprintf("No such container:path: %s:%s", cid, "/dne") + testutil.ErrorContains(t, err, expected) +} + +func TestCopyFromContainerPathIsNotDir(t *testing.T) { + defer setupTest(t)() + skip.If(t, testEnv.OSType == "windows") + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + _, _, err := apiclient.CopyFromContainer(ctx, cid, "/etc/passwd/") + require.Contains(t, err.Error(), "not a directory") +} + +func TestCopyToContainerPathDoesNotExist(t *testing.T) { + defer setupTest(t)() + skip.If(t, testEnv.OSType == "windows") + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + err := apiclient.CopyToContainer(ctx, cid, "/dne", nil, types.CopyToContainerOptions{}) + require.True(t, client.IsErrNotFound(err)) + expected := fmt.Sprintf("No such container:path: %s:%s", cid, "/dne") + testutil.ErrorContains(t, err, expected) +} + +func TestCopyToContainerPathIsNotDir(t *testing.T) { + defer setupTest(t)() + skip.If(t, testEnv.OSType == "windows") + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + err := apiclient.CopyToContainer(ctx, cid, "/etc/passwd/", nil, types.CopyToContainerOptions{}) + require.Contains(t, err.Error(), "not a directory") +} diff --git a/components/engine/integration/container/remove_test.go b/components/engine/integration/container/remove_test.go new file mode 100644 index 0000000000..bf55dd22c7 --- /dev/null +++ b/components/engine/integration/container/remove_test.go @@ -0,0 +1,113 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "context" + "os" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/testutil" + "github.com/gotestyourself/gotestyourself/fs" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getPrefixAndSlashFromDaemonPlatform() (prefix, slash string) { + if testEnv.OSType == "windows" { + return "c:", `\` + } + return "", "/" +} + +// Test case for #5244: `docker rm` fails if bind dir doesn't exist anymore +func TestRemoveContainerWithRemovedVolume(t *testing.T) { + skip.If(t, !testEnv.IsLocalDaemon()) + + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + prefix, slash := getPrefixAndSlashFromDaemonPlatform() + + tempDir := fs.NewDir(t, "test-rm-container-with-removed-volume", fs.WithMode(0755)) + defer tempDir.Remove() + + cID := container.Run(t, ctx, client, container.WithCmd("true"), container.WithBind(tempDir.Path(), prefix+slash+"test")) + poll.WaitOn(t, container.IsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond)) + + err := os.RemoveAll(tempDir.Path()) + require.NoError(t, err) + + err = client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) + require.NoError(t, err) + + _, _, err = client.ContainerInspectWithRaw(ctx, cID, true) + testutil.ErrorContains(t, err, "No such container") +} + +// Test case for #2099/#2125 +func TestRemoveContainerWithVolume(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + prefix, slash := getPrefixAndSlashFromDaemonPlatform() + + cID := container.Run(t, ctx, client, container.WithCmd("true"), container.WithVolume(prefix+slash+"srv")) + poll.WaitOn(t, container.IsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond)) + + insp, _, err := client.ContainerInspectWithRaw(ctx, cID, true) + require.NoError(t, err) + assert.Equal(t, len(insp.Mounts), 1) + volName := insp.Mounts[0].Name + + err = client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) + require.NoError(t, err) + + volumes, err := client.VolumeList(ctx, filters.NewArgs(filters.Arg("name", volName))) + require.NoError(t, err) + assert.Equal(t, len(volumes.Volumes), 0) +} + +func TestRemoveContainerRunning(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + cID := container.Run(t, ctx, client) + + err := client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{}) + testutil.ErrorContains(t, err, "cannot remove a running container") +} + +func TestRemoveContainerForceRemoveRunning(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + cID := container.Run(t, ctx, client) + + err := client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{ + Force: true, + }) + require.NoError(t, err) +} + +func TestRemoveInvalidContainer(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + err := client.ContainerRemove(ctx, "unknown", types.ContainerRemoveOptions{}) + testutil.ErrorContains(t, err, "No such container") +} diff --git a/components/engine/integration/internal/container/ops.go b/components/engine/integration/internal/container/ops.go index e3d538bef1..9360527d37 100644 --- a/components/engine/integration/internal/container/ops.go +++ b/components/engine/integration/internal/container/ops.go @@ -1,6 +1,8 @@ package container import ( + "fmt" + containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/docker/go-connections/nat" @@ -57,3 +59,20 @@ func WithWorkingDir(dir string) func(*TestContainerConfig) { c.Config.WorkingDir = dir } } + +// WithVolume sets the volume of the container +func WithVolume(name string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + if c.Config.Volumes == nil { + c.Config.Volumes = map[string]struct{}{} + } + c.Config.Volumes[name] = struct{}{} + } +} + +// WithBind sets the bind mount of the container +func WithBind(src, target string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.Binds = append(c.HostConfig.Binds, fmt.Sprintf("%s:%s", src, target)) + } +} diff --git a/components/engine/integration/secret/secret_test.go b/components/engine/integration/secret/secret_test.go index 3b5e66a5bf..8a292f005b 100644 --- a/components/engine/integration/secret/secret_test.go +++ b/components/engine/integration/secret/secret_test.go @@ -129,32 +129,7 @@ func createSecret(ctx context.Context, t *testing.T, client client.APIClient, na return secret.ID } -func TestSecretsCreate(t *testing.T) { - skip.If(t, testEnv.DaemonInfo.OSType != "linux") - - defer setupTest(t)() - d := swarm.NewSwarm(t, testEnv) - defer d.Stop(t) - client, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) - require.NoError(t, err) - - ctx := context.Background() - - testName := "test_secret" - createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil) - require.NoError(t, err) - - // create an already existin secret, daemon should return a status code of 409 - _, err = client.SecretCreate(ctx, swarmtypes.SecretSpec{ - Annotations: swarmtypes.Annotations{ - Name: testName, - }, - Data: []byte("TESTINGDATA"), - }) - testutil.ErrorContains(t, err, "already exists") -} - -func TestSecretsDelete(t *testing.T) { +func TestSecretsCreateAndDelete(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "linux") defer setupTest(t)() @@ -167,12 +142,17 @@ func TestSecretsDelete(t *testing.T) { testName := "test_secret" secretID := createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil) - require.NoError(t, err) - insp, _, err := client.SecretInspectWithRaw(ctx, secretID) - require.NoError(t, err) - assert.Equal(t, insp.ID, secretID) + // create an already existin secret, daemon should return a status code of 409 + _, err = client.SecretCreate(ctx, swarmtypes.SecretSpec{ + Annotations: swarmtypes.Annotations{ + Name: testName, + }, + Data: []byte("TESTINGDATA"), + }) + testutil.ErrorContains(t, err, "already exists") + // Ported from original TestSecretsDelete err = client.SecretRemove(ctx, secretID) require.NoError(t, err) @@ -181,6 +161,20 @@ func TestSecretsDelete(t *testing.T) { err = client.SecretRemove(ctx, "non-existin") testutil.ErrorContains(t, err, "No such secret: non-existin") + + // Ported from original TestSecretsCreteaWithLabels + testName = "test_secret_with_labels" + secretID = createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), map[string]string{ + "key1": "value1", + "key2": "value2", + }) + + insp, _, err := client.SecretInspectWithRaw(ctx, secretID) + require.NoError(t, err) + assert.Equal(t, insp.Spec.Name, testName) + assert.Equal(t, len(insp.Spec.Labels), 2) + assert.Equal(t, insp.Spec.Labels["key1"], "value1") + assert.Equal(t, insp.Spec.Labels["key2"], "value2") } func TestSecretsUpdate(t *testing.T) { diff --git a/components/engine/integration/volume/main_test.go b/components/engine/integration/volume/main_test.go new file mode 100644 index 0000000000..206f7377ae --- /dev/null +++ b/components/engine/integration/volume/main_test.go @@ -0,0 +1,33 @@ +package volume // import "github.com/docker/docker/integration/volume" + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/internal/test/environment" +) + +var testEnv *environment.Execution + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = environment.EnsureFrozenImagesLinux(testEnv) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + testEnv.Print() + os.Exit(m.Run()) +} + +func setupTest(t *testing.T) func() { + environment.ProtectAll(t, testEnv) + return func() { testEnv.Clean(t) } +} diff --git a/components/engine/integration/volume/volume_test.go b/components/engine/integration/volume/volume_test.go new file mode 100644 index 0000000000..38ce5782e1 --- /dev/null +++ b/components/engine/integration/volume/volume_test.go @@ -0,0 +1,115 @@ +package volume + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + volumetypes "github.com/docker/docker/api/types/volume" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVolumesCreateAndList(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + name := t.Name() + vol, err := client.VolumeCreate(ctx, volumetypes.VolumesCreateBody{ + Name: name, + }) + require.NoError(t, err) + + expected := types.Volume{ + // Ignore timestamp of CreatedAt + CreatedAt: vol.CreatedAt, + Driver: "local", + Scope: "local", + Name: name, + Options: map[string]string{}, + Mountpoint: fmt.Sprintf("%s/volumes/%s/_data", testEnv.DaemonInfo.DockerRootDir, name), + } + assert.Equal(t, vol, expected) + + volumes, err := client.VolumeList(ctx, filters.Args{}) + require.NoError(t, err) + + assert.Equal(t, len(volumes.Volumes), 1) + assert.NotNil(t, volumes.Volumes[0]) + assert.Equal(t, *volumes.Volumes[0], expected) +} + +func TestVolumesRemove(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + prefix, _ := getPrefixAndSlashFromDaemonPlatform() + + id := container.Create(t, ctx, client, container.WithVolume(prefix+"foo")) + + c, err := client.ContainerInspect(ctx, id) + require.NoError(t, err) + vname := c.Mounts[0].Name + + err = client.VolumeRemove(ctx, vname, false) + testutil.ErrorContains(t, err, "volume is in use") + + err = client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{ + Force: true, + }) + require.NoError(t, err) + + err = client.VolumeRemove(ctx, vname, false) + require.NoError(t, err) +} + +func TestVolumesInspect(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + // sampling current time minus a minute so to now have false positive in case of delays + now := time.Now().Truncate(time.Minute) + + name := t.Name() + _, err := client.VolumeCreate(ctx, volumetypes.VolumesCreateBody{ + Name: name, + }) + require.NoError(t, err) + + vol, err := client.VolumeInspect(ctx, name) + require.NoError(t, err) + + expected := types.Volume{ + // Ignore timestamp of CreatedAt + CreatedAt: vol.CreatedAt, + Driver: "local", + Scope: "local", + Name: name, + Options: map[string]string{}, + Mountpoint: fmt.Sprintf("%s/volumes/%s/_data", testEnv.DaemonInfo.DockerRootDir, name), + } + assert.Equal(t, vol, expected) + + // comparing CreatedAt field time for the new volume to now. Removing a minute from both to avoid false positive + testCreatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(vol.CreatedAt)) + require.NoError(t, err) + testCreatedAt = testCreatedAt.Truncate(time.Minute) + assert.Equal(t, testCreatedAt.Equal(now), true, "Time Volume is CreatedAt not equal to current time") +} + +func getPrefixAndSlashFromDaemonPlatform() (prefix, slash string) { + if testEnv.OSType == "windows" { + return "c:", `\` + } + return "", "/" +} diff --git a/components/engine/internal/test/environment/protect.go b/components/engine/internal/test/environment/protect.go index 482dde60b6..ffbf985b73 100644 --- a/components/engine/internal/test/environment/protect.go +++ b/components/engine/internal/test/environment/protect.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -var frozenImages = []string{"busybox:1.27-glibc", "hello-world:frozen", "debian:jessie"} +var frozenImages = []string{"busybox:latest", "busybox:glibc", "hello-world:frozen", "debian:jessie"} type protectedElements struct { containers map[string]struct{} diff --git a/components/engine/libcontainerd/client_local_windows.go b/components/engine/libcontainerd/client_local_windows.go index bbe127a6d7..6e3454e514 100644 --- a/components/engine/libcontainerd/client_local_windows.go +++ b/components/engine/libcontainerd/client_local_windows.go @@ -82,7 +82,7 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // | | Isolation=Process | Isolation=Hyper-V | // +-----------------+--------------------------------------------+---------------------------------------------------+ // | VolumePath | \\?\\Volume{GUIDa} | | -// | LayerFolderPath | %root%\windowsfilter\containerID | %root%\windowsfilter\containerID (servicing only) | +// | LayerFolderPath | %root%\windowsfilter\containerID | | // | Layers[] | ID=GUIDb;Path=%root%\windowsfilter\layerID | ID=GUIDb;Path=%root%\windowsfilter\layerID | // | HvRuntime | | ImagePath=%root%\BaseLayerID\UtilityVM | // +-----------------+--------------------------------------------+---------------------------------------------------+ @@ -104,7 +104,6 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // "MappedDirectories": [], // "HvPartition": false, // "EndpointList": ["eef2649d-bb17-4d53-9937-295a8efe6f2c"], -// "Servicing": false //} // // Isolation=Hyper-V example: @@ -126,7 +125,6 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // "HvRuntime": { // "ImagePath": "C:\\\\control\\\\windowsfilter\\\\65bf96e5760a09edf1790cb229e2dfb2dbd0fcdc0bf7451bae099106bfbfea0c\\\\UtilityVM" // }, -// "Servicing": false //} func (c *client) Create(_ context.Context, id string, spec *specs.Spec, runtimeOptions interface{}) error { if ctr := c.getContainer(id); ctr != nil { @@ -155,7 +153,6 @@ func (c *client) createWindows(id string, spec *specs.Spec, runtimeOptions inter IgnoreFlushesDuringBoot: spec.Windows.IgnoreFlushesDuringBoot, HostName: spec.Hostname, HvPartition: false, - Servicing: spec.Windows.Servicing, } if spec.Windows.Resources != nil { @@ -324,9 +321,6 @@ func (c *client) createWindows(id string, spec *specs.Spec, runtimeOptions inter waitCh: make(chan struct{}), } - // Start the container. If this is a servicing container, this call - // will block until the container is done with the servicing - // execution. logger.Debug("starting container") if err = hcsContainer.Start(); err != nil { c.logger.WithError(err).Error("failed to start container") @@ -525,9 +519,7 @@ func (c *client) createLinux(id string, spec *specs.Spec, runtimeOptions interfa waitCh: make(chan struct{}), } - // Start the container. If this is a servicing container, this call - // will block until the container is done with the servicing - // execution. + // Start the container. logger.Debug("starting container") if err = hcsContainer.Start(); err != nil { c.logger.WithError(err).Error("failed to start container") @@ -588,14 +580,14 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt ) if ctr.ociSpec.Process != nil { emulateConsole = ctr.ociSpec.Process.Terminal - createStdErrPipe = !ctr.ociSpec.Process.Terminal && !ctr.ociSpec.Windows.Servicing + createStdErrPipe = !ctr.ociSpec.Process.Terminal } createProcessParms := &hcsshim.ProcessConfig{ EmulateConsole: emulateConsole, WorkingDirectory: ctr.ociSpec.Process.Cwd, - CreateStdInPipe: !ctr.ociSpec.Windows.Servicing, - CreateStdOutPipe: !ctr.ociSpec.Windows.Servicing, + CreateStdInPipe: true, + CreateStdOutPipe: true, CreateStdErrPipe: createStdErrPipe, } @@ -655,21 +647,6 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt } logger.WithField("pid", p.pid).Debug("init process started") - // If this is a servicing container, wait on the process synchronously here and - // if it succeeds, wait for it cleanly shutdown and merge into the parent container. - if ctr.ociSpec.Windows.Servicing { - // reapProcess takes the lock - ctr.Unlock() - defer ctr.Lock() - exitCode := c.reapProcess(ctr, p) - - if exitCode != 0 { - return -1, errors.Errorf("libcontainerd: servicing container %s returned non-zero exit code %d", ctr.id, exitCode) - } - - return p.pid, nil - } - dio, err := newIOFromProcess(newProcess, ctr.ociSpec.Process.Terminal) if err != nil { logger.WithError(err).Error("failed to get stdio pipes") @@ -1275,7 +1252,6 @@ func (c *client) reapProcess(ctr *container, p *process) int { eventErr = fmt.Errorf("hcsProcess.Close() failed %s", err) } - var pendingUpdates bool if p.id == InitProcessName { // Update container status ctr.Lock() @@ -1285,16 +1261,6 @@ func (c *client) reapProcess(ctr *container, p *process) int { close(ctr.waitCh) ctr.Unlock() - // Handle any servicing - if exitCode == 0 && ctr.isWindows && !ctr.ociSpec.Windows.Servicing { - pendingUpdates, err = ctr.hcsContainer.HasPendingUpdates() - logger.Infof("Pending updates: %v", pendingUpdates) - if err != nil { - logger.WithError(err). - Warnf("failed to check for pending updates (container may have been killed)") - } - } - if err := c.shutdownContainer(ctr); err != nil { exitCode = -1 logger.WithError(err).Warn("failed to shutdown container") @@ -1320,37 +1286,34 @@ func (c *client) reapProcess(ctr *container, p *process) int { } } - if !(ctr.isWindows && ctr.ociSpec.Windows.Servicing) { - c.eventQ.append(ctr.id, func() { - ei := EventInfo{ - ContainerID: ctr.id, - ProcessID: p.id, - Pid: uint32(p.pid), - ExitCode: uint32(exitCode), - ExitedAt: exitedAt, - UpdatePending: pendingUpdates, - Error: eventErr, - } - c.logger.WithFields(logrus.Fields{ + c.eventQ.append(ctr.id, func() { + ei := EventInfo{ + ContainerID: ctr.id, + ProcessID: p.id, + Pid: uint32(p.pid), + ExitCode: uint32(exitCode), + ExitedAt: exitedAt, + Error: eventErr, + } + c.logger.WithFields(logrus.Fields{ + "container": ctr.id, + "event": EventExit, + "event-info": ei, + }).Info("sending event") + err := c.backend.ProcessEvent(ctr.id, EventExit, ei) + if err != nil { + c.logger.WithError(err).WithFields(logrus.Fields{ "container": ctr.id, "event": EventExit, "event-info": ei, - }).Info("sending event") - err := c.backend.ProcessEvent(ctr.id, EventExit, ei) - if err != nil { - c.logger.WithError(err).WithFields(logrus.Fields{ - "container": ctr.id, - "event": EventExit, - "event-info": ei, - }).Error("failed to process event") - } - if p.id != InitProcessName { - ctr.Lock() - delete(ctr.execs, p.id) - ctr.Unlock() - } - }) - } + }).Error("failed to process event") + } + if p.id != InitProcessName { + ctr.Lock() + delete(ctr.execs, p.id) + ctr.Unlock() + } + }) return exitCode } diff --git a/components/engine/libcontainerd/types.go b/components/engine/libcontainerd/types.go index 97e4fee735..96ffbe2676 100644 --- a/components/engine/libcontainerd/types.go +++ b/components/engine/libcontainerd/types.go @@ -71,9 +71,7 @@ type EventInfo struct { ExitCode uint32 ExitedAt time.Time OOMKilled bool - // Windows Only field - UpdatePending bool - Error error + Error error } // Backend defines callbacks that the client of the library needs to implement.