diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index aa910c751f..30c5dcd291 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -2,14 +2,23 @@ # # Usage: # -# # Assemble the full dev environment. This is slow the first time. -# docker build -t docker . +# # Use make to build a development environment image and run it in a container. +# # This is slow the first time. +# make BIND_DIR=. shell # -# # Mount your source in an interactive container for quick testing: -# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash +# The following commands are executed inside the running container. + +# # Make a dockerd binary. +# # hack/make.sh binary # -# # Run the test suite: -# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py +# # Install dockerd to /usr/local/bin +# # make install +# +# # Run unit tests +# # hack/test/unit +# +# # Run tests e.g. integration, py +# # hack/make.sh binary test-integration test-docker-py # # # Publish a release: # docker run --privileged \ @@ -133,7 +142,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ @@ -146,9 +155,12 @@ RUN pip install yamllint==1.5.0 # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb -RUN git clone https://github.com/go-swagger/go-swagger.git /go/src/github.com/go-swagger/go-swagger \ - && (cd /go/src/github.com/go-swagger/go-swagger && git checkout -q $GO_SWAGGER_COMMIT) \ - && go install -v github.com/go-swagger/go-swagger/cmd/swagger +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \ + && (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \ + && go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \ + && rm -rf "$GOPATH" # Set user.email so crosbymichael's in-container merge commits go smoothly RUN git config --global user.email 'docker-dummy@example.com' @@ -168,10 +180,9 @@ RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ -# TODO: when issue #35963 fixed, we can upgrade the busybox to multi-arch RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index c654207b20..8907384496 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -106,7 +106,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ @@ -119,9 +119,12 @@ RUN pip install yamllint==1.5.0 # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb -RUN git clone https://github.com/go-swagger/go-swagger.git /go/src/github.com/go-swagger/go-swagger \ - && (cd /go/src/github.com/go-swagger/go-swagger && git checkout -q $GO_SWAGGER_COMMIT) \ - && go install -v github.com/go-swagger/go-swagger/cmd/swagger +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \ + && (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \ + && go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \ + && rm -rf "$GOPATH" # Set user.email so crosbymichael's in-container merge commits go smoothly RUN git config --global user.email 'docker-dummy@example.com' @@ -144,7 +147,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 0a8f06c59e..b6a8ed301f 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -104,7 +104,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ @@ -133,7 +133,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index 294a6aa85a..bfac86d291 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -16,10 +16,10 @@ WORKDIR /go/src/github.com/docker/docker/ # Generate frozen images COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \ - buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ - debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ - hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # Download Docker CLI binary COPY hack/dockerfile hack/dockerfile diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 41c7318ce8..690ed491c1 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -102,7 +102,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ @@ -131,7 +131,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index f6cbe7dab3..53a567dcab 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -96,7 +96,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ @@ -125,7 +125,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index 4c831d7832..e9a5566caf 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -23,29 +23,19 @@ # a subsystem, they are responsible for doing so and holding the # subsystem maintainers accountable. If ownership is unclear, they are the de facto owners. - # For each release (including minor releases), a "release captain" is assigned from the - # pool of core maintainers. Rotation is encouraged across all maintainers, to ensure - # the release process is clear and up-to-date. - people = [ "aaronlehmann", "akihirosuda", - "albers", - "aluzzardi", "anusha", "coolljt0725", "cpuguy83", "crosbymichael", "dnephin", "duglin", - "ehazlett", "estesp", - "icecrime", "jhowardmsft", "johnstep", "justincormack", - "lk4d4", - "mavenugo", "mhbauer", "mlaventure", "runcom", @@ -103,6 +93,16 @@ # Thank you! people = [ + # Harald Albers is the mastermind behind the bash completion scripts for the + # Docker CLI. The completion scripts moved to the Docker CLI repository, so + # you can now find him perform his magic in the https://github.com/docker/cli repository. + "albers", + + # Andrea Luzzardi started contributing to the Docker codebase in the "dotCloud" + # era, even before it was called "Docker". He is one of the architects of both + # Swarm and SwarmKit, and its integration into the Docker engine. + "aluzzardi", + # David Calavera contributed many features to Docker, such as an improved # event system, dynamic configuration reloading, volume plugins, fancy # new templating options, and an external client credential store. As a @@ -120,6 +120,24 @@ # still stumble into him in our issue tracker, or on IRC. "erikh", + # Evan Hazlett is the creator of of the Shipyard and Interlock open source projects, + # and the author of "Orca", which became the foundation of Docker Universal Control + # Plane (UCP). As a maintainer, Evan helped integrating SwarmKit (secrets, tasks) + # into the Docker engine. + "ehazlett", + + # Arnaud Porterie (AKA "icecrime") was in charge of maintaining the maintainers. + # As a maintainer, he made life easier for contributors to the Docker open-source + # projects, bringing order in the chaos by designing a triage- and review workflow + # using labels (see https://icecrime.net/technology/a-structured-approach-to-labeling/), + # and automating the hell out of things with his buddies GordonTheTurtle and Poule + # (a chicken!). + # + # A lesser-known fact is that he created the first commit in the libnetwork repository + # even though he didn't know anything about it. Some say, he's now selling stuff on + # the internet ;-) + "icecrime", + # After a false start with his first PR being rejected, James Turnbull became a frequent # contributor to the documentation, and became a docs maintainer on December 5, 2013. As # a maintainer, James lifted the docs to a higher standard, and introduced the community @@ -139,13 +157,24 @@ # containers a lot more secure). Besides being a maintainer, she # set up the CI infrastructure for the project, giving everyone # something to shout at if a PR failed ("noooo Janky!"). - # Jess is currently working on the DCOS security team at Mesosphere, - # and contributing to various open source projects. # Be sure you don't miss her talks at a conference near you (a must-see), # read her blog at https://blog.jessfraz.com (a must-read), and # check out her open source projects on GitHub https://github.com/jessfraz (a must-try). "jessfraz", + # Alexander Morozov contributed many features to Docker, worked on the premise of + # what later became containerd (and worked on that too), and made a "stupid" Go + # vendor tool specificaly for docker/docker needs: vndr (https://github.com/LK4D4/vndr). + # Not many know that Alexander is a master negotiator, being able to change course + # of action with a single "Nope, we're not gonna do that". + "lk4d4", + + # Madhu Venugopal was part of the SocketPlane team that joined Docker. + # As a maintainer, he was working with Jana for the Container Network + # Model (CNM) implemented through libnetwork, and the "routing mesh" powering + # Swarm mode networking. + "mavenugo", + # As a docs maintainer, Mary Anthony contributed greatly to the Docker # docs. She wrote the Docker Contributor Guide and Getting Started # Guides. She helped create a doc build system independent of diff --git a/components/engine/api/server/router/build/build_routes.go b/components/engine/api/server/router/build/build_routes.go index 396797d9d5..09a167062a 100644 --- a/components/engine/api/server/router/build/build_routes.go +++ b/components/engine/api/server/router/build/build_routes.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "os" "runtime" "strconv" "strings" @@ -71,17 +70,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui options.Target = r.FormValue("target") options.RemoteContext = r.FormValue("remote") if versions.GreaterThanOrEqualTo(version, "1.32") { - // TODO @jhowardmsft. The following environment variable is an interim - // measure to allow the daemon to have a default platform if omitted by - // the client. This allows LCOW and WCOW to work with a down-level CLI - // for a short period of time, as the CLI changes can't be merged - // until after the daemon changes have been merged. Once the CLI is - // updated, this can be removed. PR for CLI is currently in - // https://github.com/docker/cli/pull/474. apiPlatform := r.FormValue("platform") - if system.LCOWSupported() && apiPlatform == "" { - apiPlatform = os.Getenv("LCOW_API_PLATFORM_IF_OMITTED") - } p := system.ParsePlatform(apiPlatform) if err := system.ValidatePlatform(p); err != nil { return nil, errdefs.InvalidParameter(errors.Errorf("invalid platform: %s", err)) diff --git a/components/engine/api/server/router/container/backend.go b/components/engine/api/server/router/container/backend.go index b6b1dec94c..5072083c69 100644 --- a/components/engine/api/server/router/container/backend.go +++ b/components/engine/api/server/router/container/backend.go @@ -68,8 +68,13 @@ type systemBackend interface { ContainersPrune(ctx context.Context, pruneFilters filters.Args) (*types.ContainersPruneReport, error) } +type commitBackend interface { + CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error) +} + // Backend is all the methods that need to be implemented to provide container specific functionality. type Backend interface { + commitBackend execBackend copyBackend stateBackend diff --git a/components/engine/api/server/router/container/container.go b/components/engine/api/server/router/container/container.go index c763900c60..358f2bc2c1 100644 --- a/components/engine/api/server/router/container/container.go +++ b/components/engine/api/server/router/container/container.go @@ -61,6 +61,7 @@ func (r *containerRouter) initRoutes() { router.NewPostRoute("/containers/{name:.*}/rename", r.postContainerRename), router.NewPostRoute("/containers/{name:.*}/update", r.postContainerUpdate), router.NewPostRoute("/containers/prune", r.postContainersPrune, router.WithCancel), + router.NewPostRoute("/commit", r.postCommit), // PUT router.NewPutRoute("/containers/{name:.*}/archive", r.putContainersArchive), // DELETE diff --git a/components/engine/api/server/router/container/container_routes.go b/components/engine/api/server/router/container/container_routes.go index 5de01e7639..0527810f23 100644 --- a/components/engine/api/server/router/container/container_routes.go +++ b/components/engine/api/server/router/container/container_routes.go @@ -24,6 +24,45 @@ import ( "golang.org/x/net/websocket" ) +func (s *containerRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := httputils.ParseForm(r); err != nil { + return err + } + + if err := httputils.CheckForJSON(r); err != nil { + return err + } + + // TODO: remove pause arg, and always pause in backend + pause := httputils.BoolValue(r, "pause") + version := httputils.VersionFromContext(ctx) + if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") { + pause = true + } + + config, _, _, err := s.decoder.DecodeConfig(r.Body) + if err != nil && err != io.EOF { //Do not fail if body is empty. + return err + } + + commitCfg := &backend.CreateImageConfig{ + Pause: pause, + Repo: r.Form.Get("repo"), + Tag: r.Form.Get("tag"), + Author: r.Form.Get("author"), + Comment: r.Form.Get("comment"), + Config: config, + Changes: r.Form["changes"], + } + + imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg) + if err != nil { + return err + } + + return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ID: imgID}) +} + func (s *containerRouter) getContainersJSON(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := httputils.ParseForm(r); err != nil { return err diff --git a/components/engine/api/server/router/image/backend.go b/components/engine/api/server/router/image/backend.go index dcf554cef3..e4d4b7d1bb 100644 --- a/components/engine/api/server/router/image/backend.go +++ b/components/engine/api/server/router/image/backend.go @@ -4,7 +4,6 @@ import ( "io" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/registry" @@ -14,22 +13,17 @@ import ( // Backend is all the methods that need to be implemented // to provide image specific functionality. type Backend interface { - containerBackend imageBackend importExportBackend registryBackend } -type containerBackend interface { - CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error) -} - type imageBackend interface { ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) ImageHistory(imageName string) ([]*image.HistoryResponseItem, error) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) LookupImage(name string) (*types.ImageInspect, error) - TagImage(imageName, repository, tag string) error + TagImage(imageName, repository, tag string) (string, error) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) } diff --git a/components/engine/api/server/router/image/image.go b/components/engine/api/server/router/image/image.go index 980bbd2421..6d5d87f63c 100644 --- a/components/engine/api/server/router/image/image.go +++ b/components/engine/api/server/router/image/image.go @@ -1,23 +1,18 @@ package image // import "github.com/docker/docker/api/server/router/image" import ( - "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/server/router" ) // imageRouter is a router to talk with the image controller type imageRouter struct { backend Backend - decoder httputils.ContainerDecoder routes []router.Route } // NewRouter initializes a new image router -func NewRouter(backend Backend, decoder httputils.ContainerDecoder) router.Router { - r := &imageRouter{ - backend: backend, - decoder: decoder, - } +func NewRouter(backend Backend) router.Router { + r := &imageRouter{backend: backend} r.initRoutes() return r } @@ -38,7 +33,6 @@ func (r *imageRouter) initRoutes() { router.NewGetRoute("/images/{name:.*}/history", r.getImagesHistory), router.NewGetRoute("/images/{name:.*}/json", r.getImagesByName), // POST - router.NewPostRoute("/commit", r.postCommit), router.NewPostRoute("/images/load", r.postImagesLoad), router.NewPostRoute("/images/create", r.postImagesCreate, router.WithCancel), router.NewPostRoute("/images/{name:.*}/push", r.postImagesPush, router.WithCancel), diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index 618ccdf0d8..cb497134a2 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -4,15 +4,12 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "net/http" - "os" "strconv" "strings" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/versions" "github.com/docker/docker/errdefs" @@ -25,45 +22,6 @@ import ( "golang.org/x/net/context" ) -func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := httputils.ParseForm(r); err != nil { - return err - } - - if err := httputils.CheckForJSON(r); err != nil { - return err - } - - // TODO: remove pause arg, and always pause in backend - pause := httputils.BoolValue(r, "pause") - version := httputils.VersionFromContext(ctx) - if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") { - pause = true - } - - config, _, _, err := s.decoder.DecodeConfig(r.Body) - if err != nil && err != io.EOF { //Do not fail if body is empty. - return err - } - - commitCfg := &backend.CreateImageConfig{ - Pause: pause, - Repo: r.Form.Get("repo"), - Tag: r.Form.Get("tag"), - Author: r.Form.Get("author"), - Comment: r.Form.Get("comment"), - Config: config, - Changes: r.Form["changes"], - } - - imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg) - if err != nil { - return err - } - - return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ID: imgID}) -} - // Creates an image from Pull or from Import func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { @@ -86,17 +44,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite version := httputils.VersionFromContext(ctx) if versions.GreaterThanOrEqualTo(version, "1.32") { - // TODO @jhowardmsft. The following environment variable is an interim - // measure to allow the daemon to have a default platform if omitted by - // the client. This allows LCOW and WCOW to work with a down-level CLI - // for a short period of time, as the CLI changes can't be merged - // until after the daemon changes have been merged. Once the CLI is - // updated, this can be removed. PR for CLI is currently in - // https://github.com/docker/cli/pull/474. apiPlatform := r.FormValue("platform") - if system.LCOWSupported() && apiPlatform == "" { - apiPlatform = os.Getenv("LCOW_API_PLATFORM_IF_OMITTED") - } platform = system.ParsePlatform(apiPlatform) if err = system.ValidatePlatform(platform); err != nil { err = fmt.Errorf("invalid platform: %s", err) @@ -303,7 +251,7 @@ func (s *imageRouter) postImagesTag(ctx context.Context, w http.ResponseWriter, if err := httputils.ParseForm(r); err != nil { return err } - if err := s.backend.TagImage(vars["name"], r.Form.Get("repo"), r.Form.Get("tag")); err != nil { + if _, err := s.backend.TagImage(vars["name"], r.Form.Get("repo"), r.Form.Get("tag")); err != nil { return err } w.WriteHeader(http.StatusCreated) diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index d73b63a0f0..6665cf032e 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -514,7 +514,7 @@ func initRouter(opts routerOptions) { // we need to add the checkpoint router before the container router or the DELETE gets masked checkpointrouter.NewRouter(opts.daemon, decoder), container.NewRouter(opts.daemon, decoder), - image.NewRouter(opts.daemon, decoder), + image.NewRouter(opts.daemon), systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache), volume.NewRouter(opts.daemon), build.NewRouter(opts.buildBackend, opts.daemon), diff --git a/components/engine/container/container_unix.go b/components/engine/container/container_unix.go index 32805e5271..6f4d91b919 100644 --- a/components/engine/container/container_unix.go +++ b/components/engine/container/container_unix.go @@ -6,13 +6,12 @@ import ( "io/ioutil" "os" + "github.com/containerd/continuity/fs" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" mounttypes "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/pkg/system" "github.com/docker/docker/volume" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" @@ -398,53 +397,15 @@ func (container *Container) DetachAndUnmount(volumeEventLog func(name, action st // copyExistingContents copies from the source to the destination and // ensures the ownership is appropriately set. func copyExistingContents(source, destination string) error { - volList, err := ioutil.ReadDir(source) + dstList, err := ioutil.ReadDir(destination) if err != nil { return err } - if len(volList) > 0 { - srcList, err := ioutil.ReadDir(destination) - if err != nil { - return err - } - if len(srcList) == 0 { - // If the source volume is empty, copies files from the root into the volume - if err := chrootarchive.NewArchiver(nil).CopyWithTar(source, destination); err != nil { - return err - } - } + if len(dstList) != 0 { + // destination is not empty, do not copy + return nil } - return copyOwnership(source, destination) -} - -// copyOwnership copies the permissions and uid:gid of the source file -// to the destination file -func copyOwnership(source, destination string) error { - stat, err := system.Stat(source) - if err != nil { - return err - } - - destStat, err := system.Stat(destination) - if err != nil { - return err - } - - // In some cases, even though UID/GID match and it would effectively be a no-op, - // this can return a permission denied error... for example if this is an NFS - // mount. - // Since it's not really an error that we can't chown to the same UID/GID, don't - // even bother trying in such cases. - if stat.UID() != destStat.UID() || stat.GID() != destStat.GID() { - if err := os.Chown(destination, int(stat.UID()), int(stat.GID())); err != nil { - return err - } - } - - if stat.Mode() != destStat.Mode() { - return os.Chmod(destination, os.FileMode(stat.Mode())) - } - return nil + return fs.CopyDir(destination, source) } // TmpfsMounts returns the list of tmpfs mounts diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 5f2e3c689d..8865d832f3 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/docker/distribution/reference" "github.com/docker/docker/api/types/backend" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder/dockerfile" @@ -176,9 +175,12 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma return "", err } - imageRef, err := daemon.tagCommit(c.Repo, c.Tag, id) - if err != nil { - return "", err + var imageRef string + if c.Repo != "" { + imageRef, err = daemon.TagImage(string(id), c.Repo, c.Tag) + if err != nil { + return "", err + } } daemon.LogContainerEventWithAttributes(container, "commit", map[string]string{ "comment": c.Comment, @@ -247,30 +249,6 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { return id, nil } -// TODO: remove from Daemon, move to api backend -func (daemon *Daemon) tagCommit(repo string, tag string, id image.ID) (string, error) { - imageRef := "" - if repo != "" { - newTag, err := reference.ParseNormalizedNamed(repo) // todo: should move this to API layer - if err != nil { - return "", err - } - if !reference.IsNameOnly(newTag) { - return "", errors.Errorf("unexpected repository name: %s", repo) - } - if tag != "" { - if newTag, err = reference.WithTag(newTag, tag); err != nil { - return "", err - } - } - if err := daemon.TagImageWithReference(id, newTag); err != nil { - return "", err - } - imageRef = reference.FamiliarString(newTag) - } - return imageRef, nil -} - func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) { rwlayer, err := layerStore.GetRWLayer(id) if err != nil { diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index f214ee98a0..69a9745125 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -29,7 +29,6 @@ import ( "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/system" "github.com/docker/go-units" @@ -163,7 +162,7 @@ func (d *Driver) Cleanup() error { return err } - return mount.RecursiveUnmount(d.home) + return nil } func free(p *C.char) { diff --git a/components/engine/daemon/graphdriver/zfs/zfs.go b/components/engine/daemon/graphdriver/zfs/zfs.go index 507fa8402e..7183e69421 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs.go +++ b/components/engine/daemon/graphdriver/zfs/zfs.go @@ -178,9 +178,10 @@ func (d *Driver) String() string { return "zfs" } -// Cleanup is called on daemon shutdown, it is used to clean up any remaining mounts +// Cleanup is called on daemon shutdown, it is a no-op for ZFS. +// TODO(@cpuguy83): Walk layer tree and check mounts? func (d *Driver) Cleanup() error { - return mount.RecursiveUnmount(d.options.mountPath) + return nil } // Status returns information about the ZFS filesystem. It returns a two dimensional array of information diff --git a/components/engine/daemon/image_tag.go b/components/engine/daemon/image_tag.go index 80abd9f158..56b325f66d 100644 --- a/components/engine/daemon/image_tag.go +++ b/components/engine/daemon/image_tag.go @@ -7,23 +7,24 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). -func (daemon *Daemon) TagImage(imageName, repository, tag string) error { +func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error) { imageID, _, err := daemon.GetImageIDAndOS(imageName) if err != nil { - return err + return "", err } newTag, err := reference.ParseNormalizedNamed(repository) if err != nil { - return err + return "", err } if tag != "" { if newTag, err = reference.WithTag(reference.TrimNamed(newTag), tag); err != nil { - return err + return "", err } } - return daemon.TagImageWithReference(imageID, newTag) + err = daemon.TagImageWithReference(imageID, newTag) + return reference.FamiliarString(newTag), err } // TagImageWithReference adds the given reference to the image ID provided. diff --git a/components/engine/daemon/logger/loggerutils/logfile.go b/components/engine/daemon/logger/loggerutils/logfile.go index ff7a3323f9..a9d9d633db 100644 --- a/components/engine/daemon/logger/loggerutils/logfile.go +++ b/components/engine/daemon/logger/loggerutils/logfile.go @@ -192,8 +192,12 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher) for _, f := range files { seekers = append(seekers, f) } - seekers = append(seekers, currentChunk) - tailFile(multireader.MultiReadSeeker(seekers...), watcher, w.createDecoder, config) + if currentChunk.Size() > 0 { + seekers = append(seekers, currentChunk) + } + if len(seekers) > 0 { + tailFile(multireader.MultiReadSeeker(seekers...), watcher, w.createDecoder, config) + } } w.mu.RLock() diff --git a/components/engine/daemon/oci_linux.go b/components/engine/daemon/oci_linux.go index 68243e8d21..2a19dd5db8 100644 --- a/components/engine/daemon/oci_linux.go +++ b/components/engine/daemon/oci_linux.go @@ -604,7 +604,8 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c // // For private volumes any root propagation value should work. pFlag := mountPropagationMap[m.Propagation] - if pFlag == mount.SHARED || pFlag == mount.RSHARED { + switch pFlag { + case mount.SHARED, mount.RSHARED: if err := ensureShared(m.Source); err != nil { return err } @@ -612,13 +613,34 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c if rootpg != mount.SHARED && rootpg != mount.RSHARED { s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.SHARED] } - } else if pFlag == mount.SLAVE || pFlag == mount.RSLAVE { + case mount.SLAVE, mount.RSLAVE: + var fallback bool if err := ensureSharedOrSlave(m.Source); err != nil { - return err + // For backwards compatability purposes, treat mounts from the daemon root + // as special since we automatically add rslave propagation to these mounts + // when the user did not set anything, so we should fallback to the old + // behavior which is to use private propagation which is normally the + // default. + if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) { + return err + } + + cm, ok := c.MountPoints[m.Destination] + if !ok { + return err + } + if cm.Spec.BindOptions != nil && cm.Spec.BindOptions.Propagation != "" { + // This means the user explicitly set a propagation, do not fallback in that case. + return err + } + fallback = true + logrus.WithField("container", c.ID).WithField("source", m.Source).Warn("Falling back to default propagation for bind source in daemon root") } - rootpg := mountPropagationMap[s.Linux.RootfsPropagation] - if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE { - s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE] + if !fallback { + rootpg := mountPropagationMap[s.Linux.RootfsPropagation] + if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE { + s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE] + } } } diff --git a/components/engine/daemon/volumes.go b/components/engine/daemon/volumes.go index 9a2e7d7f6e..a63a90771a 100644 --- a/components/engine/daemon/volumes.go +++ b/components/engine/daemon/volumes.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" mounttypes "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" "github.com/docker/docker/errdefs" @@ -146,6 +147,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo if err != nil { return err } + needsSlavePropagation, err := daemon.validateBindDaemonRoot(bind.Spec) + if err != nil { + return err + } + if needsSlavePropagation { + bind.Propagation = mount.PropagationRSlave + } // #10618 _, tmpfsExists := hostConfig.Tmpfs[bind.Destination] @@ -178,6 +186,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo if err != nil { return errdefs.InvalidParameter(err) } + needsSlavePropagation, err := daemon.validateBindDaemonRoot(mp.Spec) + if err != nil { + return err + } + if needsSlavePropagation { + mp.Propagation = mount.PropagationRSlave + } if binds[mp.Destination] { return duplicateMountPointError(cfg.Target) diff --git a/components/engine/daemon/volumes_linux.go b/components/engine/daemon/volumes_linux.go new file mode 100644 index 0000000000..cf3d9ed159 --- /dev/null +++ b/components/engine/daemon/volumes_linux.go @@ -0,0 +1,36 @@ +package daemon + +import ( + "strings" + + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/errdefs" + "github.com/pkg/errors" +) + +// validateBindDaemonRoot ensures that if a given mountpoint's source is within +// the daemon root path, that the propagation is setup to prevent a container +// from holding private refereneces to a mount within the daemon root, which +// can cause issues when the daemon attempts to remove the mountpoint. +func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) { + if m.Type != mount.TypeBind { + return false, nil + } + + // check if the source is within the daemon root, or if the daemon root is within the source + if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) { + return false, nil + } + + if m.BindOptions == nil { + return true, nil + } + + switch m.BindOptions.Propagation { + case mount.PropagationRSlave, mount.PropagationRShared, "": + return m.BindOptions.Propagation == "", nil + default: + } + + return false, errdefs.InvalidParameter(errors.Errorf(`invalid mount config: must use either propagation mode "rslave" or "rshared" when mount source is within the daemon root, daemon root: %q, bind mount source: %q, propagation: %q`, daemon.root, m.Source, m.BindOptions.Propagation)) +} diff --git a/components/engine/daemon/volumes_linux_test.go b/components/engine/daemon/volumes_linux_test.go new file mode 100644 index 0000000000..72830c3e81 --- /dev/null +++ b/components/engine/daemon/volumes_linux_test.go @@ -0,0 +1,56 @@ +package daemon + +import ( + "path/filepath" + "testing" + + "github.com/docker/docker/api/types/mount" +) + +func TestBindDaemonRoot(t *testing.T) { + t.Parallel() + d := &Daemon{root: "/a/b/c/daemon"} + for _, test := range []struct { + desc string + opts *mount.BindOptions + needsProp bool + err bool + }{ + {desc: "nil propagation settings", opts: nil, needsProp: true, err: false}, + {desc: "empty propagation settings", opts: &mount.BindOptions{}, needsProp: true, err: false}, + {desc: "private propagation", opts: &mount.BindOptions{Propagation: mount.PropagationPrivate}, err: true}, + {desc: "rprivate propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRPrivate}, err: true}, + {desc: "slave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationSlave}, err: true}, + {desc: "rslave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false}, + {desc: "shared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationShared}, err: true}, + {desc: "rshared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false}, + } { + t.Run(test.desc, func(t *testing.T) { + test := test + for desc, source := range map[string]string{ + "source is root": d.root, + "source is subpath": filepath.Join(d.root, "a", "b"), + "source is parent": filepath.Dir(d.root), + "source is /": "/", + } { + t.Run(desc, func(t *testing.T) { + mount := mount.Mount{ + Type: mount.TypeBind, + Source: source, + BindOptions: test.opts, + } + needsProp, err := d.validateBindDaemonRoot(mount) + if (err != nil) != test.err { + t.Fatalf("expected err=%v, got: %v", test.err, err) + } + if test.err { + return + } + if test.needsProp != needsProp { + t.Fatalf("expected needsProp=%v, got: %v", test.needsProp, needsProp) + } + }) + } + }) + } +} diff --git a/components/engine/daemon/volumes_windows.go b/components/engine/daemon/volumes_windows.go index 886679a211..458f2851da 100644 --- a/components/engine/daemon/volumes_windows.go +++ b/components/engine/daemon/volumes_windows.go @@ -3,6 +3,7 @@ package daemon // import "github.com/docker/docker/daemon" import ( "sort" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/volume" @@ -44,3 +45,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er func setBindModeIfNull(bind *volume.MountPoint) { return } + +func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) { + return false, nil +} diff --git a/components/engine/hack/dockerfile/binaries-commits b/components/engine/hack/dockerfile/binaries-commits index ca84b44e58..afab51c42a 100644 --- a/components/engine/hack/dockerfile/binaries-commits +++ b/components/engine/hack/dockerfile/binaries-commits @@ -3,12 +3,12 @@ TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly -RUNC_COMMIT=9f9c96235cc97674e935002fc3d78361b696a69e +RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5 # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=9b55aab90508bd389d7654c4baf173a981477d55 # v1.0.1 +CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2 TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 LIBNETWORK_COMMIT=fcf1c3b5e57833aaaa756ae3c4140ea54da00319 VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384 diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index dda77c94a1..5dd1d8b355 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -400,13 +400,7 @@ func (s *DockerSuite) TestBuildLastModified(c *check.C) { defer server.Close() var out, out2 string - var args []string - // Temopray workaround for #35963. Will remove this when that issue fixed - if runtime.GOARCH == "amd64" { - args = []string{"run", name, "ls", "-le", "/file"} - } else { - args = []string{"run", name, "ls", "-l", "--full-time", "/file"} - } + args := []string{"run", name, "ls", "-l", "--full-time", "/file"} dFmt := `FROM busybox ADD %s/file /` diff --git a/components/engine/integration-cli/docker_cli_diff_test.go b/components/engine/integration-cli/docker_cli_diff_test.go deleted file mode 100644 index 614beee43c..0000000000 --- a/components/engine/integration-cli/docker_cli_diff_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package main - -import ( - "strings" - "time" - - "github.com/docker/docker/integration-cli/checker" - "github.com/docker/docker/integration-cli/cli" - "github.com/go-check/check" -) - -// ensure that an added file shows up in docker diff -func (s *DockerSuite) TestDiffFilenameShownInOutput(c *check.C) { - containerCmd := `mkdir /foo; echo xyzzy > /foo/bar` - out := cli.DockerCmd(c, "run", "-d", "busybox", "sh", "-c", containerCmd).Combined() - - // Wait for it to exit as cannot diff a running container on Windows, and - // it will take a few seconds to exit. Also there's no way in Windows to - // differentiate between an Add or a Modify, and all files are under - // a "Files/" prefix. - containerID := strings.TrimSpace(out) - lookingFor := "A /foo/bar" - if testEnv.OSType == "windows" { - cli.WaitExited(c, containerID, 60*time.Second) - lookingFor = "C Files/foo/bar" - } - - cleanCID := strings.TrimSpace(out) - out = cli.DockerCmd(c, "diff", cleanCID).Combined() - - found := false - for _, line := range strings.Split(out, "\n") { - if strings.Contains(line, lookingFor) { - found = true - break - } - } - c.Assert(found, checker.True) -} - -// test to ensure GH #3840 doesn't occur any more -func (s *DockerSuite) TestDiffEnsureInitLayerFilesAreIgnored(c *check.C) { - testRequires(c, DaemonIsLinux) - // this is a list of files which shouldn't show up in `docker diff` - initLayerFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerenv"} - containerCount := 5 - - // we might not run into this problem from the first run, so start a few containers - for i := 0; i < containerCount; i++ { - containerCmd := `echo foo > /root/bar` - out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", containerCmd) - - cleanCID := strings.TrimSpace(out) - out, _ = dockerCmd(c, "diff", cleanCID) - - for _, filename := range initLayerFiles { - c.Assert(out, checker.Not(checker.Contains), filename) - } - } -} - -func (s *DockerSuite) TestDiffEnsureDefaultDevs(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "sleep", "0") - - cleanCID := strings.TrimSpace(out) - out, _ = dockerCmd(c, "diff", cleanCID) - - expected := map[string]bool{ - "C /dev": true, - "A /dev/full": true, // busybox - "C /dev/ptmx": true, // libcontainer - "A /dev/mqueue": true, - "A /dev/kmsg": true, - "A /dev/fd": true, - "A /dev/ptmx": true, - "A /dev/null": true, - "A /dev/random": true, - "A /dev/stdout": true, - "A /dev/stderr": true, - "A /dev/tty1": true, - "A /dev/stdin": true, - "A /dev/tty": true, - "A /dev/urandom": true, - "A /dev/zero": true, - } - - for _, line := range strings.Split(out, "\n") { - c.Assert(line == "" || expected[line], checker.True, check.Commentf(line)) - } -} - -// https://github.com/docker/docker/pull/14381#discussion_r33859347 -func (s *DockerSuite) TestDiffEmptyArgClientError(c *check.C) { - out, _, err := dockerCmdWithError("diff", "") - c.Assert(err, checker.NotNil) - c.Assert(strings.TrimSpace(out), checker.Contains, "Container name cannot be empty") -} diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 2a652e35e4..776f0e5ba5 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -2239,14 +2239,7 @@ func (s *DockerSuite) TestRunSlowStdoutConsumer(c *check.C) { // alternate to /dev/zero and /dev/stdout. testRequires(c, DaemonIsLinux) - // TODO will remove this if issue #35963 fixed - var args []string - if runtime.GOARCH == "amd64" { - args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | catv"} - } else { - args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"} - } - + args := []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"} cont := exec.Command(dockerBinary, args...) stdout, err := cont.StdoutPipe() diff --git a/components/engine/integration-cli/fixtures/load/frozen.go b/components/engine/integration-cli/fixtures/load/frozen.go index 5701a216a2..7e104fe76c 100644 --- a/components/engine/integration-cli/fixtures/load/frozen.go +++ b/components/engine/integration-cli/fixtures/load/frozen.go @@ -37,6 +37,9 @@ func FrozenImagesLinux(client client.APIClient, images ...string) error { if img == "hello-world:frozen" { srcName = "hello-world:latest" } + if img == "busybox:1.27-glibc" { + img = "busybox:latest" + } loadImages = append(loadImages, struct{ srcName, destName string }{ srcName: srcName, destName: img, diff --git a/components/engine/integration/container/diff_test.go b/components/engine/integration/container/diff_test.go new file mode 100644 index 0000000000..63ac19e9ad --- /dev/null +++ b/components/engine/integration/container/diff_test.go @@ -0,0 +1,98 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "context" + "testing" + "time" + + containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/pkg/archive" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ensure that an added file shows up in docker diff +func TestDiffFilenameShownInOutput(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", `mkdir /foo; echo xyzzy > /foo/bar`)) + + // Wait for it to exit as cannot diff a running container on Windows, and + // it will take a few seconds to exit. Also there's no way in Windows to + // differentiate between an Add or a Modify, and all files are under + // a "Files/" prefix. + lookingFor := containertypes.ContainerChangeResponseItem{Kind: archive.ChangeAdd, Path: "/foo/bar"} + if testEnv.OSType == "windows" { + poll.WaitOn(t, containerIsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond), poll.WithTimeout(60*time.Second)) + lookingFor = containertypes.ContainerChangeResponseItem{Kind: archive.ChangeModify, Path: "Files/foo/bar"} + } + + items, err := client.ContainerDiff(ctx, cID) + require.NoError(t, err) + assert.Contains(t, items, lookingFor) +} + +// test to ensure GH #3840 doesn't occur any more +func TestDiffEnsureInitLayerFilesAreIgnored(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux") + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + // this is a list of files which shouldn't show up in `docker diff` + initLayerFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerenv"} + containerCount := 5 + + // we might not run into this problem from the first run, so start a few containers + for i := 0; i < containerCount; i++ { + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", `echo foo > /root/bar`)) + + items, err := client.ContainerDiff(ctx, cID) + require.NoError(t, err) + for _, item := range items { + assert.NotContains(t, initLayerFiles, item.Path) + } + } +} + +func TestDiffEnsureDefaultDevs(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux") + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + cID := container.Run(t, ctx, client, container.WithCmd("sleep", "0")) + + items, err := client.ContainerDiff(ctx, cID) + require.NoError(t, err) + + expected := []containertypes.ContainerChangeResponseItem{ + {Kind: archive.ChangeModify, Path: "/dev"}, + {Kind: archive.ChangeAdd, Path: "/dev/full"}, // busybox + {Kind: archive.ChangeModify, Path: "/dev/ptmx"}, // libcontainer + {Kind: archive.ChangeAdd, Path: "/dev/mqueue"}, + {Kind: archive.ChangeAdd, Path: "/dev/kmsg"}, + {Kind: archive.ChangeAdd, Path: "/dev/fd"}, + {Kind: archive.ChangeAdd, Path: "/dev/ptmx"}, + {Kind: archive.ChangeAdd, Path: "/dev/null"}, + {Kind: archive.ChangeAdd, Path: "/dev/random"}, + {Kind: archive.ChangeAdd, Path: "/dev/stdout"}, + {Kind: archive.ChangeAdd, Path: "/dev/stderr"}, + {Kind: archive.ChangeAdd, Path: "/dev/tty1"}, + {Kind: archive.ChangeAdd, Path: "/dev/stdin"}, + {Kind: archive.ChangeAdd, Path: "/dev/tty"}, + {Kind: archive.ChangeAdd, Path: "/dev/urandom"}, + } + + for _, item := range items { + assert.Contains(t, expected, item) + } +} diff --git a/components/engine/integration/container/exec_test.go b/components/engine/integration/container/exec_test.go index a14284806d..06835678f0 100644 --- a/components/engine/integration/container/exec_test.go +++ b/components/engine/integration/container/exec_test.go @@ -6,9 +6,8 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/stretchr/testify/require" ) @@ -18,22 +17,9 @@ func TestExec(t *testing.T) { ctx := context.Background() client := request.NewAPIClient(t) - container, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Tty: true, - WorkingDir: "/root", - Cmd: strslice.StrSlice([]string{"top"}), - }, - &container.HostConfig{}, - &network.NetworkingConfig{}, - "foo", - ) - require.NoError(t, err) - err = client.ContainerStart(ctx, container.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + cID := container.Run(t, ctx, client, container.WithTty(true), container.WithWorkingDir("/root")) - id, err := client.ContainerExecCreate(ctx, container.ID, + id, err := client.ContainerExecCreate(ctx, cID, types.ExecConfig{ WorkingDir: "/tmp", Env: strslice.StrSlice([]string{"FOO=BAR"}), diff --git a/components/engine/integration/container/health_test.go b/components/engine/integration/container/health_test.go index a5c62edb57..651cb2b00c 100644 --- a/components/engine/integration/container/health_test.go +++ b/components/engine/integration/container/health_test.go @@ -6,13 +6,11 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types/strslice" + containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/gotestyourself/gotestyourself/poll" - "github.com/stretchr/testify/require" ) // TestHealthCheckWorkdir verifies that health-checks inherit the containers' @@ -22,27 +20,15 @@ func TestHealthCheckWorkdir(t *testing.T) { ctx := context.Background() client := request.NewAPIClient(t) - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Tty: true, - WorkingDir: "/foo", - Cmd: strslice.StrSlice([]string{"top"}), - Healthcheck: &container.HealthConfig{ - Test: []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"}, - Interval: 50 * time.Millisecond, - Retries: 3, - }, - }, - &container.HostConfig{}, - &network.NetworkingConfig{}, - "healthtest", - ) - require.NoError(t, err) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + cID := container.Run(t, ctx, client, container.WithTty(true), container.WithWorkingDir("/foo"), func(c *container.TestContainerConfig) { + c.Config.Healthcheck = &containertypes.HealthConfig{ + Test: []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"}, + Interval: 50 * time.Millisecond, + Retries: 3, + } + }) - poll.WaitOn(t, pollForHealthStatus(ctx, client, c.ID, types.Healthy), poll.WithDelay(100*time.Millisecond)) + poll.WaitOn(t, pollForHealthStatus(ctx, client, cID, types.Healthy), poll.WithDelay(100*time.Millisecond)) } func pollForHealthStatus(ctx context.Context, client client.APIClient, containerID string, healthStatus string) func(log poll.LogT) poll.Result { diff --git a/components/engine/integration/container/links_linux_test.go b/components/engine/integration/container/links_linux_test.go index b1dc654c20..94e87451de 100644 --- a/components/engine/integration/container/links_linux_test.go +++ b/components/engine/integration/container/links_linux_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/docker/docker/pkg/stdcopy" "github.com/gotestyourself/gotestyourself/poll" @@ -28,24 +28,11 @@ func TestLinksEtcHostsContentMatch(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Cmd: []string{"cat", "/etc/hosts"}, - }, - &container.HostConfig{ - NetworkMode: "host", - }, - nil, - "") - require.NoError(t, err) + cID := container.Run(t, ctx, client, container.WithCmd("cat", "/etc/hosts"), container.WithNetworkMode("host")) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond)) - poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond)) - - body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{ + body, err := client.ContainerLogs(ctx, cID, types.ContainerLogsOptions{ ShowStdout: true, }) require.NoError(t, err) diff --git a/components/engine/integration/container/logs_test.go b/components/engine/integration/container/logs_test.go new file mode 100644 index 0000000000..1157da14b8 --- /dev/null +++ b/components/engine/integration/container/logs_test.go @@ -0,0 +1,33 @@ +package container + +import ( + "context" + "io/ioutil" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/pkg/stdcopy" + "github.com/stretchr/testify/assert" +) + +// Regression test for #35370 +// Makes sure that when following we don't get an EOF error when there are no logs +func TestLogsFollowTailEmpty(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + id := container.Run(t, ctx, client, container.WithCmd("sleep", "100000")) + defer client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true}) + + logs, err := client.ContainerLogs(ctx, id, types.ContainerLogsOptions{ShowStdout: true, Tail: "2"}) + if logs != nil { + defer logs.Close() + } + assert.NoError(t, err) + + _, err = stdcopy.StdCopy(ioutil.Discard, ioutil.Discard, logs) + assert.NoError(t, err) +} diff --git a/components/engine/integration/container/mounts_linux_test.go b/components/engine/integration/container/mounts_linux_test.go index 88e52524d7..60d61d9c94 100644 --- a/components/engine/integration/container/mounts_linux_test.go +++ b/components/engine/integration/container/mounts_linux_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "path/filepath" "testing" "github.com/docker/docker/api/types" @@ -12,6 +13,7 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/integration/util/request" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/system" "github.com/gotestyourself/gotestyourself/fs" @@ -51,10 +53,10 @@ func TestContainerShmNoLeak(t *testing.T) { hc := container.HostConfig{ Mounts: []mount.Mount{ { - Type: mount.TypeBind, - Source: d.Root, - Target: "/testdaemonroot", - BindOptions: &mount.BindOptions{Propagation: mount.PropagationRPrivate}}, + Type: mount.TypeBind, + Source: d.Root, + Target: "/testdaemonroot", + }, }, } cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)} @@ -150,3 +152,129 @@ func TestContainerNetworkMountsNoChown(t *testing.T) { require.NoError(t, err) assert.Equal(t, uint32(0), statT.UID(), "bind mounted network file should not change ownership from root") } + +func TestMountDaemonRoot(t *testing.T) { + t.Parallel() + + client := request.NewAPIClient(t) + ctx := context.Background() + info, err := client.Info(ctx) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + desc string + propagation mount.Propagation + expected mount.Propagation + }{ + { + desc: "default", + propagation: "", + expected: mount.PropagationRSlave, + }, + { + desc: "private", + propagation: mount.PropagationPrivate, + }, + { + desc: "rprivate", + propagation: mount.PropagationRPrivate, + }, + { + desc: "slave", + propagation: mount.PropagationSlave, + }, + { + desc: "rslave", + propagation: mount.PropagationRSlave, + expected: mount.PropagationRSlave, + }, + { + desc: "shared", + propagation: mount.PropagationShared, + }, + { + desc: "rshared", + propagation: mount.PropagationRShared, + expected: mount.PropagationRShared, + }, + } { + t.Run(test.desc, func(t *testing.T) { + test := test + t.Parallel() + + propagationSpec := fmt.Sprintf(":%s", test.propagation) + if test.propagation == "" { + propagationSpec = "" + } + bindSpecRoot := info.DockerRootDir + ":" + "/foo" + propagationSpec + bindSpecSub := filepath.Join(info.DockerRootDir, "containers") + ":/foo" + propagationSpec + + for name, hc := range map[string]*container.HostConfig{ + "bind root": {Binds: []string{bindSpecRoot}}, + "bind subpath": {Binds: []string{bindSpecSub}}, + "mount root": { + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: info.DockerRootDir, + Target: "/foo", + BindOptions: &mount.BindOptions{Propagation: test.propagation}, + }, + }, + }, + "mount subpath": { + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: filepath.Join(info.DockerRootDir, "containers"), + Target: "/foo", + BindOptions: &mount.BindOptions{Propagation: test.propagation}, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + hc := hc + t.Parallel() + + c, err := client.ContainerCreate(ctx, &container.Config{ + Image: "busybox", + Cmd: []string{"true"}, + }, hc, nil, "") + + if err != nil { + if test.expected != "" { + t.Fatal(err) + } + // expected an error, so this is ok and should not continue + return + } + if test.expected == "" { + t.Fatal("expected create to fail") + } + + defer func() { + if err := client.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{Force: true}); err != nil { + panic(err) + } + }() + + inspect, err := client.ContainerInspect(ctx, c.ID) + if err != nil { + t.Fatal(err) + } + if len(inspect.Mounts) != 1 { + t.Fatalf("unexpected number of mounts: %+v", inspect.Mounts) + } + + m := inspect.Mounts[0] + if m.Propagation != test.expected { + t.Fatalf("got unexpected propagation mode, expected %q, got: %v", test.expected, m.Propagation) + } + }) + } + }) + } +} diff --git a/components/engine/integration/container/nat_test.go b/components/engine/integration/container/nat_test.go index 0732e2d852..df3451fbc5 100644 --- a/components/engine/integration/container/nat_test.go +++ b/components/engine/integration/container/nat_test.go @@ -12,8 +12,7 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/docker/go-connections/nat" "github.com/gotestyourself/gotestyourself/poll" @@ -67,25 +66,12 @@ func TestNetworkLoopbackNat(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Cmd: []string{"sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())}, - Tty: true, - }, - &container.HostConfig{ - NetworkMode: "container:server", - }, - nil, - "") - require.NoError(t, err) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())), container.WithTty(true), container.WithNetworkMode("container:server")) - poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond)) + poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond)) - body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{ + body, err := client.ContainerLogs(ctx, cID, types.ContainerLogsOptions{ ShowStdout: true, }) require.NoError(t, err) @@ -102,34 +88,19 @@ func startServerContainer(t *testing.T, msg string, port int) string { client := request.NewAPIClient(t) ctx := context.Background() - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Cmd: []string{"sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)}, - ExposedPorts: map[nat.Port]struct{}{ - nat.Port(fmt.Sprintf("%d/tcp", port)): {}, - }, - }, - &container.HostConfig{ - PortBindings: nat.PortMap{ - nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{ - { - HostPort: fmt.Sprintf("%d", port), - }, + cID := container.Run(t, ctx, client, container.WithName("server"), container.WithCmd("sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)), container.WithExposedPorts(fmt.Sprintf("%d/tcp", port)), func(c *container.TestContainerConfig) { + c.HostConfig.PortBindings = nat.PortMap{ + nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{ + { + HostPort: fmt.Sprintf("%d", port), }, }, - }, - &network.NetworkingConfig{}, - "server", - ) - require.NoError(t, err) + } + }) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + poll.WaitOn(t, containerIsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond)) - poll.WaitOn(t, containerIsInState(ctx, client, c.ID, "running"), poll.WithDelay(100*time.Millisecond)) - - return c.ID + return cID } func getExternalAddress(t *testing.T) net.IP { diff --git a/components/engine/integration/container/stats_test.go b/components/engine/integration/container/stats_test.go index 577d446d15..fdf85f44f6 100644 --- a/components/engine/integration/container/stats_test.go +++ b/components/engine/integration/container/stats_test.go @@ -8,8 +8,7 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/gotestyourself/gotestyourself/poll" "github.com/gotestyourself/gotestyourself/skip" @@ -27,23 +26,11 @@ func TestStats(t *testing.T) { info, err := client.Info(ctx) require.NoError(t, err) - c, err := client.ContainerCreate(ctx, - &container.Config{ - Cmd: []string{"top"}, - Image: "busybox", - }, - &container.HostConfig{}, - &network.NetworkingConfig{}, - "", - ) - require.NoError(t, err) + cID := container.Run(t, ctx, client) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + poll.WaitOn(t, containerIsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond)) - poll.WaitOn(t, containerIsInState(ctx, client, c.ID, "running"), poll.WithDelay(100*time.Millisecond)) - - resp, err := client.ContainerStats(context.Background(), c.ID, false) + resp, err := client.ContainerStats(ctx, cID, false) require.NoError(t, err) defer resp.Body.Close() diff --git a/components/engine/integration/internal/container/ops.go b/components/engine/integration/internal/container/ops.go index 940fd68923..e3d538bef1 100644 --- a/components/engine/integration/internal/container/ops.go +++ b/components/engine/integration/internal/container/ops.go @@ -1,6 +1,10 @@ package container -import "github.com/docker/docker/api/types/strslice" +import ( + containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/go-connections/nat" +) // WithName sets the name of the container func WithName(name string) func(*TestContainerConfig) { @@ -22,3 +26,34 @@ func WithCmd(cmds ...string) func(*TestContainerConfig) { c.Config.Cmd = strslice.StrSlice(cmds) } } + +// WithNetworkMode sets the network mode of the container +func WithNetworkMode(mode string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.NetworkMode = containertypes.NetworkMode(mode) + } +} + +// WithExposedPorts sets the exposed ports of the container +func WithExposedPorts(ports ...string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.Config.ExposedPorts = map[nat.Port]struct{}{} + for _, port := range ports { + c.Config.ExposedPorts[nat.Port(port)] = struct{}{} + } + } +} + +// WithTty sets the TTY mode of the container +func WithTty(tty bool) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.Config.Tty = tty + } +} + +// WithWorkingDir sets the working dir of the container +func WithWorkingDir(dir string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.Config.WorkingDir = dir + } +} diff --git a/components/engine/integration/plugin/authz/authz_plugin_test.go b/components/engine/integration/plugin/authz/authz_plugin_test.go index befebe4080..94f7b896a7 100644 --- a/components/engine/integration/plugin/authz/authz_plugin_test.go +++ b/components/engine/integration/plugin/authz/authz_plugin_test.go @@ -19,10 +19,9 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" eventtypes "github.com/docker/docker/api/types/events" - networktypes "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/docker/docker/internal/test/environment" "github.com/docker/docker/pkg/authorization" @@ -91,17 +90,15 @@ func TestAuthZPluginAllowRequest(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) - // Ensure command successful - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) + ctx := context.Background() - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) + // Ensure command successful + cID := container.Run(t, ctx, client) assertURIRecorded(t, ctrl.requestsURIs, "/containers/create") - assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", createResponse.ID)) + assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", cID)) - _, err = client.ServerVersion(context.Background()) + _, err = client.ServerVersion(ctx) require.Nil(t, err) require.Equal(t, 1, ctrl.versionReqCount) require.Equal(t, 1, ctrl.versionResCount) @@ -213,19 +210,17 @@ func TestAuthZPluginAllowEventStream(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) + ctx := context.Background() + startTime := strconv.FormatInt(systemTime(t, client, testEnv).Unix(), 10) events, errs, cancel := systemEventsSince(client, startTime) defer cancel() // Create a container and wait for the creation events - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) - - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) + cID := container.Run(t, ctx, client) for i := 0; i < 100; i++ { - c, err := client.ContainerInspect(context.Background(), createResponse.ID) + c, err := client.ContainerInspect(ctx, cID) require.Nil(t, err) if c.State.Running { break @@ -241,7 +236,7 @@ func TestAuthZPluginAllowEventStream(t *testing.T) { for !created && !started { select { case event := <-events: - if event.Type == eventtypes.ContainerEventType && event.Actor.ID == createResponse.ID { + if event.Type == eventtypes.ContainerEventType && event.Actor.ID == cID { if event.Action == "create" { created = true } @@ -264,7 +259,7 @@ func TestAuthZPluginAllowEventStream(t *testing.T) { // authorization plugin assertURIRecorded(t, ctrl.requestsURIs, "/events") assertURIRecorded(t, ctrl.requestsURIs, "/containers/create") - assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", createResponse.ID)) + assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", cID)) } func systemTime(t *testing.T, client client.APIClient, testEnv *environment.Execution) time.Time { @@ -347,6 +342,8 @@ func TestAuthZPluginEnsureLoadImportWorking(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) + ctx := context.Background() + tmp, err := ioutil.TempDir("", "test-authz-load-import") require.Nil(t, err) defer os.RemoveAll(tmp) @@ -360,13 +357,9 @@ func TestAuthZPluginEnsureLoadImportWorking(t *testing.T) { exportedImagePath := filepath.Join(tmp, "export.tar") - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) + cID := container.Run(t, ctx, client) - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) - - responseReader, err := client.ContainerExport(context.Background(), createResponse.ID) + responseReader, err := client.ContainerExport(context.Background(), cID) require.Nil(t, err) defer responseReader.Close() file, err := os.Create(exportedImagePath) diff --git a/components/engine/integration/plugin/authz/authz_plugin_v2_test.go b/components/engine/integration/plugin/authz/authz_plugin_v2_test.go index 3f07f48f01..5efa421e88 100644 --- a/components/engine/integration/plugin/authz/authz_plugin_v2_test.go +++ b/components/engine/integration/plugin/authz/authz_plugin_v2_test.go @@ -11,11 +11,10 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - networktypes "github.com/docker/docker/api/types/network" volumetypes "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/requirement" "github.com/gotestyourself/gotestyourself/skip" "github.com/stretchr/testify/require" @@ -47,6 +46,8 @@ func TestAuthZPluginV2AllowNonVolumeRequest(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) + ctx := context.Background() + // Install authz plugin err = pluginInstallGrantAllPermissions(client, authzPluginNameWithTag) require.Nil(t, err) @@ -56,13 +57,9 @@ func TestAuthZPluginV2AllowNonVolumeRequest(t *testing.T) { d.LoadBusybox(t) // Ensure docker run command and accompanying docker ps are successful - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) + cID := container.Run(t, ctx, client) - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) - - _, err = client.ContainerInspect(context.Background(), createResponse.ID) + _, err = client.ContainerInspect(ctx, cID) require.Nil(t, err) } diff --git a/components/engine/internal/test/environment/protect.go b/components/engine/internal/test/environment/protect.go index 9aa6fdd602..482dde60b6 100644 --- a/components/engine/internal/test/environment/protect.go +++ b/components/engine/internal/test/environment/protect.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -var frozenImages = []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} +var frozenImages = []string{"busybox:1.27-glibc", "hello-world:frozen", "debian:jessie"} type protectedElements struct { containers map[string]struct{} diff --git a/components/engine/poule.yml b/components/engine/poule.yml index 2abf0df7f0..7afb79eada 100644 --- a/components/engine/poule.yml +++ b/components/engine/poule.yml @@ -107,16 +107,13 @@ users: [ "aaronlehmann", "akihirosuda", - "aluzzardi", "coolljt0725", "cpuguy83", "crosbymichael", "dnephin", "duglin", - "ehazlett", "johnstep", "justincormack", - "lk4d4", "mhbauer", "mlaventure", "runcom", diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index d2e1a21ed9..12f0d352b6 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -66,7 +66,7 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.3.0 # When updating, also update RUNC_COMMIT in hack/dockerfile/binaries-commits accordingly -github.com/opencontainers/runc 9f9c96235cc97674e935002fc3d78361b696a69e +github.com/opencontainers/runc 6c55f98695e902427906eed2c799e566e3d3dfb5 github.com/opencontainers/runtime-spec v1.0.1 github.com/opencontainers/image-spec v1.0.1 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 @@ -106,13 +106,13 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 # containerd github.com/containerd/containerd 3fa104f843ec92328912e042b767d26825f202aa github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6 -github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83 -github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f +github.com/containerd/continuity 992a5f112bd2211d0983a1cc8562d2882848f3a3 +github.com/containerd/cgroups c0710c92e8b3a44681d1321dcfd1360fc5c6c089 github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e -github.com/containerd/go-runc ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7 +github.com/containerd/go-runc 4f6e87ae043f859a38255247b49c9abc262d002f github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788 github.com/dmcgowan/go-tar go1.10 -github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f +github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577 # cluster github.com/docker/swarmkit 68a376dc30d8c4001767c39456b990dbd821371b diff --git a/components/engine/vendor/github.com/containerd/cgroups/blkio.go b/components/engine/vendor/github.com/containerd/cgroups/blkio.go index 078d12b2a4..9b15b8a62f 100644 --- a/components/engine/vendor/github.com/containerd/cgroups/blkio.go +++ b/components/engine/vendor/github.com/containerd/cgroups/blkio.go @@ -3,12 +3,12 @@ package cgroups import ( "bufio" "fmt" + "io" "io/ioutil" "os" "path/filepath" "strconv" "strings" - "syscall" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -105,8 +105,13 @@ func (b *blkioController) Stat(path string, stats *Metrics) error { }, ) } + f, err := os.Open("/proc/diskstats") + if err != nil { + return err + } + defer f.Close() - devices, err := getDevices("/dev") + devices, err := getDevices(f) if err != nil { return err } @@ -268,50 +273,32 @@ type deviceKey struct { // getDevices makes a best effort attempt to read all the devices into a map // keyed by major and minor number. Since devices may be mapped multiple times, // we err on taking the first occurrence. -func getDevices(path string) (map[deviceKey]string, error) { - // TODO(stevvooe): We are ignoring lots of errors. It might be kind of - // challenging to debug this if we aren't mapping devices correctly. - // Consider logging these errors. - devices := map[deviceKey]string{} - if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error { +func getDevices(r io.Reader) (map[deviceKey]string, error) { + + var ( + s = bufio.NewScanner(r) + devices = make(map[deviceKey]string) + ) + for s.Scan() { + fields := strings.Fields(s.Text()) + major, err := strconv.Atoi(fields[0]) if err != nil { - return err + return nil, err } - switch { - case fi.IsDir(): - switch fi.Name() { - case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts": - return filepath.SkipDir - default: - return nil - } - case fi.Name() == "console": - return nil - default: - if fi.Mode()&os.ModeDevice == 0 { - // skip non-devices - return nil - } - - st, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("%s: unable to convert to system stat", p) - } - - key := deviceKey{major(st.Rdev), minor(st.Rdev)} - if _, ok := devices[key]; ok { - return nil // skip it if we have already populated the path. - } - - devices[key] = p + minor, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err } - - return nil - }); err != nil { - return nil, err + key := deviceKey{ + major: uint64(major), + minor: uint64(minor), + } + if _, ok := devices[key]; ok { + continue + } + devices[key] = filepath.Join("/dev", fields[2]) } - - return devices, nil + return devices, s.Err() } func major(devNumber uint64) uint64 { diff --git a/components/engine/vendor/github.com/containerd/cgroups/errors.go b/components/engine/vendor/github.com/containerd/cgroups/errors.go index d18b4b1df6..a5824fe23f 100644 --- a/components/engine/vendor/github.com/containerd/cgroups/errors.go +++ b/components/engine/vendor/github.com/containerd/cgroups/errors.go @@ -12,7 +12,7 @@ var ( ErrFreezerNotSupported = errors.New("cgroups: freezer cgroup not supported on this system") ErrMemoryNotSupported = errors.New("cgroups: memory cgroup not supported on this system") ErrCgroupDeleted = errors.New("cgroups: cgroup deleted") - ErrNoCgroupMountDestination = errors.New("cgroups: cannot found cgroup mount destination") + ErrNoCgroupMountDestination = errors.New("cgroups: cannot find cgroup mount destination") ) // ErrorHandler is a function that handles and acts on errors diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go deleted file mode 100644 index 5041e66611..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go +++ /dev/null @@ -1,15 +0,0 @@ -package devices - -// from /usr/include/sys/types.h - -func getmajor(dev int32) uint64 { - return (uint64(dev) >> 24) & 0xff -} - -func getminor(dev int32) uint64 { - return uint64(dev) & 0xffffff -} - -func makedev(major int, minor int) int { - return ((major << 24) | minor) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go deleted file mode 100644 index 9a48330a56..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build solaris,!cgo - -// -// Implementing the functions below requires cgo support. Non-cgo stubs -// versions are defined below to enable cross-compilation of source code -// that depends on these functions, but the resultant cross-compiled -// binaries cannot actually be used. If the stub function(s) below are -// actually invoked they will cause the calling process to exit. -// - -package devices - -func getmajor(dev uint64) uint64 { - panic("getmajor() support requires cgo.") -} - -func getminor(dev uint64) uint64 { - panic("getminor() support requires cgo.") -} - -func makedev(major int, minor int) int { - panic("makedev() support requires cgo.") -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go deleted file mode 100644 index a5c7b93189..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go +++ /dev/null @@ -1,15 +0,0 @@ -package devices - -// from /usr/include/sys/types.h - -func getmajor(dev uint32) uint64 { - return (uint64(dev) >> 24) & 0xff -} - -func getminor(dev uint32) uint64 { - return uint64(dev) & 0xffffff -} - -func makedev(major int, minor int) int { - return ((major << 24) | minor) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go deleted file mode 100644 index 454cf668f5..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go +++ /dev/null @@ -1,15 +0,0 @@ -package devices - -// from /usr/include/linux/kdev_t.h - -func getmajor(dev uint64) uint64 { - return dev >> 8 -} - -func getminor(dev uint64) uint64 { - return dev & 0xff -} - -func makedev(major int, minor int) int { - return ((major << 8) | minor) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go deleted file mode 100644 index 8819ac82f5..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build cgo - -package devices - -//#include -import "C" - -func getmajor(dev uint64) uint64 { - return uint64(C.major(C.dev_t(dev))) -} - -func getminor(dev uint64) uint64 { - return uint64(C.minor(C.dev_t(dev))) -} - -func makedev(major int, minor int) int { - return int(C.makedev(C.major_t(major), C.minor_t(minor))) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go index 85e9a68c49..97fe6b19d2 100644 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go +++ b/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "syscall" + + "golang.org/x/sys/unix" ) func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) { @@ -14,42 +16,43 @@ func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) { return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo") } - return getmajor(sys.Rdev), getminor(sys.Rdev), nil + dev := uint64(sys.Rdev) + return uint64(unix.Major(dev)), uint64(unix.Minor(dev)), nil } // mknod provides a shortcut for syscall.Mknod func Mknod(p string, mode os.FileMode, maj, min int) error { var ( m = syscallMode(mode.Perm()) - dev int + dev uint64 ) if mode&os.ModeDevice != 0 { - dev = makedev(maj, min) + dev = unix.Mkdev(uint32(maj), uint32(min)) if mode&os.ModeCharDevice != 0 { - m |= syscall.S_IFCHR + m |= unix.S_IFCHR } else { - m |= syscall.S_IFBLK + m |= unix.S_IFBLK } } else if mode&os.ModeNamedPipe != 0 { - m |= syscall.S_IFIFO + m |= unix.S_IFIFO } - return syscall.Mknod(p, m, dev) + return unix.Mknod(p, m, int(dev)) } // syscallMode returns the syscall-specific mode bits from Go's portable mode bits. func syscallMode(i os.FileMode) (o uint32) { o |= uint32(i.Perm()) if i&os.ModeSetuid != 0 { - o |= syscall.S_ISUID + o |= unix.S_ISUID } if i&os.ModeSetgid != 0 { - o |= syscall.S_ISGID + o |= unix.S_ISGID } if i&os.ModeSticky != 0 { - o |= syscall.S_ISVTX + o |= unix.S_ISVTX } return } diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy.go new file mode 100644 index 0000000000..e8f452819b --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy.go @@ -0,0 +1,119 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/pkg/errors" +) + +var bufferPool = &sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 32*1024) + return &buffer + }, +} + +// CopyDir copies the directory from src to dst. +// Most efficient copy of files is attempted. +func CopyDir(dst, src string) error { + inodes := map[uint64]string{} + return copyDirectory(dst, src, inodes) +} + +func copyDirectory(dst, src string, inodes map[uint64]string) error { + stat, err := os.Stat(src) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", src) + } + if !stat.IsDir() { + return errors.Errorf("source is not directory") + } + + if st, err := os.Stat(dst); err != nil { + if err := os.Mkdir(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to mkdir %s", dst) + } + } else if !st.IsDir() { + return errors.Errorf("cannot copy to non-directory: %s", dst) + } else { + if err := os.Chmod(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod on %s", dst) + } + } + + fis, err := ioutil.ReadDir(src) + if err != nil { + return errors.Wrapf(err, "failed to read %s", src) + } + + if err := copyFileInfo(stat, dst); err != nil { + return errors.Wrapf(err, "failed to copy file info for %s", dst) + } + + for _, fi := range fis { + source := filepath.Join(src, fi.Name()) + target := filepath.Join(dst, fi.Name()) + + switch { + case fi.IsDir(): + if err := copyDirectory(target, source, inodes); err != nil { + return err + } + continue + case (fi.Mode() & os.ModeType) == 0: + link, err := getLinkSource(target, fi, inodes) + if err != nil { + return errors.Wrap(err, "failed to get hardlink") + } + if link != "" { + if err := os.Link(link, target); err != nil { + return errors.Wrap(err, "failed to create hard link") + } + } else if err := copyFile(source, target); err != nil { + return errors.Wrap(err, "failed to copy files") + } + case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink: + link, err := os.Readlink(source) + if err != nil { + return errors.Wrapf(err, "failed to read link: %s", source) + } + if err := os.Symlink(link, target); err != nil { + return errors.Wrapf(err, "failed to create symlink: %s", target) + } + case (fi.Mode() & os.ModeDevice) == os.ModeDevice: + if err := copyDevice(target, fi); err != nil { + return errors.Wrapf(err, "failed to create device") + } + default: + // TODO: Support pipes and sockets + return errors.Wrapf(err, "unsupported mode %s", fi.Mode()) + } + if err := copyFileInfo(fi, target); err != nil { + return errors.Wrap(err, "failed to copy file info") + } + + if err := copyXAttrs(target, source); err != nil { + return errors.Wrap(err, "failed to copy xattrs") + } + } + + return nil +} + +func copyFile(source, target string) error { + src, err := os.Open(source) + if err != nil { + return errors.Wrapf(err, "failed to open source %s", source) + } + defer src.Close() + tgt, err := os.Create(target) + if err != nil { + return errors.Wrapf(err, "failed to open target %s", target) + } + defer tgt.Close() + + return copyFileContent(tgt, src) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go new file mode 100644 index 0000000000..cfab6756b8 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go @@ -0,0 +1,95 @@ +package fs + +import ( + "io" + "os" + "syscall" + + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + st := fi.Sys().(*syscall.Stat_t) + if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { + if os.IsPermission(err) { + // Normally if uid/gid are the same this would be a no-op, but some + // filesystems may still return EPERM... for instance NFS does this. + // In such a case, this is not an error. + if dstStat, err2 := os.Lstat(name); err2 == nil { + st2 := dstStat.Sys().(*syscall.Stat_t) + if st.Uid == st2.Uid && st.Gid == st2.Gid { + err = nil + } + } + } + if err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + } + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + } + + timespec := []unix.Timespec{unix.Timespec(StatAtime(st)), unix.Timespec(StatMtime(st))} + if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return errors.Wrapf(err, "failed to utime %s", name) + } + + return nil +} + +func copyFileContent(dst, src *os.File) error { + st, err := src.Stat() + if err != nil { + return errors.Wrap(err, "unable to stat source") + } + + n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0) + if err != nil { + if err != unix.ENOSYS && err != unix.EXDEV { + return errors.Wrap(err, "copy file range failed") + } + + buf := bufferPool.Get().(*[]byte) + _, err = io.CopyBuffer(dst, src, *buf) + bufferPool.Put(buf) + return err + } + + if int64(n) != st.Size() { + return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size()) + } + + return nil +} + +func copyXAttrs(dst, src string) error { + xattrKeys, err := sysx.LListxattr(src) + if err != nil { + return errors.Wrapf(err, "failed to list xattrs on %s", src) + } + for _, xattr := range xattrKeys { + data, err := sysx.LGetxattr(src, xattr) + if err != nil { + return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + } + if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { + return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + } + } + + return nil +} + +func copyDevice(dst string, fi os.FileInfo) error { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("unsupported stat type") + } + return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev)) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go new file mode 100644 index 0000000000..29cbb81ed5 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go @@ -0,0 +1,80 @@ +// +build solaris darwin freebsd + +package fs + +import ( + "io" + "os" + "syscall" + + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + st := fi.Sys().(*syscall.Stat_t) + if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { + if os.IsPermission(err) { + // Normally if uid/gid are the same this would be a no-op, but some + // filesystems may still return EPERM... for instance NFS does this. + // In such a case, this is not an error. + if dstStat, err2 := os.Lstat(name); err2 == nil { + st2 := dstStat.Sys().(*syscall.Stat_t) + if st.Uid == st2.Uid && st.Gid == st2.Gid { + err = nil + } + } + } + if err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + } + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + } + + timespec := []syscall.Timespec{StatAtime(st), StatMtime(st)} + if err := syscall.UtimesNano(name, timespec); err != nil { + return errors.Wrapf(err, "failed to utime %s", name) + } + + return nil +} + +func copyFileContent(dst, src *os.File) error { + buf := bufferPool.Get().(*[]byte) + _, err := io.CopyBuffer(dst, src, *buf) + bufferPool.Put(buf) + + return err +} + +func copyXAttrs(dst, src string) error { + xattrKeys, err := sysx.LListxattr(src) + if err != nil { + return errors.Wrapf(err, "failed to list xattrs on %s", src) + } + for _, xattr := range xattrKeys { + data, err := sysx.LGetxattr(src, xattr) + if err != nil { + return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + } + if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { + return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + } + } + + return nil +} + +func copyDevice(dst string, fi os.FileInfo) error { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("unsupported stat type") + } + return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev)) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go new file mode 100644 index 0000000000..6fb3de5710 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go @@ -0,0 +1,33 @@ +package fs + +import ( + "io" + "os" + + "github.com/pkg/errors" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + + // TODO: copy windows specific metadata + + return nil +} + +func copyFileContent(dst, src *os.File) error { + buf := bufferPool.Get().(*[]byte) + _, err := io.CopyBuffer(dst, src, *buf) + bufferPool.Put(buf) + return err +} + +func copyXAttrs(dst, src string) error { + return nil +} + +func copyDevice(dst string, fi os.FileInfo) error { + return errors.New("device copy not supported") +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/diff.go b/components/engine/vendor/github.com/containerd/continuity/fs/diff.go new file mode 100644 index 0000000000..f2300e845d --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/diff.go @@ -0,0 +1,310 @@ +package fs + +import ( + "context" + "os" + "path/filepath" + "strings" + + "golang.org/x/sync/errgroup" + + "github.com/sirupsen/logrus" +) + +// ChangeKind is the type of modification that +// a change is making. +type ChangeKind int + +const ( + // ChangeKindUnmodified represents an unmodified + // file + ChangeKindUnmodified = iota + + // ChangeKindAdd represents an addition of + // a file + ChangeKindAdd + + // ChangeKindModify represents a change to + // an existing file + ChangeKindModify + + // ChangeKindDelete represents a delete of + // a file + ChangeKindDelete +) + +func (k ChangeKind) String() string { + switch k { + case ChangeKindUnmodified: + return "unmodified" + case ChangeKindAdd: + return "add" + case ChangeKindModify: + return "modify" + case ChangeKindDelete: + return "delete" + default: + return "" + } +} + +// Change represents single change between a diff and its parent. +type Change struct { + Kind ChangeKind + Path string +} + +// ChangeFunc is the type of function called for each change +// computed during a directory changes calculation. +type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error + +// Changes computes changes between two directories calling the +// given change function for each computed change. The first +// directory is intended to the base directory and second +// directory the changed directory. +// +// The change callback is called by the order of path names and +// should be appliable in that order. +// Due to this apply ordering, the following is true +// - Removed directory trees only create a single change for the root +// directory removed. Remaining changes are implied. +// - A directory which is modified to become a file will not have +// delete entries for sub-path items, their removal is implied +// by the removal of the parent directory. +// +// Opaque directories will not be treated specially and each file +// removed from the base directory will show up as a removal. +// +// File content comparisons will be done on files which have timestamps +// which may have been truncated. If either of the files being compared +// has a zero value nanosecond value, each byte will be compared for +// differences. If 2 files have the same seconds value but different +// nanosecond values where one of those values is zero, the files will +// be considered unchanged if the content is the same. This behavior +// is to account for timestamp truncation during archiving. +func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error { + if a == "" { + logrus.Debugf("Using single walk diff for %s", b) + return addDirChanges(ctx, changeFn, b) + } else if diffOptions := detectDirDiff(b, a); diffOptions != nil { + logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a) + return diffDirChanges(ctx, changeFn, a, diffOptions) + } + + logrus.Debugf("Using double walk diff for %s from %s", b, a) + return doubleWalkDiff(ctx, changeFn, a, b) +} + +func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + return changeFn(ChangeKindAdd, path, f, nil) + }) +} + +// diffDirOptions is used when the diff can be directly calculated from +// a diff directory to its base, without walking both trees. +type diffDirOptions struct { + diffDir string + skipChange func(string) (bool, error) + deleteChange func(string, string, os.FileInfo) (string, error) +} + +// diffDirChanges walks the diff directory and compares changes against the base. +func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error { + changedDirs := make(map[string]struct{}) + return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(o.diffDir, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + // TODO: handle opaqueness, start new double walker at this + // location to get deletes, and skip tree in single walker + + if o.skipChange != nil { + if skip, err := o.skipChange(path); skip { + return err + } + } + + var kind ChangeKind + + deletedFile, err := o.deleteChange(o.diffDir, path, f) + if err != nil { + return err + } + + // Find out what kind of modification happened + if deletedFile != "" { + path = deletedFile + kind = ChangeKindDelete + f = nil + } else { + // Otherwise, the file was added + kind = ChangeKindAdd + + // ...Unless it already existed in a base, in which case, it's a modification + stat, err := os.Stat(filepath.Join(base, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the base, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { + // Both directories are the same, don't record the change + return nil + } + } + kind = ChangeKindModify + } + } + + // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files. + // This block is here to ensure the change is recorded even if the + // modify time, mode and size of the parent directory in the rw and ro layers are all equal. + // Check https://github.com/docker/docker/pull/13590 for details. + if f.IsDir() { + changedDirs[path] = struct{}{} + } + if kind == ChangeKindAdd || kind == ChangeKindDelete { + parent := filepath.Dir(path) + if _, ok := changedDirs[parent]; !ok && parent != "/" { + pi, err := os.Stat(filepath.Join(o.diffDir, parent)) + if err := changeFn(ChangeKindModify, parent, pi, err); err != nil { + return err + } + changedDirs[parent] = struct{}{} + } + } + + return changeFn(kind, path, f, nil) + }) +} + +// doubleWalkDiff walks both directories to create a diff +func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) { + g, ctx := errgroup.WithContext(ctx) + + var ( + c1 = make(chan *currentPath) + c2 = make(chan *currentPath) + + f1, f2 *currentPath + rmdir string + ) + g.Go(func() error { + defer close(c1) + return pathWalk(ctx, a, c1) + }) + g.Go(func() error { + defer close(c2) + return pathWalk(ctx, b, c2) + }) + g.Go(func() error { + for c1 != nil || c2 != nil { + if f1 == nil && c1 != nil { + f1, err = nextPath(ctx, c1) + if err != nil { + return err + } + if f1 == nil { + c1 = nil + } + } + + if f2 == nil && c2 != nil { + f2, err = nextPath(ctx, c2) + if err != nil { + return err + } + if f2 == nil { + c2 = nil + } + } + if f1 == nil && f2 == nil { + continue + } + + var f os.FileInfo + k, p := pathChange(f1, f2) + switch k { + case ChangeKindAdd: + if rmdir != "" { + rmdir = "" + } + f = f2.f + f2 = nil + case ChangeKindDelete: + // Check if this file is already removed by being + // under of a removed directory + if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { + f1 = nil + continue + } else if f1.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f1 = nil + case ChangeKindModify: + same, err := sameFile(f1, f2) + if err != nil { + return err + } + if f1.f.IsDir() && !f2.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f = f2.f + f1 = nil + f2 = nil + if same { + if !isLinked(f) { + continue + } + k = ChangeKindUnmodified + } + } + if err := changeFn(k, p, f, nil); err != nil { + return err + } + } + return nil + }) + + return g.Wait() +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go new file mode 100644 index 0000000000..3751814443 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go @@ -0,0 +1,58 @@ +// +build !windows + +package fs + +import ( + "bytes" + "os" + "syscall" + + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" +) + +// detectDirDiff returns diff dir options if a directory could +// be found in the mount info for upper which is the direct +// diff with the provided lower directory +func detectDirDiff(upper, lower string) *diffDirOptions { + // TODO: get mount options for upper + // TODO: detect AUFS + // TODO: detect overlay + return nil +} + +// compareSysStat returns whether the stats are equivalent, +// whether the files are considered the same file, and +// an error +func compareSysStat(s1, s2 interface{}) (bool, error) { + ls1, ok := s1.(*syscall.Stat_t) + if !ok { + return false, nil + } + ls2, ok := s2.(*syscall.Stat_t) + if !ok { + return false, nil + } + + return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + c1, err := sysx.LGetxattr(p1, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p1) + } + c2, err := sysx.LGetxattr(p2, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p2) + } + return bytes.Equal(c1, c2), nil +} + +func isLinked(f os.FileInfo) bool { + s, ok := f.Sys().(*syscall.Stat_t) + if !ok { + return false + } + return !f.IsDir() && s.Nlink > 1 +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go new file mode 100644 index 0000000000..8eed36507e --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go @@ -0,0 +1,32 @@ +package fs + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func detectDirDiff(upper, lower string) *diffDirOptions { + return nil +} + +func compareSysStat(s1, s2 interface{}) (bool, error) { + f1, ok := s1.(windows.Win32FileAttributeData) + if !ok { + return false, nil + } + f2, ok := s2.(windows.Win32FileAttributeData) + if !ok { + return false, nil + } + return f1.FileAttributes == f2.FileAttributes, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + // TODO: Use windows equivalent + return true, nil +} + +func isLinked(os.FileInfo) bool { + return false +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go b/components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go new file mode 100644 index 0000000000..cc06573f1b --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go @@ -0,0 +1,87 @@ +// +build linux + +package fs + +import ( + "fmt" + "io/ioutil" + "os" + "syscall" + "unsafe" +) + +func locateDummyIfEmpty(path string) (string, error) { + children, err := ioutil.ReadDir(path) + if err != nil { + return "", err + } + if len(children) != 0 { + return "", nil + } + dummyFile, err := ioutil.TempFile(path, "fsutils-dummy") + if err != nil { + return "", err + } + name := dummyFile.Name() + err = dummyFile.Close() + return name, err +} + +// SupportsDType returns whether the filesystem mounted on path supports d_type +func SupportsDType(path string) (bool, error) { + // locate dummy so that we have at least one dirent + dummy, err := locateDummyIfEmpty(path) + if err != nil { + return false, err + } + if dummy != "" { + defer os.Remove(dummy) + } + + visited := 0 + supportsDType := true + fn := func(ent *syscall.Dirent) bool { + visited++ + if ent.Type == syscall.DT_UNKNOWN { + supportsDType = false + // stop iteration + return true + } + // continue iteration + return false + } + if err = iterateReadDir(path, fn); err != nil { + return false, err + } + if visited == 0 { + return false, fmt.Errorf("did not hit any dirent during iteration %s", path) + } + return supportsDType, nil +} + +func iterateReadDir(path string, fn func(*syscall.Dirent) bool) error { + d, err := os.Open(path) + if err != nil { + return err + } + defer d.Close() + fd := int(d.Fd()) + buf := make([]byte, 4096) + for { + nbytes, err := syscall.ReadDirent(fd, buf) + if err != nil { + return err + } + if nbytes == 0 { + break + } + for off := 0; off < nbytes; { + ent := (*syscall.Dirent)(unsafe.Pointer(&buf[off])) + if stop := fn(ent); stop { + return nil + } + off += int(ent.Reclen) + } + } + return nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/du.go b/components/engine/vendor/github.com/containerd/continuity/fs/du.go new file mode 100644 index 0000000000..26f5333154 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/du.go @@ -0,0 +1,22 @@ +package fs + +import "context" + +// Usage of disk information +type Usage struct { + Inodes int64 + Size int64 +} + +// DiskUsage counts the number of inodes and disk usage for the resources under +// path. +func DiskUsage(roots ...string) (Usage, error) { + return diskUsage(roots...) +} + +// DiffUsage counts the numbers of inodes and disk usage in the +// diff between the 2 directories. The first path is intended +// as the base directory and the second as the changed directory. +func DiffUsage(ctx context.Context, a, b string) (Usage, error) { + return diffUsage(ctx, a, b) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go new file mode 100644 index 0000000000..fe3426d278 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go @@ -0,0 +1,88 @@ +// +build !windows + +package fs + +import ( + "context" + "os" + "path/filepath" + "syscall" +) + +type inode struct { + // TODO(stevvooe): Can probably reduce memory usage by not tracking + // device, but we can leave this right for now. + dev, ino uint64 +} + +func newInode(stat *syscall.Stat_t) inode { + return inode{ + // Dev is uint32 on darwin/bsd, uint64 on linux/solaris + dev: uint64(stat.Dev), // nolint: unconvert + // Ino is uint32 on bsd, uint64 on darwin/linux/solaris + ino: uint64(stat.Ino), // nolint: unconvert + } +} + +func diskUsage(roots ...string) (Usage, error) { + + var ( + size int64 + inodes = map[inode]struct{}{} // expensive! + ) + + for _, root := range roots { + if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + inoKey := newInode(fi.Sys().(*syscall.Stat_t)) + if _, ok := inodes[inoKey]; !ok { + inodes[inoKey] = struct{}{} + size += fi.Size() + } + + return nil + }); err != nil { + return Usage{}, err + } + } + + return Usage{ + Inodes: int64(len(inodes)), + Size: size, + }, nil +} + +func diffUsage(ctx context.Context, a, b string) (Usage, error) { + var ( + size int64 + inodes = map[inode]struct{}{} // expensive! + ) + + if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if kind == ChangeKindAdd || kind == ChangeKindModify { + inoKey := newInode(fi.Sys().(*syscall.Stat_t)) + if _, ok := inodes[inoKey]; !ok { + inodes[inoKey] = struct{}{} + size += fi.Size() + } + + return nil + + } + return nil + }); err != nil { + return Usage{}, err + } + + return Usage{ + Inodes: int64(len(inodes)), + Size: size, + }, nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go new file mode 100644 index 0000000000..3f852fc15e --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go @@ -0,0 +1,60 @@ +// +build windows + +package fs + +import ( + "context" + "os" + "path/filepath" +) + +func diskUsage(roots ...string) (Usage, error) { + var ( + size int64 + ) + + // TODO(stevvooe): Support inodes (or equivalent) for windows. + + for _, root := range roots { + if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + size += fi.Size() + return nil + }); err != nil { + return Usage{}, err + } + } + + return Usage{ + Size: size, + }, nil +} + +func diffUsage(ctx context.Context, a, b string) (Usage, error) { + var ( + size int64 + ) + + if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if kind == ChangeKindAdd || kind == ChangeKindModify { + size += fi.Size() + + return nil + + } + return nil + }); err != nil { + return Usage{}, err + } + + return Usage{ + Size: size, + }, nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go new file mode 100644 index 0000000000..38da93813c --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go @@ -0,0 +1,27 @@ +package fs + +import "os" + +// GetLinkInfo returns an identifier representing the node a hardlink is pointing +// to. If the file is not hard linked then 0 will be returned. +func GetLinkInfo(fi os.FileInfo) (uint64, bool) { + return getLinkInfo(fi) +} + +// getLinkSource returns a path for the given name and +// file info to its link source in the provided inode +// map. If the given file name is not in the map and +// has other links, it is added to the inode map +// to be a source for other link locations. +func getLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) { + inode, isHardlink := getLinkInfo(fi) + if !isHardlink { + return "", nil + } + + path, ok := inodes[inode] + if !ok { + inodes[inode] = name + } + return path, nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go new file mode 100644 index 0000000000..a6f99778de --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go @@ -0,0 +1,18 @@ +// +build !windows + +package fs + +import ( + "os" + "syscall" +) + +func getLinkInfo(fi os.FileInfo) (uint64, bool) { + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + + // Ino is uint32 on bsd, uint64 on darwin/linux/solaris + return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 // nolint: unconvert +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go new file mode 100644 index 0000000000..ad8845a7fb --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go @@ -0,0 +1,7 @@ +package fs + +import "os" + +func getLinkInfo(fi os.FileInfo) (uint64, bool) { + return 0, false +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/path.go b/components/engine/vendor/github.com/containerd/continuity/fs/path.go new file mode 100644 index 0000000000..13fb826385 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/path.go @@ -0,0 +1,276 @@ +package fs + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +var ( + errTooManyLinks = errors.New("too many links") +) + +type currentPath struct { + path string + f os.FileInfo + fullPath string +} + +func pathChange(lower, upper *currentPath) (ChangeKind, string) { + if lower == nil { + if upper == nil { + panic("cannot compare nil paths") + } + return ChangeKindAdd, upper.path + } + if upper == nil { + return ChangeKindDelete, lower.path + } + // TODO: compare by directory + + switch i := strings.Compare(lower.path, upper.path); { + case i < 0: + // File in lower that is not in upper + return ChangeKindDelete, lower.path + case i > 0: + // File in upper that is not in lower + return ChangeKindAdd, upper.path + default: + return ChangeKindModify, upper.path + } +} + +func sameFile(f1, f2 *currentPath) (bool, error) { + if os.SameFile(f1.f, f2.f) { + return true, nil + } + + equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys()) + if err != nil || !equalStat { + return equalStat, err + } + + if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq { + return eq, err + } + + // If not a directory also check size, modtime, and content + if !f1.f.IsDir() { + if f1.f.Size() != f2.f.Size() { + return false, nil + } + t1 := f1.f.ModTime() + t2 := f2.f.ModTime() + + if t1.Unix() != t2.Unix() { + return false, nil + } + + // If the timestamp may have been truncated in both of the + // files, check content of file to determine difference + if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 { + var eq bool + if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink { + eq, err = compareSymlinkTarget(f1.fullPath, f2.fullPath) + } else if f1.f.Size() > 0 { + eq, err = compareFileContent(f1.fullPath, f2.fullPath) + } + if err != nil || !eq { + return eq, err + } + } else if t1.Nanosecond() != t2.Nanosecond() { + return false, nil + } + } + + return true, nil +} + +func compareSymlinkTarget(p1, p2 string) (bool, error) { + t1, err := os.Readlink(p1) + if err != nil { + return false, err + } + t2, err := os.Readlink(p2) + if err != nil { + return false, err + } + return t1 == t2, nil +} + +const compareChuckSize = 32 * 1024 + +// compareFileContent compares the content of 2 same sized files +// by comparing each byte. +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, compareChuckSize) + b2 := make([]byte, compareChuckSize) + for { + n1, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } +} + +func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + p := ¤tPath{ + path: path, + f: f, + fullPath: filepath.Join(root, path), + } + + select { + case <-ctx.Done(): + return ctx.Err() + case pathC <- p: + return nil + } + }) +} + +func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case p := <-pathC: + return p, nil + } +} + +// RootPath joins a path with a root, evaluating and bounding any +// symlink to the root directory. +func RootPath(root, path string) (string, error) { + if path == "" { + return root, nil + } + var linksWalked int // to protect against cycles + for { + i := linksWalked + newpath, err := walkLinks(root, path, &linksWalked) + if err != nil { + return "", err + } + path = newpath + if i == linksWalked { + newpath = filepath.Join("/", newpath) + if path == newpath { + return filepath.Join(root, newpath), nil + } + path = newpath + } + } +} + +func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) { + if *linksWalked > 255 { + return "", false, errTooManyLinks + } + + path = filepath.Join("/", path) + if path == "/" { + return path, false, nil + } + realPath := filepath.Join(root, path) + + fi, err := os.Lstat(realPath) + if err != nil { + // If path does not yet exist, treat as non-symlink + if os.IsNotExist(err) { + return path, false, nil + } + return "", false, err + } + if fi.Mode()&os.ModeSymlink == 0 { + return path, false, nil + } + newpath, err = os.Readlink(realPath) + if err != nil { + return "", false, err + } + if filepath.IsAbs(newpath) && strings.HasPrefix(newpath, root) { + newpath = newpath[:len(root)] + if !strings.HasPrefix(newpath, "/") { + newpath = "/" + newpath + } + } + *linksWalked++ + return newpath, true, nil +} + +func walkLinks(root, path string, linksWalked *int) (string, error) { + switch dir, file := filepath.Split(path); { + case dir == "": + newpath, _, err := walkLink(root, file, linksWalked) + return newpath, err + case file == "": + if os.IsPathSeparator(dir[len(dir)-1]) { + if dir == "/" { + return dir, nil + } + return walkLinks(root, dir[:len(dir)-1], linksWalked) + } + newpath, _, err := walkLink(root, dir, linksWalked) + return newpath, err + default: + newdir, err := walkLinks(root, dir, linksWalked) + if err != nil { + return "", err + } + newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked) + if err != nil { + return "", err + } + if !islink { + return newpath, nil + } + if filepath.IsAbs(newpath) { + return newpath, nil + } + return filepath.Join(newdir, newpath), nil + } +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go b/components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go new file mode 100644 index 0000000000..a1b776fdf5 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go @@ -0,0 +1,28 @@ +// +build darwin freebsd + +package fs + +import ( + "syscall" + "time" +) + +// StatAtime returns the access time from a stat struct +func StatAtime(st *syscall.Stat_t) syscall.Timespec { + return st.Atimespec +} + +// StatCtime returns the created time from a stat struct +func StatCtime(st *syscall.Stat_t) syscall.Timespec { + return st.Ctimespec +} + +// StatMtime returns the modified time from a stat struct +func StatMtime(st *syscall.Stat_t) syscall.Timespec { + return st.Mtimespec +} + +// StatATimeAsTime returns the access time as a time.Time +func StatATimeAsTime(st *syscall.Stat_t) time.Time { + return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) // nolint: unconvert +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go b/components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go new file mode 100644 index 0000000000..25bc652581 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go @@ -0,0 +1,26 @@ +package fs + +import ( + "syscall" + "time" +) + +// StatAtime returns the Atim +func StatAtime(st *syscall.Stat_t) syscall.Timespec { + return st.Atim +} + +// StatCtime returns the Ctim +func StatCtime(st *syscall.Stat_t) syscall.Timespec { + return st.Ctim +} + +// StatMtime returns the Mtim +func StatMtime(st *syscall.Stat_t) syscall.Timespec { + return st.Mtim +} + +// StatATimeAsTime returns st.Atim as a time.Time +func StatATimeAsTime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Atim.Sec, st.Atim.Nsec) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/time.go b/components/engine/vendor/github.com/containerd/continuity/fs/time.go new file mode 100644 index 0000000000..c336f4d881 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/time.go @@ -0,0 +1,13 @@ +package fs + +import "time" + +// Gnu tar and the go tar writer don't have sub-second mtime +// precision, which is problematic when we apply changes via tar +// files, we handle this by comparing for exact times, *or* same +// second count and either a or b having exactly 0 nanoseconds +func sameFsTime(a, b time.Time) bool { + return a == b || + (a.Unix() == b.Unix() && + (a.Nanosecond() == 0 || b.Nanosecond() == 0)) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go deleted file mode 100644 index 4d8581284a..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go +++ /dev/null @@ -1,11 +0,0 @@ -package sysx - -// These functions will be generated by generate.sh -// $ GOOS=linux GOARCH=386 ./generate.sh copy -// $ GOOS=linux GOARCH=amd64 ./generate.sh copy -// $ GOOS=linux GOARCH=arm ./generate.sh copy -// $ GOOS=linux GOARCH=arm64 ./generate.sh copy -// $ GOOS=linux GOARCH=ppc64le ./generate.sh copy -// $ GOOS=linux GOARCH=s390x ./generate.sh copy - -//sys CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go deleted file mode 100644 index c1368c5723..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl -l32 copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go deleted file mode 100644 index c1368c5723..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl -l32 copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go deleted file mode 100644 index 0063f8a913..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_32.h - SYS_COPY_FILE_RANGE = 377 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go deleted file mode 100644 index 4170540c5d..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_64.h - SYS_COPY_FILE_RANGE = 326 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go deleted file mode 100644 index a05dcbb5ef..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPY_FILE_RANGE defined in Kernel 4.5+ - // Number defined in /usr/include/arm-linux-gnueabihf/asm/unistd.h - SYS_COPY_FILE_RANGE = 391 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go deleted file mode 100644 index da31bbd908..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPY_FILE_RANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm-generic/unistd.h - SYS_COPY_FILE_RANGE = 285 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go deleted file mode 100644 index 5dea25a3c4..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_64.h - SYS_COPY_FILE_RANGE = 379 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go deleted file mode 100644 index 8a6f2a7ec0..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_64.h - SYS_COPY_FILE_RANGE = 375 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/vendor.conf b/components/engine/vendor/github.com/containerd/continuity/vendor.conf new file mode 100644 index 0000000000..7c80deec58 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/vendor.conf @@ -0,0 +1,13 @@ +bazil.org/fuse 371fbbdaa8987b715bdd21d6adc4c9b20155f748 +github.com/dustin/go-humanize bb3d318650d48840a39aa21a027c6630e198e626 +github.com/golang/protobuf 1e59b77b52bf8e4b449a57e6f79f21226d571845 +github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +github.com/opencontainers/go-digest 279bed98673dd5bef374d3b6e4b09e2af76183bf +github.com/pkg/errors f15c970de5b76fac0b59abb32d62c17cc7bed265 +github.com/sirupsen/logrus 89742aefa4b206dcf400792f3bd35b542998eb3b +github.com/spf13/cobra 2da4a54c5ceefcee7ca5dd0eea1e18a3b6366489 +github.com/spf13/pflag 4c012f6dcd9546820e378d0bdda4d8fc772cdfea +golang.org/x/crypto 9f005a07e0d31d45e6656d241bb5c0f2efd4bc94 +golang.org/x/net a337091b0525af65de94df2eb7e98bd9962dcbe2 +golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +golang.org/x/sys 665f6529cca930e27b831a0d1dafffbe1c172924 diff --git a/components/engine/vendor/github.com/containerd/go-runc/runc.go b/components/engine/vendor/github.com/containerd/go-runc/runc.go index c5a66a1990..df76ad77a6 100644 --- a/components/engine/vendor/github.com/containerd/go-runc/runc.go +++ b/components/engine/vendor/github.com/containerd/go-runc/runc.go @@ -1,7 +1,6 @@ package runc import ( - "bytes" "context" "encoding/json" "errors" @@ -532,7 +531,9 @@ func (r *Runc) Restore(context context.Context, id, bundle string, opts *Restore // Update updates the current container with the provided resource spec func (r *Runc) Update(context context.Context, id string, resources *specs.LinuxResources) error { - buf := bytes.NewBuffer(nil) + buf := getBuf() + defer putBuf(buf) + if err := json.NewEncoder(buf).Encode(resources); err != nil { return err } @@ -638,11 +639,12 @@ func (r *Runc) runOrError(cmd *exec.Cmd) error { } func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) { - var b bytes.Buffer + b := getBuf() + defer putBuf(b) - cmd.Stdout = &b + cmd.Stdout = b if combined { - cmd.Stderr = &b + cmd.Stderr = b } ec, err := Monitor.Start(cmd) if err != nil { diff --git a/components/engine/vendor/github.com/containerd/go-runc/utils.go b/components/engine/vendor/github.com/containerd/go-runc/utils.go index 81fcd3f2d8..8cb241aca7 100644 --- a/components/engine/vendor/github.com/containerd/go-runc/utils.go +++ b/components/engine/vendor/github.com/containerd/go-runc/utils.go @@ -1,8 +1,10 @@ package runc import ( + "bytes" "io/ioutil" "strconv" + "sync" "syscall" ) @@ -26,3 +28,18 @@ func exitStatus(status syscall.WaitStatus) int { } return status.ExitStatus() } + +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(nil) + }, +} + +func getBuf() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func putBuf(b *bytes.Buffer) { + b.Reset() + bytesBufferPool.Put(b) +} diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index a6a107e6e6..2c69cee5d6 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -22,7 +22,6 @@ #include #include - #include #include #include @@ -32,15 +31,15 @@ /* Synchronisation values. */ enum sync_t { - SYNC_USERMAP_PLS = 0x40, /* Request parent to map our users. */ - SYNC_USERMAP_ACK = 0x41, /* Mapping finished by the parent. */ - SYNC_RECVPID_PLS = 0x42, /* Tell parent we're sending the PID. */ - SYNC_RECVPID_ACK = 0x43, /* PID was correctly received by parent. */ - SYNC_GRANDCHILD = 0x44, /* The grandchild is ready to run. */ - SYNC_CHILD_READY = 0x45, /* The child or grandchild is ready to return. */ + SYNC_USERMAP_PLS = 0x40, /* Request parent to map our users. */ + SYNC_USERMAP_ACK = 0x41, /* Mapping finished by the parent. */ + SYNC_RECVPID_PLS = 0x42, /* Tell parent we're sending the PID. */ + SYNC_RECVPID_ACK = 0x43, /* PID was correctly received by parent. */ + SYNC_GRANDCHILD = 0x44, /* The grandchild is ready to run. */ + SYNC_CHILD_READY = 0x45, /* The child or grandchild is ready to return. */ /* XXX: This doesn't help with segfaults and other such issues. */ - SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ + SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ }; /* longjmp() arguments. */ @@ -73,7 +72,7 @@ struct nlconfig_t { char *oom_score_adj; size_t oom_score_adj_len; - /* User namespace settings.*/ + /* User namespace settings. */ char *uidmap; size_t uidmap_len; char *gidmap; @@ -82,7 +81,7 @@ struct nlconfig_t { size_t namespaces_len; uint8_t is_setgroup; - /* Rootless container settings.*/ + /* Rootless container settings. */ uint8_t is_rootless; char *uidmappath; size_t uidmappath_len; @@ -167,7 +166,7 @@ static int write_file(char *data, size_t data_len, char *pathfmt, ...) goto out; } -out: + out: close(fd); return ret; } @@ -184,16 +183,16 @@ static void update_setgroups(int pid, enum policy_t setgroup) char *policy; switch (setgroup) { - case SETGROUPS_ALLOW: - policy = "allow"; - break; - case SETGROUPS_DENY: - policy = "deny"; - break; - case SETGROUPS_DEFAULT: - default: - /* Nothing to do. */ - return; + case SETGROUPS_ALLOW: + policy = "allow"; + break; + case SETGROUPS_DENY: + policy = "deny"; + break; + case SETGROUPS_DEFAULT: + default: + /* Nothing to do. */ + return; } if (write_file(policy, strlen(policy), "/proc/%d/setgroups", pid) < 0) { @@ -226,14 +225,14 @@ static int try_mapping_tool(const char *app, int pid, char *map, size_t map_len) if (!child) { #define MAX_ARGV 20 char *argv[MAX_ARGV]; - char *envp[] = {NULL}; + char *envp[] = { NULL }; char pid_fmt[16]; int argc = 0; char *next; snprintf(pid_fmt, 16, "%d", pid); - argv[argc++] = (char *) app; + argv[argc++] = (char *)app; argv[argc++] = pid_fmt; /* * Convert the map string into a list of argument that @@ -319,7 +318,7 @@ static int clone_parent(jmp_buf *env, int jmpval) __attribute__ ((noinline)); static int clone_parent(jmp_buf *env, int jmpval) { struct clone_t ca = { - .env = env, + .env = env, .jmpval = jmpval, }; @@ -533,7 +532,7 @@ void nsexec(void) int pipenum; jmp_buf env; int sync_child_pipe[2], sync_grandchild_pipe[2]; - struct nlconfig_t config = {0}; + struct nlconfig_t config = { 0 }; /* * If we don't have an init pipe, just return to the go routine. @@ -630,21 +629,21 @@ void nsexec(void) */ switch (setjmp(env)) { - /* - * Stage 0: We're in the parent. Our job is just to create a new child - * (stage 1: JUMP_CHILD) process and write its uid_map and - * gid_map. That process will go on to create a new process, then - * it will send us its PID which we will send to the bootstrap - * process. - */ - case JUMP_PARENT: { + /* + * Stage 0: We're in the parent. Our job is just to create a new child + * (stage 1: JUMP_CHILD) process and write its uid_map and + * gid_map. That process will go on to create a new process, then + * it will send us its PID which we will send to the bootstrap + * process. + */ + case JUMP_PARENT:{ int len; pid_t child, first_child = -1; char buf[JSON_MAX]; bool ready = false; /* For debugging. */ - prctl(PR_SET_NAME, (unsigned long) "runc:[0:PARENT]", 0, 0, 0); + prctl(PR_SET_NAME, (unsigned long)"runc:[0:PARENT]", 0, 0, 0); /* Start the process of getting a container. */ child = clone_parent(&env, JUMP_CHILD); @@ -702,7 +701,7 @@ void nsexec(void) bail("failed to sync with child: write(SYNC_USERMAP_ACK)"); } break; - case SYNC_RECVPID_PLS: { + case SYNC_RECVPID_PLS:{ first_child = child; /* Get the init_func pid. */ @@ -781,16 +780,16 @@ void nsexec(void) exit(0); } - /* - * Stage 1: We're in the first child process. Our job is to join any - * provided namespaces in the netlink payload and unshare all - * of the requested namespaces. If we've been asked to - * CLONE_NEWUSER, we will ask our parent (stage 0) to set up - * our user mappings for us. Then, we create a new child - * (stage 2: JUMP_INIT) for PID namespace. We then send the - * child's PID to our parent (stage 0). - */ - case JUMP_CHILD: { + /* + * Stage 1: We're in the first child process. Our job is to join any + * provided namespaces in the netlink payload and unshare all + * of the requested namespaces. If we've been asked to + * CLONE_NEWUSER, we will ask our parent (stage 0) to set up + * our user mappings for us. Then, we create a new child + * (stage 2: JUMP_INIT) for PID namespace. We then send the + * child's PID to our parent (stage 0). + */ + case JUMP_CHILD:{ pid_t child; enum sync_t s; @@ -799,7 +798,7 @@ void nsexec(void) close(sync_child_pipe[1]); /* For debugging. */ - prctl(PR_SET_NAME, (unsigned long) "runc:[1:CHILD]", 0, 0, 0); + prctl(PR_SET_NAME, (unsigned long)"runc:[1:CHILD]", 0, 0, 0); /* * We need to setns first. We cannot do this earlier (in stage 0) @@ -901,13 +900,13 @@ void nsexec(void) exit(0); } - /* - * Stage 2: We're the final child process, and the only process that will - * actually return to the Go runtime. Our job is to just do the - * final cleanup steps and then return to the Go runtime to allow - * init_linux.go to run. - */ - case JUMP_INIT: { + /* + * Stage 2: We're the final child process, and the only process that will + * actually return to the Go runtime. Our job is to just do the + * final cleanup steps and then return to the Go runtime to allow + * init_linux.go to run. + */ + case JUMP_INIT:{ /* * We're inside the child now, having jumped from the * start_child() code after forking in the parent. @@ -921,7 +920,7 @@ void nsexec(void) close(sync_child_pipe[1]); /* For debugging. */ - prctl(PR_SET_NAME, (unsigned long) "runc:[2:INIT]", 0, 0, 0); + prctl(PR_SET_NAME, (unsigned long)"runc:[2:INIT]", 0, 0, 0); if (read(syncfd, &s, sizeof(s)) != sizeof(s)) bail("failed to sync with parent: read(SYNC_GRANDCHILD)"); diff --git a/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go b/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go index 4a33827a43..9493d68624 100644 --- a/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go +++ b/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go @@ -5,6 +5,7 @@ import ( "context" "encoding/binary" "io" + "net" "sync" "github.com/pkg/errors" @@ -60,16 +61,18 @@ func writeMessageHeader(w io.Writer, p []byte, mh messageHeader) error { var buffers sync.Pool type channel struct { + conn net.Conn bw *bufio.Writer br *bufio.Reader hrbuf [messageHeaderLength]byte // avoid alloc when reading header hwbuf [messageHeaderLength]byte } -func newChannel(w io.Writer, r io.Reader) *channel { +func newChannel(conn net.Conn) *channel { return &channel{ - bw: bufio.NewWriter(w), - br: bufio.NewReader(r), + conn: conn, + bw: bufio.NewWriter(conn), + br: bufio.NewReader(conn), } } diff --git a/components/engine/vendor/github.com/stevvooe/ttrpc/client.go b/components/engine/vendor/github.com/stevvooe/ttrpc/client.go index ca76afe19a..f047181678 100644 --- a/components/engine/vendor/github.com/stevvooe/ttrpc/client.go +++ b/components/engine/vendor/github.com/stevvooe/ttrpc/client.go @@ -2,8 +2,12 @@ package ttrpc import ( "context" + "io" "net" + "os" + "strings" "sync" + "syscall" "github.com/containerd/containerd/log" "github.com/gogo/protobuf/proto" @@ -11,6 +15,10 @@ import ( "google.golang.org/grpc/status" ) +// ErrClosed is returned by client methods when the underlying connection is +// closed. +var ErrClosed = errors.New("ttrpc: closed") + type Client struct { codec codec conn net.Conn @@ -19,18 +27,20 @@ type Client struct { closed chan struct{} closeOnce sync.Once + closeFunc func() done chan struct{} err error } func NewClient(conn net.Conn) *Client { c := &Client{ - codec: codec{}, - conn: conn, - channel: newChannel(conn, conn), - calls: make(chan *callRequest), - closed: make(chan struct{}), - done: make(chan struct{}), + codec: codec{}, + conn: conn, + channel: newChannel(conn), + calls: make(chan *callRequest), + closed: make(chan struct{}), + done: make(chan struct{}), + closeFunc: func() {}, } go c.run() @@ -91,7 +101,7 @@ func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) err select { case err := <-errs: - return err + return filterCloseErr(err) case <-c.done: return c.err } @@ -105,6 +115,11 @@ func (c *Client) Close() error { return nil } +// OnClose allows a close func to be called when the server is closed +func (c *Client) OnClose(closer func()) { + c.closeFunc = closer +} + type message struct { messageHeader p []byte @@ -150,6 +165,7 @@ func (c *Client) run() { defer c.conn.Close() defer close(c.done) + defer c.closeFunc() for { select { @@ -171,7 +187,14 @@ func (c *Client) run() { call.errs <- c.recv(call.resp, msg) delete(waiters, msg.StreamID) case <-shutdown: + if shutdownErr != nil { + shutdownErr = filterCloseErr(shutdownErr) + } else { + shutdownErr = ErrClosed + } + shutdownErr = errors.Wrapf(shutdownErr, "ttrpc: client shutting down") + c.err = shutdownErr for _, waiter := range waiters { waiter.errs <- shutdownErr @@ -179,9 +202,12 @@ func (c *Client) run() { c.Close() return case <-c.closed: + if c.err == nil { + c.err = ErrClosed + } // broadcast the shutdown error to the remaining waiters. for _, waiter := range waiters { - waiter.errs <- shutdownErr + waiter.errs <- c.err } return } @@ -209,3 +235,30 @@ func (c *Client) recv(resp *Response, msg *message) error { defer c.channel.putmbuf(msg.p) return proto.Unmarshal(msg.p, resp) } + +// filterCloseErr rewrites EOF and EPIPE errors to ErrClosed. Use when +// returning from call or handling errors from main read loop. +// +// This purposely ignores errors with a wrapped cause. +func filterCloseErr(err error) error { + if err == nil { + return nil + } + + if err == io.EOF { + return ErrClosed + } + + if strings.Contains(err.Error(), "use of closed network connection") { + return ErrClosed + } + + // if we have an epipe on a write, we cast to errclosed + if oerr, ok := err.(*net.OpError); ok && oerr.Op == "write" { + if serr, ok := oerr.Err.(*os.SyscallError); ok && serr.Err == syscall.EPIPE { + return ErrClosed + } + } + + return err +} diff --git a/components/engine/vendor/github.com/stevvooe/ttrpc/server.go b/components/engine/vendor/github.com/stevvooe/ttrpc/server.go index edfca0c52c..fd29b719e4 100644 --- a/components/engine/vendor/github.com/stevvooe/ttrpc/server.go +++ b/components/engine/vendor/github.com/stevvooe/ttrpc/server.go @@ -16,7 +16,7 @@ import ( ) var ( - ErrServerClosed = errors.New("ttrpc: server close") + ErrServerClosed = errors.New("ttrpc: server closed") ) type Server struct { @@ -281,7 +281,7 @@ func (c *serverConn) run(sctx context.Context) { ) var ( - ch = newChannel(c.conn, c.conn) + ch = newChannel(c.conn) ctx, cancel = context.WithCancel(sctx) active int state connState = connStateIdle