Merge component 'engine' from git@github.com:moby/moby master

components/engine/Dockerfile

@@ -2,14 +2,23 @@
 #
 # Usage:
 #
-# # Assemble the full dev environment. This is slow the first time.
-# docker build -t docker .
+# # Use make to build a development environment image and run it in a container.
+# # This is slow the first time.
+# make BIND_DIR=. shell
 #
-# # Mount your source in an interactive container for quick testing:
-# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
+# The following commands are executed inside the running container.
+
+# # Make a dockerd binary.
+# # hack/make.sh binary
 #
-# # Run the test suite:
-# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py
+# # Install dockerd to /usr/local/bin
+# # make install
+#
+# # Run unit tests
+# # hack/test/unit
+#
+# # Run tests e.g. integration, py
+# # hack/make.sh binary test-integration test-docker-py
 #
 # # Publish a release:
 # docker run --privileged \
@@ -133,7 +142,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a
+ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916
 # To run integration tests docker-pycreds is required.
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
@@ -146,9 +155,12 @@ RUN pip install yamllint==1.5.0
 
 # Install go-swagger for validating swagger.yaml
 ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb
-RUN git clone https://github.com/go-swagger/go-swagger.git /go/src/github.com/go-swagger/go-swagger \
-	&& (cd /go/src/github.com/go-swagger/go-swagger && git checkout -q $GO_SWAGGER_COMMIT) \
-	&& go install -v github.com/go-swagger/go-swagger/cmd/swagger
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \
+	&& (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \
+	&& go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \
+	&& rm -rf "$GOPATH"
 
 # Set user.email so crosbymichael's in-container merge commits go smoothly
 RUN git config --global user.email 'docker-dummy@example.com'
@@ -168,10 +180,9 @@ RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc
 
 # Get useful and necessary Hub images so we can "docker load" locally instead of pulling
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
-# TODO: when issue #35963 fixed, we can upgrade the busybox to multi-arch
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-	busybox:latest@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f \
+	busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)

components/engine/Dockerfile.aarch64

@@ -106,7 +106,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a
+ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916
 # To run integration tests docker-pycreds is required.
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
@@ -119,9 +119,12 @@ RUN pip install yamllint==1.5.0
 
 # Install go-swagger for validating swagger.yaml
 ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb
-RUN git clone https://github.com/go-swagger/go-swagger.git /go/src/github.com/go-swagger/go-swagger \
-	&& (cd /go/src/github.com/go-swagger/go-swagger && git checkout -q $GO_SWAGGER_COMMIT) \
-	&& go install -v github.com/go-swagger/go-swagger/cmd/swagger
+RUN set -x \
+	&& export GOPATH="$(mktemp -d)" \
+	&& git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \
+	&& (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \
+	&& go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \
+	&& rm -rf "$GOPATH"
 
 # Set user.email so crosbymichael's in-container merge commits go smoothly
 RUN git config --global user.email 'docker-dummy@example.com'
@@ -144,7 +147,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
+	busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)

components/engine/Dockerfile.armhf

@@ -104,7 +104,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a
+ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916
 # To run integration tests docker-pycreds is required.
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
@@ -133,7 +133,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
+	busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)

components/engine/Dockerfile.e2e

@@ -16,10 +16,10 @@ WORKDIR /go/src/github.com/docker/docker/
 # Generate frozen images
 COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh
 RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \
-  buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-  busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
-  debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
-  hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
+	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
+	busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \
+	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
+	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 
 # Download Docker CLI binary
 COPY hack/dockerfile hack/dockerfile

components/engine/Dockerfile.ppc64le

@@ -102,7 +102,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a
+ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916
 # To run integration tests docker-pycreds is required.
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
@@ -131,7 +131,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
+	busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)

components/engine/Dockerfile.s390x

@@ -96,7 +96,7 @@ RUN set -x \
 	&& rm -rf "$GOPATH"
 
 # Get the "docker-py" source so we can run their integration tests
-ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a
+ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916
 # To run integration tests docker-pycreds is required.
 RUN git clone https://github.com/docker/docker-py.git /docker-py \
 	&& cd /docker-py \
@@ -125,7 +125,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
 COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
 RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
-	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
+	busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)

components/engine/MAINTAINERS

@@ -23,29 +23,19 @@
 	# a subsystem, they are responsible for doing so and holding the
 	# subsystem maintainers accountable. If ownership is unclear, they are the de facto owners.
 
-	# For each release (including minor releases), a "release captain" is assigned from the
-	# pool of core maintainers. Rotation is encouraged across all maintainers, to ensure
-	# the release process is clear and up-to-date.
-
 		people = [
 			"aaronlehmann",
 			"akihirosuda",
-			"albers",
-			"aluzzardi",
 			"anusha",
 			"coolljt0725",
 			"cpuguy83",
 			"crosbymichael",
 			"dnephin",
 			"duglin",
-			"ehazlett",
 			"estesp",
-			"icecrime",
 			"jhowardmsft",
 			"johnstep",
 			"justincormack",
-			"lk4d4",
-			"mavenugo",
 			"mhbauer",
 			"mlaventure",
 			"runcom",
@@ -103,6 +93,16 @@
 	# Thank you!
 
 		people = [
+			# Harald Albers is the mastermind behind the bash completion scripts for the
+			# Docker CLI. The completion scripts moved to the Docker CLI repository, so
+			# you can now find him perform his magic in the https://github.com/docker/cli repository.
+			"albers",
+
+			# Andrea Luzzardi started contributing to the Docker codebase in the "dotCloud"
+			# era, even before it was called "Docker". He is one of the architects of both
+			# Swarm and SwarmKit, and its integration into the Docker engine.
+			"aluzzardi",
+			
 			# David Calavera contributed many features to Docker, such as an improved
 			# event system, dynamic configuration reloading, volume plugins, fancy
 			# new templating options, and an external client credential store. As a
@@ -120,6 +120,24 @@
 			# still stumble into him in our issue tracker, or on IRC.
 			"erikh",
 
+			# Evan Hazlett is the creator of of the Shipyard and Interlock open source projects,
+			# and the author of "Orca", which became the foundation of Docker Universal Control
+			# Plane (UCP). As a maintainer, Evan helped integrating SwarmKit (secrets, tasks)
+			# into the Docker engine.
+			"ehazlett",
+
+			# Arnaud Porterie (AKA "icecrime") was in charge of maintaining the maintainers.
+			# As a maintainer, he made life easier for contributors to the Docker open-source
+			# projects, bringing order in the chaos by designing a triage- and review workflow
+			# using labels (see https://icecrime.net/technology/a-structured-approach-to-labeling/),
+			# and automating the hell out of things with his buddies GordonTheTurtle and Poule
+			# (a chicken!).
+			# 
+			# A lesser-known fact is that he created the first commit in the libnetwork repository
+			# even though he didn't know anything about it. Some say, he's now selling stuff on
+			# the internet ;-)
+			"icecrime",
+
 			# After a false start with his first PR being rejected, James Turnbull became a frequent
 			# contributor to the documentation, and became a docs maintainer on December 5, 2013. As
 			# a maintainer, James lifted the docs to a higher standard, and introduced the community
@@ -139,13 +157,24 @@
 			# containers a lot more secure). Besides being a maintainer, she
 			# set up the CI infrastructure for the project, giving everyone
 			# something to shout at if a PR failed ("noooo Janky!").
-			# Jess is currently working on the DCOS security team at Mesosphere,
-			# and contributing to various open source projects.
 			# Be sure you don't miss her talks at a conference near you (a must-see),
 			# read her blog at https://blog.jessfraz.com (a must-read), and
 			# check out her open source projects on GitHub https://github.com/jessfraz (a must-try).
 			"jessfraz",
 
+			# Alexander Morozov contributed many features to Docker, worked on the premise of 
+			# what later became containerd (and worked on that too), and made a "stupid" Go
+			# vendor tool specificaly for docker/docker needs: vndr (https://github.com/LK4D4/vndr).
+			# Not many know that Alexander is a master negotiator, being able to change course
+			# of action with a single "Nope, we're not gonna do that".
+			"lk4d4",
+
+			# Madhu Venugopal was part of the SocketPlane team that joined Docker.
+			# As a maintainer, he was working with Jana for the Container Network
+			# Model (CNM) implemented through libnetwork, and the "routing mesh" powering
+			# Swarm mode networking.
+			"mavenugo",
+
 			# As a docs maintainer, Mary Anthony contributed greatly to the Docker
 			# docs. She wrote the Docker Contributor Guide and Getting Started
 			# Guides. She helped create a doc build system independent of

components/engine/api/server/router/build/build_routes.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
 	"runtime"
 	"strconv"
 	"strings"
@@ -71,17 +70,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	options.Target = r.FormValue("target")
 	options.RemoteContext = r.FormValue("remote")
 	if versions.GreaterThanOrEqualTo(version, "1.32") {
-		// TODO @jhowardmsft. The following environment variable is an interim
-		// measure to allow the daemon to have a default platform if omitted by
-		// the client. This allows LCOW and WCOW to work with a down-level CLI
-		// for a short period of time, as the CLI changes can't be merged
-		// until after the daemon changes have been merged. Once the CLI is
-		// updated, this can be removed. PR for CLI is currently in
-		// https://github.com/docker/cli/pull/474.
 		apiPlatform := r.FormValue("platform")
-		if system.LCOWSupported() && apiPlatform == "" {
-			apiPlatform = os.Getenv("LCOW_API_PLATFORM_IF_OMITTED")
-		}
 		p := system.ParsePlatform(apiPlatform)
 		if err := system.ValidatePlatform(p); err != nil {
 			return nil, errdefs.InvalidParameter(errors.Errorf("invalid platform: %s", err))

components/engine/api/server/router/container/backend.go

@@ -68,8 +68,13 @@ type systemBackend interface {
 	ContainersPrune(ctx context.Context, pruneFilters filters.Args) (*types.ContainersPruneReport, error)
 }
 
+type commitBackend interface {
+	CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error)
+}
+
 // Backend is all the methods that need to be implemented to provide container specific functionality.
 type Backend interface {
+	commitBackend
 	execBackend
 	copyBackend
 	stateBackend

components/engine/api/server/router/container/container.go

@@ -61,6 +61,7 @@ func (r *containerRouter) initRoutes() {
 		router.NewPostRoute("/containers/{name:.*}/rename", r.postContainerRename),
 		router.NewPostRoute("/containers/{name:.*}/update", r.postContainerUpdate),
 		router.NewPostRoute("/containers/prune", r.postContainersPrune, router.WithCancel),
+		router.NewPostRoute("/commit", r.postCommit),
 		// PUT
 		router.NewPutRoute("/containers/{name:.*}/archive", r.putContainersArchive),
 		// DELETE

components/engine/api/server/router/container/container_routes.go

@@ -24,6 +24,45 @@ import (
 	"golang.org/x/net/websocket"
 )
 
+func (s *containerRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	if err := httputils.ParseForm(r); err != nil {
+		return err
+	}
+
+	if err := httputils.CheckForJSON(r); err != nil {
+		return err
+	}
+
+	// TODO: remove pause arg, and always pause in backend
+	pause := httputils.BoolValue(r, "pause")
+	version := httputils.VersionFromContext(ctx)
+	if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") {
+		pause = true
+	}
+
+	config, _, _, err := s.decoder.DecodeConfig(r.Body)
+	if err != nil && err != io.EOF { //Do not fail if body is empty.
+		return err
+	}
+
+	commitCfg := &backend.CreateImageConfig{
+		Pause:   pause,
+		Repo:    r.Form.Get("repo"),
+		Tag:     r.Form.Get("tag"),
+		Author:  r.Form.Get("author"),
+		Comment: r.Form.Get("comment"),
+		Config:  config,
+		Changes: r.Form["changes"],
+	}
+
+	imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg)
+	if err != nil {
+		return err
+	}
+
+	return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ID: imgID})
+}
+
 func (s *containerRouter) getContainersJSON(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := httputils.ParseForm(r); err != nil {
 		return err

components/engine/api/server/router/image/backend.go

@@ -4,7 +4,6 @@ import (
 	"io"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/registry"
@@ -14,22 +13,17 @@ import (
 // Backend is all the methods that need to be implemented
 // to provide image specific functionality.
 type Backend interface {
-	containerBackend
 	imageBackend
 	importExportBackend
 	registryBackend
 }
 
-type containerBackend interface {
-	CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error)
-}
-
 type imageBackend interface {
 	ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error)
 	ImageHistory(imageName string) ([]*image.HistoryResponseItem, error)
 	Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error)
 	LookupImage(name string) (*types.ImageInspect, error)
-	TagImage(imageName, repository, tag string) error
+	TagImage(imageName, repository, tag string) (string, error)
 	ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error)
 }
 

components/engine/api/server/router/image/image.go

@@ -1,23 +1,18 @@
 package image // import "github.com/docker/docker/api/server/router/image"
 
 import (
-	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/server/router"
 )
 
 // imageRouter is a router to talk with the image controller
 type imageRouter struct {
 	backend Backend
-	decoder httputils.ContainerDecoder
 	routes  []router.Route
 }
 
 // NewRouter initializes a new image router
-func NewRouter(backend Backend, decoder httputils.ContainerDecoder) router.Router {
-	r := &imageRouter{
-		backend: backend,
-		decoder: decoder,
-	}
+func NewRouter(backend Backend) router.Router {
+	r := &imageRouter{backend: backend}
 	r.initRoutes()
 	return r
 }
@@ -38,7 +33,6 @@ func (r *imageRouter) initRoutes() {
 		router.NewGetRoute("/images/{name:.*}/history", r.getImagesHistory),
 		router.NewGetRoute("/images/{name:.*}/json", r.getImagesByName),
 		// POST
-		router.NewPostRoute("/commit", r.postCommit),
 		router.NewPostRoute("/images/load", r.postImagesLoad),
 		router.NewPostRoute("/images/create", r.postImagesCreate, router.WithCancel),
 		router.NewPostRoute("/images/{name:.*}/push", r.postImagesPush, router.WithCancel),

components/engine/api/server/router/image/image_routes.go

@@ -4,15 +4,12 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
-	"os"
 	"strconv"
 	"strings"
 
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/errdefs"
@@ -25,45 +22,6 @@ import (
 	"golang.org/x/net/context"
 )
 
-func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	if err := httputils.ParseForm(r); err != nil {
-		return err
-	}
-
-	if err := httputils.CheckForJSON(r); err != nil {
-		return err
-	}
-
-	// TODO: remove pause arg, and always pause in backend
-	pause := httputils.BoolValue(r, "pause")
-	version := httputils.VersionFromContext(ctx)
-	if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") {
-		pause = true
-	}
-
-	config, _, _, err := s.decoder.DecodeConfig(r.Body)
-	if err != nil && err != io.EOF { //Do not fail if body is empty.
-		return err
-	}
-
-	commitCfg := &backend.CreateImageConfig{
-		Pause:   pause,
-		Repo:    r.Form.Get("repo"),
-		Tag:     r.Form.Get("tag"),
-		Author:  r.Form.Get("author"),
-		Comment: r.Form.Get("comment"),
-		Config:  config,
-		Changes: r.Form["changes"],
-	}
-
-	imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg)
-	if err != nil {
-		return err
-	}
-
-	return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ID: imgID})
-}
-
 // Creates an image from Pull or from Import
 func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 
@@ -86,17 +44,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
 
 	version := httputils.VersionFromContext(ctx)
 	if versions.GreaterThanOrEqualTo(version, "1.32") {
-		// TODO @jhowardmsft. The following environment variable is an interim
-		// measure to allow the daemon to have a default platform if omitted by
-		// the client. This allows LCOW and WCOW to work with a down-level CLI
-		// for a short period of time, as the CLI changes can't be merged
-		// until after the daemon changes have been merged. Once the CLI is
-		// updated, this can be removed. PR for CLI is currently in
-		// https://github.com/docker/cli/pull/474.
 		apiPlatform := r.FormValue("platform")
-		if system.LCOWSupported() && apiPlatform == "" {
-			apiPlatform = os.Getenv("LCOW_API_PLATFORM_IF_OMITTED")
-		}
 		platform = system.ParsePlatform(apiPlatform)
 		if err = system.ValidatePlatform(platform); err != nil {
 			err = fmt.Errorf("invalid platform: %s", err)
@@ -303,7 +251,7 @@ func (s *imageRouter) postImagesTag(ctx context.Context, w http.ResponseWriter,
 	if err := httputils.ParseForm(r); err != nil {
 		return err
 	}
-	if err := s.backend.TagImage(vars["name"], r.Form.Get("repo"), r.Form.Get("tag")); err != nil {
+	if _, err := s.backend.TagImage(vars["name"], r.Form.Get("repo"), r.Form.Get("tag")); err != nil {
 		return err
 	}
 	w.WriteHeader(http.StatusCreated)

components/engine/cmd/dockerd/daemon.go

@@ -514,7 +514,7 @@ func initRouter(opts routerOptions) {
 		// we need to add the checkpoint router before the container router or the DELETE gets masked
 		checkpointrouter.NewRouter(opts.daemon, decoder),
 		container.NewRouter(opts.daemon, decoder),
-		image.NewRouter(opts.daemon, decoder),
+		image.NewRouter(opts.daemon),
 		systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache),
 		volume.NewRouter(opts.daemon),
 		build.NewRouter(opts.buildBackend, opts.daemon),

components/engine/container/container_unix.go

@@ -6,13 +6,12 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/containerd/continuity/fs"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	mounttypes "github.com/docker/docker/api/types/mount"
-	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/volume"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
@@ -398,53 +397,15 @@ func (container *Container) DetachAndUnmount(volumeEventLog func(name, action st
 // copyExistingContents copies from the source to the destination and
 // ensures the ownership is appropriately set.
 func copyExistingContents(source, destination string) error {
-	volList, err := ioutil.ReadDir(source)
+	dstList, err := ioutil.ReadDir(destination)
 	if err != nil {
 		return err
 	}
-	if len(volList) > 0 {
-		srcList, err := ioutil.ReadDir(destination)
-		if err != nil {
-			return err
-		}
-		if len(srcList) == 0 {
-			// If the source volume is empty, copies files from the root into the volume
-			if err := chrootarchive.NewArchiver(nil).CopyWithTar(source, destination); err != nil {
-				return err
-			}
-		}
+	if len(dstList) != 0 {
+		// destination is not empty, do not copy
+		return nil
 	}
-	return copyOwnership(source, destination)
-}
-
-// copyOwnership copies the permissions and uid:gid of the source file
-// to the destination file
-func copyOwnership(source, destination string) error {
-	stat, err := system.Stat(source)
-	if err != nil {
-		return err
-	}
-
-	destStat, err := system.Stat(destination)
-	if err != nil {
-		return err
-	}
-
-	// In some cases, even though UID/GID match and it would effectively be a no-op,
-	// this can return a permission denied error... for example if this is an NFS
-	// mount.
-	// Since it's not really an error that we can't chown to the same UID/GID, don't
-	// even bother trying in such cases.
-	if stat.UID() != destStat.UID() || stat.GID() != destStat.GID() {
-		if err := os.Chown(destination, int(stat.UID()), int(stat.GID())); err != nil {
-			return err
-		}
-	}
-
-	if stat.Mode() != destStat.Mode() {
-		return os.Chmod(destination, os.FileMode(stat.Mode()))
-	}
-	return nil
+	return fs.CopyDir(destination, source)
 }
 
 // TmpfsMounts returns the list of tmpfs mounts

components/engine/daemon/commit.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types/backend"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder/dockerfile"
@@ -176,9 +175,12 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma
 		return "", err
 	}
 
-	imageRef, err := daemon.tagCommit(c.Repo, c.Tag, id)
-	if err != nil {
-		return "", err
+	var imageRef string
+	if c.Repo != "" {
+		imageRef, err = daemon.TagImage(string(id), c.Repo, c.Tag)
+		if err != nil {
+			return "", err
+		}
 	}
 	daemon.LogContainerEventWithAttributes(container, "commit", map[string]string{
 		"comment":  c.Comment,
@@ -247,30 +249,6 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
 	return id, nil
 }
 
-// TODO: remove from Daemon, move to api backend
-func (daemon *Daemon) tagCommit(repo string, tag string, id image.ID) (string, error) {
-	imageRef := ""
-	if repo != "" {
-		newTag, err := reference.ParseNormalizedNamed(repo) // todo: should move this to API layer
-		if err != nil {
-			return "", err
-		}
-		if !reference.IsNameOnly(newTag) {
-			return "", errors.Errorf("unexpected repository name: %s", repo)
-		}
-		if tag != "" {
-			if newTag, err = reference.WithTag(newTag, tag); err != nil {
-				return "", err
-			}
-		}
-		if err := daemon.TagImageWithReference(id, newTag); err != nil {
-			return "", err
-		}
-		imageRef = reference.FamiliarString(newTag)
-	}
-	return imageRef, nil
-}
-
 func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
 	rwlayer, err := layerStore.GetRWLayer(id)
 	if err != nil {

components/engine/daemon/graphdriver/btrfs/btrfs.go

@@ -29,7 +29,6 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
-	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-units"
@@ -163,7 +162,7 @@ func (d *Driver) Cleanup() error {
 		return err
 	}
 
-	return mount.RecursiveUnmount(d.home)
+	return nil
 }
 
 func free(p *C.char) {

components/engine/daemon/graphdriver/zfs/zfs.go

@@ -178,9 +178,10 @@ func (d *Driver) String() string {
 	return "zfs"
 }
 
-// Cleanup is called on daemon shutdown, it is used to clean up any remaining mounts
+// Cleanup is called on daemon shutdown, it is a no-op for ZFS.
+// TODO(@cpuguy83): Walk layer tree and check mounts?
 func (d *Driver) Cleanup() error {
-	return mount.RecursiveUnmount(d.options.mountPath)
+	return nil
 }
 
 // Status returns information about the ZFS filesystem. It returns a two dimensional array of information

components/engine/daemon/image_tag.go

@@ -7,23 +7,24 @@ import (
 
 // TagImage creates the tag specified by newTag, pointing to the image named
 // imageName (alternatively, imageName can also be an image ID).
-func (daemon *Daemon) TagImage(imageName, repository, tag string) error {
+func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error) {
 	imageID, _, err := daemon.GetImageIDAndOS(imageName)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	newTag, err := reference.ParseNormalizedNamed(repository)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if tag != "" {
 		if newTag, err = reference.WithTag(reference.TrimNamed(newTag), tag); err != nil {
-			return err
+			return "", err
 		}
 	}
 
-	return daemon.TagImageWithReference(imageID, newTag)
+	err = daemon.TagImageWithReference(imageID, newTag)
+	return reference.FamiliarString(newTag), err
 }
 
 // TagImageWithReference adds the given reference to the image ID provided.

components/engine/daemon/logger/loggerutils/logfile.go

@@ -192,8 +192,12 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher)
 		for _, f := range files {
 			seekers = append(seekers, f)
 		}
-		seekers = append(seekers, currentChunk)
-		tailFile(multireader.MultiReadSeeker(seekers...), watcher, w.createDecoder, config)
+		if currentChunk.Size() > 0 {
+			seekers = append(seekers, currentChunk)
+		}
+		if len(seekers) > 0 {
+			tailFile(multireader.MultiReadSeeker(seekers...), watcher, w.createDecoder, config)
+		}
 	}
 
 	w.mu.RLock()

components/engine/daemon/oci_linux.go

@@ -604,7 +604,8 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c
 		//
 		// For private volumes any root propagation value should work.
 		pFlag := mountPropagationMap[m.Propagation]
-		if pFlag == mount.SHARED || pFlag == mount.RSHARED {
+		switch pFlag {
+		case mount.SHARED, mount.RSHARED:
 			if err := ensureShared(m.Source); err != nil {
 				return err
 			}
@@ -612,13 +613,34 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c
 			if rootpg != mount.SHARED && rootpg != mount.RSHARED {
 				s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.SHARED]
 			}
-		} else if pFlag == mount.SLAVE || pFlag == mount.RSLAVE {
+		case mount.SLAVE, mount.RSLAVE:
+			var fallback bool
 			if err := ensureSharedOrSlave(m.Source); err != nil {
-				return err
+				// For backwards compatability purposes, treat mounts from the daemon root
+				// as special since we automatically add rslave propagation to these mounts
+				// when the user did not set anything, so we should fallback to the old
+				// behavior which is to use private propagation which is normally the
+				// default.
+				if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) {
+					return err
+				}
+
+				cm, ok := c.MountPoints[m.Destination]
+				if !ok {
+					return err
+				}
+				if cm.Spec.BindOptions != nil && cm.Spec.BindOptions.Propagation != "" {
+					// This means the user explicitly set a propagation, do not fallback in that case.
+					return err
+				}
+				fallback = true
+				logrus.WithField("container", c.ID).WithField("source", m.Source).Warn("Falling back to default propagation for bind source in daemon root")
 			}
-			rootpg := mountPropagationMap[s.Linux.RootfsPropagation]
-			if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE {
-				s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE]
+			if !fallback {
+				rootpg := mountPropagationMap[s.Linux.RootfsPropagation]
+				if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE {
+					s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE]
+				}
 			}
 		}
 

components/engine/daemon/volumes.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
 	mounttypes "github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/errdefs"
@@ -146,6 +147,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo
 		if err != nil {
 			return err
 		}
+		needsSlavePropagation, err := daemon.validateBindDaemonRoot(bind.Spec)
+		if err != nil {
+			return err
+		}
+		if needsSlavePropagation {
+			bind.Propagation = mount.PropagationRSlave
+		}
 
 		// #10618
 		_, tmpfsExists := hostConfig.Tmpfs[bind.Destination]
@@ -178,6 +186,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo
 		if err != nil {
 			return errdefs.InvalidParameter(err)
 		}
+		needsSlavePropagation, err := daemon.validateBindDaemonRoot(mp.Spec)
+		if err != nil {
+			return err
+		}
+		if needsSlavePropagation {
+			mp.Propagation = mount.PropagationRSlave
+		}
 
 		if binds[mp.Destination] {
 			return duplicateMountPointError(cfg.Target)

components/engine/daemon/volumes_linux.go

@@ -0,0 +1,36 @@
+package daemon
+
+import (
+	"strings"
+
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/errdefs"
+	"github.com/pkg/errors"
+)
+
+// validateBindDaemonRoot ensures that if a given mountpoint's source is within
+// the daemon root path, that the propagation is setup to prevent a container
+// from holding private refereneces to a mount within the daemon root, which
+// can cause issues when the daemon attempts to remove the mountpoint.
+func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) {
+	if m.Type != mount.TypeBind {
+		return false, nil
+	}
+
+	// check if the source is within the daemon root, or if the daemon root is within the source
+	if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) {
+		return false, nil
+	}
+
+	if m.BindOptions == nil {
+		return true, nil
+	}
+
+	switch m.BindOptions.Propagation {
+	case mount.PropagationRSlave, mount.PropagationRShared, "":
+		return m.BindOptions.Propagation == "", nil
+	default:
+	}
+
+	return false, errdefs.InvalidParameter(errors.Errorf(`invalid mount config: must use either propagation mode "rslave" or "rshared" when mount source is within the daemon root, daemon root: %q, bind mount source: %q, propagation: %q`, daemon.root, m.Source, m.BindOptions.Propagation))
+}

components/engine/daemon/volumes_linux_test.go

@@ -0,0 +1,56 @@
+package daemon
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/mount"
+)
+
+func TestBindDaemonRoot(t *testing.T) {
+	t.Parallel()
+	d := &Daemon{root: "/a/b/c/daemon"}
+	for _, test := range []struct {
+		desc      string
+		opts      *mount.BindOptions
+		needsProp bool
+		err       bool
+	}{
+		{desc: "nil propagation settings", opts: nil, needsProp: true, err: false},
+		{desc: "empty propagation settings", opts: &mount.BindOptions{}, needsProp: true, err: false},
+		{desc: "private propagation", opts: &mount.BindOptions{Propagation: mount.PropagationPrivate}, err: true},
+		{desc: "rprivate propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRPrivate}, err: true},
+		{desc: "slave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationSlave}, err: true},
+		{desc: "rslave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false},
+		{desc: "shared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationShared}, err: true},
+		{desc: "rshared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false},
+	} {
+		t.Run(test.desc, func(t *testing.T) {
+			test := test
+			for desc, source := range map[string]string{
+				"source is root":    d.root,
+				"source is subpath": filepath.Join(d.root, "a", "b"),
+				"source is parent":  filepath.Dir(d.root),
+				"source is /":       "/",
+			} {
+				t.Run(desc, func(t *testing.T) {
+					mount := mount.Mount{
+						Type:        mount.TypeBind,
+						Source:      source,
+						BindOptions: test.opts,
+					}
+					needsProp, err := d.validateBindDaemonRoot(mount)
+					if (err != nil) != test.err {
+						t.Fatalf("expected err=%v, got: %v", test.err, err)
+					}
+					if test.err {
+						return
+					}
+					if test.needsProp != needsProp {
+						t.Fatalf("expected needsProp=%v, got: %v", test.needsProp, needsProp)
+					}
+				})
+			}
+		})
+	}
+}

components/engine/daemon/volumes_windows.go

@@ -3,6 +3,7 @@ package daemon // import "github.com/docker/docker/daemon"
 import (
 	"sort"
 
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/volume"
@@ -44,3 +45,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er
 func setBindModeIfNull(bind *volume.MountPoint) {
 	return
 }
+
+func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) {
+	return false, nil
+}

components/engine/hack/dockerfile/binaries-commits

@@ -3,12 +3,12 @@
 TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
 
 # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
-RUNC_COMMIT=9f9c96235cc97674e935002fc3d78361b696a69e
+RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5
 
 # containerd is also pinned in vendor.conf. When updating the binary
 # version you may also need to update the vendor version to pick up bug
 # fixes or new APIs.
-CONTAINERD_COMMIT=9b55aab90508bd389d7654c4baf173a981477d55 # v1.0.1
+CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2
 TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574
 LIBNETWORK_COMMIT=fcf1c3b5e57833aaaa756ae3c4140ea54da00319
 VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384

components/engine/integration-cli/docker_cli_build_test.go

@@ -400,13 +400,7 @@ func (s *DockerSuite) TestBuildLastModified(c *check.C) {
 	defer server.Close()
 
 	var out, out2 string
-	var args []string
-	// Temopray workaround for #35963. Will remove this when that issue fixed
-	if runtime.GOARCH == "amd64" {
-		args = []string{"run", name, "ls", "-le", "/file"}
-	} else {
-		args = []string{"run", name, "ls", "-l", "--full-time", "/file"}
-	}
+	args := []string{"run", name, "ls", "-l", "--full-time", "/file"}
 
 	dFmt := `FROM busybox
 ADD %s/file /`

components/engine/integration-cli/docker_cli_diff_test.go

@@ -1,98 +0,0 @@
-package main
-
-import (
-	"strings"
-	"time"
-
-	"github.com/docker/docker/integration-cli/checker"
-	"github.com/docker/docker/integration-cli/cli"
-	"github.com/go-check/check"
-)
-
-// ensure that an added file shows up in docker diff
-func (s *DockerSuite) TestDiffFilenameShownInOutput(c *check.C) {
-	containerCmd := `mkdir /foo; echo xyzzy > /foo/bar`
-	out := cli.DockerCmd(c, "run", "-d", "busybox", "sh", "-c", containerCmd).Combined()
-
-	// Wait for it to exit as cannot diff a running container on Windows, and
-	// it will take a few seconds to exit. Also there's no way in Windows to
-	// differentiate between an Add or a Modify, and all files are under
-	// a "Files/" prefix.
-	containerID := strings.TrimSpace(out)
-	lookingFor := "A /foo/bar"
-	if testEnv.OSType == "windows" {
-		cli.WaitExited(c, containerID, 60*time.Second)
-		lookingFor = "C Files/foo/bar"
-	}
-
-	cleanCID := strings.TrimSpace(out)
-	out = cli.DockerCmd(c, "diff", cleanCID).Combined()
-
-	found := false
-	for _, line := range strings.Split(out, "\n") {
-		if strings.Contains(line, lookingFor) {
-			found = true
-			break
-		}
-	}
-	c.Assert(found, checker.True)
-}
-
-// test to ensure GH #3840 doesn't occur any more
-func (s *DockerSuite) TestDiffEnsureInitLayerFilesAreIgnored(c *check.C) {
-	testRequires(c, DaemonIsLinux)
-	// this is a list of files which shouldn't show up in `docker diff`
-	initLayerFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerenv"}
-	containerCount := 5
-
-	// we might not run into this problem from the first run, so start a few containers
-	for i := 0; i < containerCount; i++ {
-		containerCmd := `echo foo > /root/bar`
-		out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", containerCmd)
-
-		cleanCID := strings.TrimSpace(out)
-		out, _ = dockerCmd(c, "diff", cleanCID)
-
-		for _, filename := range initLayerFiles {
-			c.Assert(out, checker.Not(checker.Contains), filename)
-		}
-	}
-}
-
-func (s *DockerSuite) TestDiffEnsureDefaultDevs(c *check.C) {
-	testRequires(c, DaemonIsLinux)
-	out, _ := dockerCmd(c, "run", "-d", "busybox", "sleep", "0")
-
-	cleanCID := strings.TrimSpace(out)
-	out, _ = dockerCmd(c, "diff", cleanCID)
-
-	expected := map[string]bool{
-		"C /dev":         true,
-		"A /dev/full":    true, // busybox
-		"C /dev/ptmx":    true, // libcontainer
-		"A /dev/mqueue":  true,
-		"A /dev/kmsg":    true,
-		"A /dev/fd":      true,
-		"A /dev/ptmx":    true,
-		"A /dev/null":    true,
-		"A /dev/random":  true,
-		"A /dev/stdout":  true,
-		"A /dev/stderr":  true,
-		"A /dev/tty1":    true,
-		"A /dev/stdin":   true,
-		"A /dev/tty":     true,
-		"A /dev/urandom": true,
-		"A /dev/zero":    true,
-	}
-
-	for _, line := range strings.Split(out, "\n") {
-		c.Assert(line == "" || expected[line], checker.True, check.Commentf(line))
-	}
-}
-
-// https://github.com/docker/docker/pull/14381#discussion_r33859347
-func (s *DockerSuite) TestDiffEmptyArgClientError(c *check.C) {
-	out, _, err := dockerCmdWithError("diff", "")
-	c.Assert(err, checker.NotNil)
-	c.Assert(strings.TrimSpace(out), checker.Contains, "Container name cannot be empty")
-}

components/engine/integration-cli/docker_cli_run_test.go

@@ -2239,14 +2239,7 @@ func (s *DockerSuite) TestRunSlowStdoutConsumer(c *check.C) {
 	// alternate to /dev/zero and /dev/stdout.
 	testRequires(c, DaemonIsLinux)
 
-	// TODO will remove this if issue #35963 fixed
-	var args []string
-	if runtime.GOARCH == "amd64" {
-		args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | catv"}
-	} else {
-		args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"}
-	}
-
+	args := []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"}
 	cont := exec.Command(dockerBinary, args...)
 
 	stdout, err := cont.StdoutPipe()

components/engine/integration-cli/fixtures/load/frozen.go

@@ -37,6 +37,9 @@ func FrozenImagesLinux(client client.APIClient, images ...string) error {
 			if img == "hello-world:frozen" {
 				srcName = "hello-world:latest"
 			}
+			if img == "busybox:1.27-glibc" {
+				img = "busybox:latest"
+			}
 			loadImages = append(loadImages, struct{ srcName, destName string }{
 				srcName:  srcName,
 				destName: img,

components/engine/integration/container/diff_test.go

@@ -0,0 +1,98 @@
+package container // import "github.com/docker/docker/integration/container"
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/integration/internal/container"
+	"github.com/docker/docker/integration/internal/request"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/gotestyourself/gotestyourself/poll"
+	"github.com/gotestyourself/gotestyourself/skip"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// ensure that an added file shows up in docker diff
+func TestDiffFilenameShownInOutput(t *testing.T) {
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+	ctx := context.Background()
+
+	cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", `mkdir /foo; echo xyzzy > /foo/bar`))
+
+	// Wait for it to exit as cannot diff a running container on Windows, and
+	// it will take a few seconds to exit. Also there's no way in Windows to
+	// differentiate between an Add or a Modify, and all files are under
+	// a "Files/" prefix.
+	lookingFor := containertypes.ContainerChangeResponseItem{Kind: archive.ChangeAdd, Path: "/foo/bar"}
+	if testEnv.OSType == "windows" {
+		poll.WaitOn(t, containerIsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond), poll.WithTimeout(60*time.Second))
+		lookingFor = containertypes.ContainerChangeResponseItem{Kind: archive.ChangeModify, Path: "Files/foo/bar"}
+	}
+
+	items, err := client.ContainerDiff(ctx, cID)
+	require.NoError(t, err)
+	assert.Contains(t, items, lookingFor)
+}
+
+// test to ensure GH #3840 doesn't occur any more
+func TestDiffEnsureInitLayerFilesAreIgnored(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
+
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+	ctx := context.Background()
+
+	// this is a list of files which shouldn't show up in `docker diff`
+	initLayerFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerenv"}
+	containerCount := 5
+
+	// we might not run into this problem from the first run, so start a few containers
+	for i := 0; i < containerCount; i++ {
+		cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", `echo foo > /root/bar`))
+
+		items, err := client.ContainerDiff(ctx, cID)
+		require.NoError(t, err)
+		for _, item := range items {
+			assert.NotContains(t, initLayerFiles, item.Path)
+		}
+	}
+}
+
+func TestDiffEnsureDefaultDevs(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
+
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+	ctx := context.Background()
+
+	cID := container.Run(t, ctx, client, container.WithCmd("sleep", "0"))
+
+	items, err := client.ContainerDiff(ctx, cID)
+	require.NoError(t, err)
+
+	expected := []containertypes.ContainerChangeResponseItem{
+		{Kind: archive.ChangeModify, Path: "/dev"},
+		{Kind: archive.ChangeAdd, Path: "/dev/full"},    // busybox
+		{Kind: archive.ChangeModify, Path: "/dev/ptmx"}, // libcontainer
+		{Kind: archive.ChangeAdd, Path: "/dev/mqueue"},
+		{Kind: archive.ChangeAdd, Path: "/dev/kmsg"},
+		{Kind: archive.ChangeAdd, Path: "/dev/fd"},
+		{Kind: archive.ChangeAdd, Path: "/dev/ptmx"},
+		{Kind: archive.ChangeAdd, Path: "/dev/null"},
+		{Kind: archive.ChangeAdd, Path: "/dev/random"},
+		{Kind: archive.ChangeAdd, Path: "/dev/stdout"},
+		{Kind: archive.ChangeAdd, Path: "/dev/stderr"},
+		{Kind: archive.ChangeAdd, Path: "/dev/tty1"},
+		{Kind: archive.ChangeAdd, Path: "/dev/stdin"},
+		{Kind: archive.ChangeAdd, Path: "/dev/tty"},
+		{Kind: archive.ChangeAdd, Path: "/dev/urandom"},
+	}
+
+	for _, item := range items {
+		assert.Contains(t, expected, item)
+	}
+}

components/engine/integration/container/exec_test.go

@@ -6,9 +6,8 @@ import (
 	"testing"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/request"
 	"github.com/stretchr/testify/require"
 )
@@ -18,22 +17,9 @@ func TestExec(t *testing.T) {
 	ctx := context.Background()
 	client := request.NewAPIClient(t)
 
-	container, err := client.ContainerCreate(ctx,
-		&container.Config{
-			Image:      "busybox",
-			Tty:        true,
-			WorkingDir: "/root",
-			Cmd:        strslice.StrSlice([]string{"top"}),
-		},
-		&container.HostConfig{},
-		&network.NetworkingConfig{},
-		"foo",
-	)
-	require.NoError(t, err)
-	err = client.ContainerStart(ctx, container.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
+	cID := container.Run(t, ctx, client, container.WithTty(true), container.WithWorkingDir("/root"))
 
-	id, err := client.ContainerExecCreate(ctx, container.ID,
+	id, err := client.ContainerExecCreate(ctx, cID,
 		types.ExecConfig{
 			WorkingDir:   "/tmp",
 			Env:          strslice.StrSlice([]string{"FOO=BAR"}),

components/engine/integration/container/health_test.go

@@ -6,13 +6,11 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/network"
-	"github.com/docker/docker/api/types/strslice"
+	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/request"
 	"github.com/gotestyourself/gotestyourself/poll"
-	"github.com/stretchr/testify/require"
 )
 
 // TestHealthCheckWorkdir verifies that health-checks inherit the containers'
@@ -22,27 +20,15 @@ func TestHealthCheckWorkdir(t *testing.T) {
 	ctx := context.Background()
 	client := request.NewAPIClient(t)
 
-	c, err := client.ContainerCreate(ctx,
-		&container.Config{
-			Image:      "busybox",
-			Tty:        true,
-			WorkingDir: "/foo",
-			Cmd:        strslice.StrSlice([]string{"top"}),
-			Healthcheck: &container.HealthConfig{
-				Test:     []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"},
-				Interval: 50 * time.Millisecond,
-				Retries:  3,
-			},
-		},
-		&container.HostConfig{},
-		&network.NetworkingConfig{},
-		"healthtest",
-	)
-	require.NoError(t, err)
-	err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
+	cID := container.Run(t, ctx, client, container.WithTty(true), container.WithWorkingDir("/foo"), func(c *container.TestContainerConfig) {
+		c.Config.Healthcheck = &containertypes.HealthConfig{
+			Test:     []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"},
+			Interval: 50 * time.Millisecond,
+			Retries:  3,
+		}
+	})
 
-	poll.WaitOn(t, pollForHealthStatus(ctx, client, c.ID, types.Healthy), poll.WithDelay(100*time.Millisecond))
+	poll.WaitOn(t, pollForHealthStatus(ctx, client, cID, types.Healthy), poll.WithDelay(100*time.Millisecond))
 }
 
 func pollForHealthStatus(ctx context.Context, client client.APIClient, containerID string, healthStatus string) func(log poll.LogT) poll.Result {

components/engine/integration/container/links_linux_test.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/request"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/gotestyourself/gotestyourself/poll"
@@ -28,24 +28,11 @@ func TestLinksEtcHostsContentMatch(t *testing.T) {
 	client := request.NewAPIClient(t)
 	ctx := context.Background()
 
-	c, err := client.ContainerCreate(ctx,
-		&container.Config{
-			Image: "busybox",
-			Cmd:   []string{"cat", "/etc/hosts"},
-		},
-		&container.HostConfig{
-			NetworkMode: "host",
-		},
-		nil,
-		"")
-	require.NoError(t, err)
+	cID := container.Run(t, ctx, client, container.WithCmd("cat", "/etc/hosts"), container.WithNetworkMode("host"))
 
-	err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
+	poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond))
 
-	poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond))
-
-	body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{
+	body, err := client.ContainerLogs(ctx, cID, types.ContainerLogsOptions{
 		ShowStdout: true,
 	})
 	require.NoError(t, err)

components/engine/integration/container/logs_test.go

@@ -0,0 +1,33 @@
+package container
+
+import (
+	"context"
+	"io/ioutil"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/integration/internal/container"
+	"github.com/docker/docker/integration/internal/request"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/stretchr/testify/assert"
+)
+
+// Regression test for #35370
+// Makes sure that when following we don't get an EOF error when there are no logs
+func TestLogsFollowTailEmpty(t *testing.T) {
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+	ctx := context.Background()
+
+	id := container.Run(t, ctx, client, container.WithCmd("sleep", "100000"))
+	defer client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true})
+
+	logs, err := client.ContainerLogs(ctx, id, types.ContainerLogsOptions{ShowStdout: true, Tail: "2"})
+	if logs != nil {
+		defer logs.Close()
+	}
+	assert.NoError(t, err)
+
+	_, err = stdcopy.StdCopy(ioutil.Discard, ioutil.Discard, logs)
+	assert.NoError(t, err)
+}

components/engine/integration/container/mounts_linux_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"path/filepath"
 	"testing"
 
 	"github.com/docker/docker/api/types"
@@ -12,6 +13,7 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/integration-cli/daemon"
+	"github.com/docker/docker/integration/util/request"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/pkg/system"
 	"github.com/gotestyourself/gotestyourself/fs"
@@ -51,10 +53,10 @@ func TestContainerShmNoLeak(t *testing.T) {
 	hc := container.HostConfig{
 		Mounts: []mount.Mount{
 			{
-				Type:        mount.TypeBind,
-				Source:      d.Root,
-				Target:      "/testdaemonroot",
-				BindOptions: &mount.BindOptions{Propagation: mount.PropagationRPrivate}},
+				Type:   mount.TypeBind,
+				Source: d.Root,
+				Target: "/testdaemonroot",
+			},
 		},
 	}
 	cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)}
@@ -150,3 +152,129 @@ func TestContainerNetworkMountsNoChown(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, uint32(0), statT.UID(), "bind mounted network file should not change ownership from root")
 }
+
+func TestMountDaemonRoot(t *testing.T) {
+	t.Parallel()
+
+	client := request.NewAPIClient(t)
+	ctx := context.Background()
+	info, err := client.Info(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range []struct {
+		desc        string
+		propagation mount.Propagation
+		expected    mount.Propagation
+	}{
+		{
+			desc:        "default",
+			propagation: "",
+			expected:    mount.PropagationRSlave,
+		},
+		{
+			desc:        "private",
+			propagation: mount.PropagationPrivate,
+		},
+		{
+			desc:        "rprivate",
+			propagation: mount.PropagationRPrivate,
+		},
+		{
+			desc:        "slave",
+			propagation: mount.PropagationSlave,
+		},
+		{
+			desc:        "rslave",
+			propagation: mount.PropagationRSlave,
+			expected:    mount.PropagationRSlave,
+		},
+		{
+			desc:        "shared",
+			propagation: mount.PropagationShared,
+		},
+		{
+			desc:        "rshared",
+			propagation: mount.PropagationRShared,
+			expected:    mount.PropagationRShared,
+		},
+	} {
+		t.Run(test.desc, func(t *testing.T) {
+			test := test
+			t.Parallel()
+
+			propagationSpec := fmt.Sprintf(":%s", test.propagation)
+			if test.propagation == "" {
+				propagationSpec = ""
+			}
+			bindSpecRoot := info.DockerRootDir + ":" + "/foo" + propagationSpec
+			bindSpecSub := filepath.Join(info.DockerRootDir, "containers") + ":/foo" + propagationSpec
+
+			for name, hc := range map[string]*container.HostConfig{
+				"bind root":    {Binds: []string{bindSpecRoot}},
+				"bind subpath": {Binds: []string{bindSpecSub}},
+				"mount root": {
+					Mounts: []mount.Mount{
+						{
+							Type:        mount.TypeBind,
+							Source:      info.DockerRootDir,
+							Target:      "/foo",
+							BindOptions: &mount.BindOptions{Propagation: test.propagation},
+						},
+					},
+				},
+				"mount subpath": {
+					Mounts: []mount.Mount{
+						{
+							Type:        mount.TypeBind,
+							Source:      filepath.Join(info.DockerRootDir, "containers"),
+							Target:      "/foo",
+							BindOptions: &mount.BindOptions{Propagation: test.propagation},
+						},
+					},
+				},
+			} {
+				t.Run(name, func(t *testing.T) {
+					hc := hc
+					t.Parallel()
+
+					c, err := client.ContainerCreate(ctx, &container.Config{
+						Image: "busybox",
+						Cmd:   []string{"true"},
+					}, hc, nil, "")
+
+					if err != nil {
+						if test.expected != "" {
+							t.Fatal(err)
+						}
+						// expected an error, so this is ok and should not continue
+						return
+					}
+					if test.expected == "" {
+						t.Fatal("expected create to fail")
+					}
+
+					defer func() {
+						if err := client.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{Force: true}); err != nil {
+							panic(err)
+						}
+					}()
+
+					inspect, err := client.ContainerInspect(ctx, c.ID)
+					if err != nil {
+						t.Fatal(err)
+					}
+					if len(inspect.Mounts) != 1 {
+						t.Fatalf("unexpected number of mounts: %+v", inspect.Mounts)
+					}
+
+					m := inspect.Mounts[0]
+					if m.Propagation != test.expected {
+						t.Fatalf("got unexpected propagation mode, expected %q, got: %v", test.expected, m.Propagation)
+					}
+				})
+			}
+		})
+	}
+}

components/engine/integration/container/nat_test.go

@@ -12,8 +12,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/request"
 	"github.com/docker/go-connections/nat"
 	"github.com/gotestyourself/gotestyourself/poll"
@@ -67,25 +66,12 @@ func TestNetworkLoopbackNat(t *testing.T) {
 
 	client := request.NewAPIClient(t)
 	ctx := context.Background()
-	c, err := client.ContainerCreate(ctx,
-		&container.Config{
-			Image: "busybox",
-			Cmd:   []string{"sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())},
-			Tty:   true,
-		},
-		&container.HostConfig{
-			NetworkMode: "container:server",
-		},
-		nil,
-		"")
-	require.NoError(t, err)
 
-	err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
+	cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())), container.WithTty(true), container.WithNetworkMode("container:server"))
 
-	poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond))
+	poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond))
 
-	body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{
+	body, err := client.ContainerLogs(ctx, cID, types.ContainerLogsOptions{
 		ShowStdout: true,
 	})
 	require.NoError(t, err)
@@ -102,34 +88,19 @@ func startServerContainer(t *testing.T, msg string, port int) string {
 	client := request.NewAPIClient(t)
 	ctx := context.Background()
 
-	c, err := client.ContainerCreate(ctx,
-		&container.Config{
-			Image: "busybox",
-			Cmd:   []string{"sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)},
-			ExposedPorts: map[nat.Port]struct{}{
-				nat.Port(fmt.Sprintf("%d/tcp", port)): {},
-			},
-		},
-		&container.HostConfig{
-			PortBindings: nat.PortMap{
-				nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{
-					{
-						HostPort: fmt.Sprintf("%d", port),
-					},
+	cID := container.Run(t, ctx, client, container.WithName("server"), container.WithCmd("sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)), container.WithExposedPorts(fmt.Sprintf("%d/tcp", port)), func(c *container.TestContainerConfig) {
+		c.HostConfig.PortBindings = nat.PortMap{
+			nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{
+				{
+					HostPort: fmt.Sprintf("%d", port),
 				},
 			},
-		},
-		&network.NetworkingConfig{},
-		"server",
-	)
-	require.NoError(t, err)
+		}
+	})
 
-	err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
+	poll.WaitOn(t, containerIsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond))
 
-	poll.WaitOn(t, containerIsInState(ctx, client, c.ID, "running"), poll.WithDelay(100*time.Millisecond))
-
-	return c.ID
+	return cID
 }
 
 func getExternalAddress(t *testing.T) net.IP {

components/engine/integration/container/stats_test.go

@@ -8,8 +8,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/request"
 	"github.com/gotestyourself/gotestyourself/poll"
 	"github.com/gotestyourself/gotestyourself/skip"
@@ -27,23 +26,11 @@ func TestStats(t *testing.T) {
 	info, err := client.Info(ctx)
 	require.NoError(t, err)
 
-	c, err := client.ContainerCreate(ctx,
-		&container.Config{
-			Cmd:   []string{"top"},
-			Image: "busybox",
-		},
-		&container.HostConfig{},
-		&network.NetworkingConfig{},
-		"",
-	)
-	require.NoError(t, err)
+	cID := container.Run(t, ctx, client)
 
-	err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
+	poll.WaitOn(t, containerIsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond))
 
-	poll.WaitOn(t, containerIsInState(ctx, client, c.ID, "running"), poll.WithDelay(100*time.Millisecond))
-
-	resp, err := client.ContainerStats(context.Background(), c.ID, false)
+	resp, err := client.ContainerStats(ctx, cID, false)
 	require.NoError(t, err)
 	defer resp.Body.Close()
 

components/engine/integration/internal/container/ops.go

@@ -1,6 +1,10 @@
 package container
 
-import "github.com/docker/docker/api/types/strslice"
+import (
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/go-connections/nat"
+)
 
 // WithName sets the name of the container
 func WithName(name string) func(*TestContainerConfig) {
@@ -22,3 +26,34 @@ func WithCmd(cmds ...string) func(*TestContainerConfig) {
 		c.Config.Cmd = strslice.StrSlice(cmds)
 	}
 }
+
+// WithNetworkMode sets the network mode of the container
+func WithNetworkMode(mode string) func(*TestContainerConfig) {
+	return func(c *TestContainerConfig) {
+		c.HostConfig.NetworkMode = containertypes.NetworkMode(mode)
+	}
+}
+
+// WithExposedPorts sets the exposed ports of the container
+func WithExposedPorts(ports ...string) func(*TestContainerConfig) {
+	return func(c *TestContainerConfig) {
+		c.Config.ExposedPorts = map[nat.Port]struct{}{}
+		for _, port := range ports {
+			c.Config.ExposedPorts[nat.Port(port)] = struct{}{}
+		}
+	}
+}
+
+// WithTty sets the TTY mode of the container
+func WithTty(tty bool) func(*TestContainerConfig) {
+	return func(c *TestContainerConfig) {
+		c.Config.Tty = tty
+	}
+}
+
+// WithWorkingDir sets the working dir of the container
+func WithWorkingDir(dir string) func(*TestContainerConfig) {
+	return func(c *TestContainerConfig) {
+		c.Config.WorkingDir = dir
+	}
+}

components/engine/integration/plugin/authz/authz_plugin_test.go

@@ -19,10 +19,9 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
 	eventtypes "github.com/docker/docker/api/types/events"
-	networktypes "github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/request"
 	"github.com/docker/docker/internal/test/environment"
 	"github.com/docker/docker/pkg/authorization"
@@ -91,17 +90,15 @@ func TestAuthZPluginAllowRequest(t *testing.T) {
 	client, err := d.NewClient()
 	require.Nil(t, err)
 
-	// Ensure command successful
-	createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "")
-	require.Nil(t, err)
+	ctx := context.Background()
 
-	err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{})
-	require.Nil(t, err)
+	// Ensure command successful
+	cID := container.Run(t, ctx, client)
 
 	assertURIRecorded(t, ctrl.requestsURIs, "/containers/create")
-	assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", createResponse.ID))
+	assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", cID))
 
-	_, err = client.ServerVersion(context.Background())
+	_, err = client.ServerVersion(ctx)
 	require.Nil(t, err)
 	require.Equal(t, 1, ctrl.versionReqCount)
 	require.Equal(t, 1, ctrl.versionResCount)
@@ -213,19 +210,17 @@ func TestAuthZPluginAllowEventStream(t *testing.T) {
 	client, err := d.NewClient()
 	require.Nil(t, err)
 
+	ctx := context.Background()
+
 	startTime := strconv.FormatInt(systemTime(t, client, testEnv).Unix(), 10)
 	events, errs, cancel := systemEventsSince(client, startTime)
 	defer cancel()
 
 	// Create a container and wait for the creation events
-	createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "")
-	require.Nil(t, err)
-
-	err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{})
-	require.Nil(t, err)
+	cID := container.Run(t, ctx, client)
 
 	for i := 0; i < 100; i++ {
-		c, err := client.ContainerInspect(context.Background(), createResponse.ID)
+		c, err := client.ContainerInspect(ctx, cID)
 		require.Nil(t, err)
 		if c.State.Running {
 			break
@@ -241,7 +236,7 @@ func TestAuthZPluginAllowEventStream(t *testing.T) {
 	for !created && !started {
 		select {
 		case event := <-events:
-			if event.Type == eventtypes.ContainerEventType && event.Actor.ID == createResponse.ID {
+			if event.Type == eventtypes.ContainerEventType && event.Actor.ID == cID {
 				if event.Action == "create" {
 					created = true
 				}
@@ -264,7 +259,7 @@ func TestAuthZPluginAllowEventStream(t *testing.T) {
 	// authorization plugin
 	assertURIRecorded(t, ctrl.requestsURIs, "/events")
 	assertURIRecorded(t, ctrl.requestsURIs, "/containers/create")
-	assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", createResponse.ID))
+	assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", cID))
 }
 
 func systemTime(t *testing.T, client client.APIClient, testEnv *environment.Execution) time.Time {
@@ -347,6 +342,8 @@ func TestAuthZPluginEnsureLoadImportWorking(t *testing.T) {
 	client, err := d.NewClient()
 	require.Nil(t, err)
 
+	ctx := context.Background()
+
 	tmp, err := ioutil.TempDir("", "test-authz-load-import")
 	require.Nil(t, err)
 	defer os.RemoveAll(tmp)
@@ -360,13 +357,9 @@ func TestAuthZPluginEnsureLoadImportWorking(t *testing.T) {
 
 	exportedImagePath := filepath.Join(tmp, "export.tar")
 
-	createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "")
-	require.Nil(t, err)
+	cID := container.Run(t, ctx, client)
 
-	err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{})
-	require.Nil(t, err)
-
-	responseReader, err := client.ContainerExport(context.Background(), createResponse.ID)
+	responseReader, err := client.ContainerExport(context.Background(), cID)
 	require.Nil(t, err)
 	defer responseReader.Close()
 	file, err := os.Create(exportedImagePath)

components/engine/integration/plugin/authz/authz_plugin_v2_test.go

@@ -11,11 +11,10 @@ import (
 	"testing"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
-	networktypes "github.com/docker/docker/api/types/network"
 	volumetypes "github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/requirement"
 	"github.com/gotestyourself/gotestyourself/skip"
 	"github.com/stretchr/testify/require"
@@ -47,6 +46,8 @@ func TestAuthZPluginV2AllowNonVolumeRequest(t *testing.T) {
 	client, err := d.NewClient()
 	require.Nil(t, err)
 
+	ctx := context.Background()
+
 	// Install authz plugin
 	err = pluginInstallGrantAllPermissions(client, authzPluginNameWithTag)
 	require.Nil(t, err)
@@ -56,13 +57,9 @@ func TestAuthZPluginV2AllowNonVolumeRequest(t *testing.T) {
 	d.LoadBusybox(t)
 
 	// Ensure docker run command and accompanying docker ps are successful
-	createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "")
-	require.Nil(t, err)
+	cID := container.Run(t, ctx, client)
 
-	err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{})
-	require.Nil(t, err)
-
-	_, err = client.ContainerInspect(context.Background(), createResponse.ID)
+	_, err = client.ContainerInspect(ctx, cID)
 	require.Nil(t, err)
 }
 

components/engine/internal/test/environment/protect.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var frozenImages = []string{"busybox:latest", "hello-world:frozen", "debian:jessie"}
+var frozenImages = []string{"busybox:1.27-glibc", "hello-world:frozen", "debian:jessie"}
 
 type protectedElements struct {
 	containers map[string]struct{}

components/engine/poule.yml

@@ -107,16 +107,13 @@
             users: [
                 "aaronlehmann",
                 "akihirosuda",
-                "aluzzardi",
                 "coolljt0725",
                 "cpuguy83",
                 "crosbymichael",
                 "dnephin",
                 "duglin",
-                "ehazlett",
                 "johnstep",
                 "justincormack",
-                "lk4d4",
                 "mhbauer",
                 "mlaventure",
                 "runcom",

components/engine/vendor.conf

@@ -66,7 +66,7 @@ github.com/pborman/uuid v1.0
 google.golang.org/grpc v1.3.0
 
 # When updating, also update RUNC_COMMIT in hack/dockerfile/binaries-commits accordingly
-github.com/opencontainers/runc 9f9c96235cc97674e935002fc3d78361b696a69e
+github.com/opencontainers/runc 6c55f98695e902427906eed2c799e566e3d3dfb5
 github.com/opencontainers/runtime-spec v1.0.1
 github.com/opencontainers/image-spec v1.0.1
 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
@@ -106,13 +106,13 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 # containerd
 github.com/containerd/containerd 3fa104f843ec92328912e042b767d26825f202aa
 github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6
-github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83
-github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f
+github.com/containerd/continuity 992a5f112bd2211d0983a1cc8562d2882848f3a3
+github.com/containerd/cgroups c0710c92e8b3a44681d1321dcfd1360fc5c6c089
 github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e
-github.com/containerd/go-runc ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7
+github.com/containerd/go-runc 4f6e87ae043f859a38255247b49c9abc262d002f
 github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788
 github.com/dmcgowan/go-tar go1.10
-github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f
+github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577
 
 # cluster
 github.com/docker/swarmkit 68a376dc30d8c4001767c39456b990dbd821371b

components/engine/vendor/github.com/containerd/cgroups/blkio.go

@@ -3,12 +3,12 @@ package cgroups
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -105,8 +105,13 @@ func (b *blkioController) Stat(path string, stats *Metrics) error {
 			},
 		)
 	}
+	f, err := os.Open("/proc/diskstats")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
 
-	devices, err := getDevices("/dev")
+	devices, err := getDevices(f)
 	if err != nil {
 		return err
 	}
@@ -268,50 +273,32 @@ type deviceKey struct {
 // getDevices makes a best effort attempt to read all the devices into a map
 // keyed by major and minor number. Since devices may be mapped multiple times,
 // we err on taking the first occurrence.
-func getDevices(path string) (map[deviceKey]string, error) {
-	// TODO(stevvooe): We are ignoring lots of errors. It might be kind of
-	// challenging to debug this if we aren't mapping devices correctly.
-	// Consider logging these errors.
-	devices := map[deviceKey]string{}
-	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
+func getDevices(r io.Reader) (map[deviceKey]string, error) {
+
+	var (
+		s       = bufio.NewScanner(r)
+		devices = make(map[deviceKey]string)
+	)
+	for s.Scan() {
+		fields := strings.Fields(s.Text())
+		major, err := strconv.Atoi(fields[0])
 		if err != nil {
-			return err
+			return nil, err
 		}
-		switch {
-		case fi.IsDir():
-			switch fi.Name() {
-			case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts":
-				return filepath.SkipDir
-			default:
-				return nil
-			}
-		case fi.Name() == "console":
-			return nil
-		default:
-			if fi.Mode()&os.ModeDevice == 0 {
-				// skip non-devices
-				return nil
-			}
-
-			st, ok := fi.Sys().(*syscall.Stat_t)
-			if !ok {
-				return fmt.Errorf("%s: unable to convert to system stat", p)
-			}
-
-			key := deviceKey{major(st.Rdev), minor(st.Rdev)}
-			if _, ok := devices[key]; ok {
-				return nil // skip it if we have already populated the path.
-			}
-
-			devices[key] = p
+		minor, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return nil, err
 		}
-
-		return nil
-	}); err != nil {
-		return nil, err
+		key := deviceKey{
+			major: uint64(major),
+			minor: uint64(minor),
+		}
+		if _, ok := devices[key]; ok {
+			continue
+		}
+		devices[key] = filepath.Join("/dev", fields[2])
 	}
-
-	return devices, nil
+	return devices, s.Err()
 }
 
 func major(devNumber uint64) uint64 {

components/engine/vendor/github.com/containerd/cgroups/errors.go

@@ -12,7 +12,7 @@ var (
 	ErrFreezerNotSupported      = errors.New("cgroups: freezer cgroup not supported on this system")
 	ErrMemoryNotSupported       = errors.New("cgroups: memory cgroup not supported on this system")
 	ErrCgroupDeleted            = errors.New("cgroups: cgroup deleted")
-	ErrNoCgroupMountDestination = errors.New("cgroups: cannot found cgroup mount destination")
+	ErrNoCgroupMountDestination = errors.New("cgroups: cannot find cgroup mount destination")
 )
 
 // ErrorHandler is a function that handles and acts on errors

components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go

@@ -1,15 +0,0 @@
-package devices
-
-// from /usr/include/sys/types.h
-
-func getmajor(dev int32) uint64 {
-	return (uint64(dev) >> 24) & 0xff
-}
-
-func getminor(dev int32) uint64 {
-	return uint64(dev) & 0xffffff
-}
-
-func makedev(major int, minor int) int {
-	return ((major << 24) | minor)
-}

components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go

@@ -1,23 +0,0 @@
-// +build solaris,!cgo
-
-//
-// Implementing the functions below requires cgo support.  Non-cgo stubs
-// versions are defined below to enable cross-compilation of source code
-// that depends on these functions, but the resultant cross-compiled
-// binaries cannot actually be used.  If the stub function(s) below are
-// actually invoked they will cause the calling process to exit.
-//
-
-package devices
-
-func getmajor(dev uint64) uint64 {
-	panic("getmajor() support requires cgo.")
-}
-
-func getminor(dev uint64) uint64 {
-	panic("getminor() support requires cgo.")
-}
-
-func makedev(major int, minor int) int {
-	panic("makedev() support requires cgo.")
-}

components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go

@@ -1,15 +0,0 @@
-package devices
-
-// from /usr/include/sys/types.h
-
-func getmajor(dev uint32) uint64 {
-	return (uint64(dev) >> 24) & 0xff
-}
-
-func getminor(dev uint32) uint64 {
-	return uint64(dev) & 0xffffff
-}
-
-func makedev(major int, minor int) int {
-	return ((major << 24) | minor)
-}

components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go

@@ -1,15 +0,0 @@
-package devices
-
-// from /usr/include/linux/kdev_t.h
-
-func getmajor(dev uint64) uint64 {
-	return dev >> 8
-}
-
-func getminor(dev uint64) uint64 {
-	return dev & 0xff
-}
-
-func makedev(major int, minor int) int {
-	return ((major << 8) | minor)
-}

components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go

@@ -1,18 +0,0 @@
-// +build cgo
-
-package devices
-
-//#include <sys/mkdev.h>
-import "C"
-
-func getmajor(dev uint64) uint64 {
-	return uint64(C.major(C.dev_t(dev)))
-}
-
-func getminor(dev uint64) uint64 {
-	return uint64(C.minor(C.dev_t(dev)))
-}
-
-func makedev(major int, minor int) int {
-	return int(C.makedev(C.major_t(major), C.minor_t(minor)))
-}

components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"syscall"
+
+	"golang.org/x/sys/unix"
 )
 
 func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
@@ -14,42 +16,43 @@ func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
 		return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo")
 	}
 
-	return getmajor(sys.Rdev), getminor(sys.Rdev), nil
+	dev := uint64(sys.Rdev)
+	return uint64(unix.Major(dev)), uint64(unix.Minor(dev)), nil
 }
 
 // mknod provides a shortcut for syscall.Mknod
 func Mknod(p string, mode os.FileMode, maj, min int) error {
 	var (
 		m   = syscallMode(mode.Perm())
-		dev int
+		dev uint64
 	)
 
 	if mode&os.ModeDevice != 0 {
-		dev = makedev(maj, min)
+		dev = unix.Mkdev(uint32(maj), uint32(min))
 
 		if mode&os.ModeCharDevice != 0 {
-			m |= syscall.S_IFCHR
+			m |= unix.S_IFCHR
 		} else {
-			m |= syscall.S_IFBLK
+			m |= unix.S_IFBLK
 		}
 	} else if mode&os.ModeNamedPipe != 0 {
-		m |= syscall.S_IFIFO
+		m |= unix.S_IFIFO
 	}
 
-	return syscall.Mknod(p, m, dev)
+	return unix.Mknod(p, m, int(dev))
 }
 
 // syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
 func syscallMode(i os.FileMode) (o uint32) {
 	o |= uint32(i.Perm())
 	if i&os.ModeSetuid != 0 {
-		o |= syscall.S_ISUID
+		o |= unix.S_ISUID
 	}
 	if i&os.ModeSetgid != 0 {
-		o |= syscall.S_ISGID
+		o |= unix.S_ISGID
 	}
 	if i&os.ModeSticky != 0 {
-		o |= syscall.S_ISVTX
+		o |= unix.S_ISVTX
 	}
 	return
 }

components/engine/vendor/github.com/containerd/continuity/fs/copy.go

@@ -0,0 +1,119 @@
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var bufferPool = &sync.Pool{
+	New: func() interface{} {
+		buffer := make([]byte, 32*1024)
+		return &buffer
+	},
+}
+
+// CopyDir copies the directory from src to dst.
+// Most efficient copy of files is attempted.
+func CopyDir(dst, src string) error {
+	inodes := map[uint64]string{}
+	return copyDirectory(dst, src, inodes)
+}
+
+func copyDirectory(dst, src string, inodes map[uint64]string) error {
+	stat, err := os.Stat(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to stat %s", src)
+	}
+	if !stat.IsDir() {
+		return errors.Errorf("source is not directory")
+	}
+
+	if st, err := os.Stat(dst); err != nil {
+		if err := os.Mkdir(dst, stat.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to mkdir %s", dst)
+		}
+	} else if !st.IsDir() {
+		return errors.Errorf("cannot copy to non-directory: %s", dst)
+	} else {
+		if err := os.Chmod(dst, stat.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod on %s", dst)
+		}
+	}
+
+	fis, err := ioutil.ReadDir(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read %s", src)
+	}
+
+	if err := copyFileInfo(stat, dst); err != nil {
+		return errors.Wrapf(err, "failed to copy file info for %s", dst)
+	}
+
+	for _, fi := range fis {
+		source := filepath.Join(src, fi.Name())
+		target := filepath.Join(dst, fi.Name())
+
+		switch {
+		case fi.IsDir():
+			if err := copyDirectory(target, source, inodes); err != nil {
+				return err
+			}
+			continue
+		case (fi.Mode() & os.ModeType) == 0:
+			link, err := getLinkSource(target, fi, inodes)
+			if err != nil {
+				return errors.Wrap(err, "failed to get hardlink")
+			}
+			if link != "" {
+				if err := os.Link(link, target); err != nil {
+					return errors.Wrap(err, "failed to create hard link")
+				}
+			} else if err := copyFile(source, target); err != nil {
+				return errors.Wrap(err, "failed to copy files")
+			}
+		case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink:
+			link, err := os.Readlink(source)
+			if err != nil {
+				return errors.Wrapf(err, "failed to read link: %s", source)
+			}
+			if err := os.Symlink(link, target); err != nil {
+				return errors.Wrapf(err, "failed to create symlink: %s", target)
+			}
+		case (fi.Mode() & os.ModeDevice) == os.ModeDevice:
+			if err := copyDevice(target, fi); err != nil {
+				return errors.Wrapf(err, "failed to create device")
+			}
+		default:
+			// TODO: Support pipes and sockets
+			return errors.Wrapf(err, "unsupported mode %s", fi.Mode())
+		}
+		if err := copyFileInfo(fi, target); err != nil {
+			return errors.Wrap(err, "failed to copy file info")
+		}
+
+		if err := copyXAttrs(target, source); err != nil {
+			return errors.Wrap(err, "failed to copy xattrs")
+		}
+	}
+
+	return nil
+}
+
+func copyFile(source, target string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open source %s", source)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open target %s", target)
+	}
+	defer tgt.Close()
+
+	return copyFileContent(tgt, src)
+}

components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go

@@ -0,0 +1,95 @@
+package fs
+
+import (
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/containerd/continuity/sysx"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	st := fi.Sys().(*syscall.Stat_t)
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
+		if os.IsPermission(err) {
+			// Normally if uid/gid are the same this would be a no-op, but some
+			// filesystems may still return EPERM... for instance NFS does this.
+			// In such a case, this is not an error.
+			if dstStat, err2 := os.Lstat(name); err2 == nil {
+				st2 := dstStat.Sys().(*syscall.Stat_t)
+				if st.Uid == st2.Uid && st.Gid == st2.Gid {
+					err = nil
+				}
+			}
+		}
+		if err != nil {
+			return errors.Wrapf(err, "failed to chown %s", name)
+		}
+	}
+
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
+		if err := os.Chmod(name, fi.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod %s", name)
+		}
+	}
+
+	timespec := []unix.Timespec{unix.Timespec(StatAtime(st)), unix.Timespec(StatMtime(st))}
+	if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
+		return errors.Wrapf(err, "failed to utime %s", name)
+	}
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	st, err := src.Stat()
+	if err != nil {
+		return errors.Wrap(err, "unable to stat source")
+	}
+
+	n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0)
+	if err != nil {
+		if err != unix.ENOSYS && err != unix.EXDEV {
+			return errors.Wrap(err, "copy file range failed")
+		}
+
+		buf := bufferPool.Get().(*[]byte)
+		_, err = io.CopyBuffer(dst, src, *buf)
+		bufferPool.Put(buf)
+		return err
+	}
+
+	if int64(n) != st.Size() {
+		return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
+	}
+
+	return nil
+}
+
+func copyXAttrs(dst, src string) error {
+	xattrKeys, err := sysx.LListxattr(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
+	}
+	for _, xattr := range xattrKeys {
+		data, err := sysx.LGetxattr(src, xattr)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
+		}
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
+		}
+	}
+
+	return nil
+}
+
+func copyDevice(dst string, fi os.FileInfo) error {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return errors.New("unsupported stat type")
+	}
+	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
+}

components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go

@@ -0,0 +1,80 @@
+// +build solaris darwin freebsd
+
+package fs
+
+import (
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/containerd/continuity/sysx"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	st := fi.Sys().(*syscall.Stat_t)
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
+		if os.IsPermission(err) {
+			// Normally if uid/gid are the same this would be a no-op, but some
+			// filesystems may still return EPERM... for instance NFS does this.
+			// In such a case, this is not an error.
+			if dstStat, err2 := os.Lstat(name); err2 == nil {
+				st2 := dstStat.Sys().(*syscall.Stat_t)
+				if st.Uid == st2.Uid && st.Gid == st2.Gid {
+					err = nil
+				}
+			}
+		}
+		if err != nil {
+			return errors.Wrapf(err, "failed to chown %s", name)
+		}
+	}
+
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
+		if err := os.Chmod(name, fi.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod %s", name)
+		}
+	}
+
+	timespec := []syscall.Timespec{StatAtime(st), StatMtime(st)}
+	if err := syscall.UtimesNano(name, timespec); err != nil {
+		return errors.Wrapf(err, "failed to utime %s", name)
+	}
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	buf := bufferPool.Get().(*[]byte)
+	_, err := io.CopyBuffer(dst, src, *buf)
+	bufferPool.Put(buf)
+
+	return err
+}
+
+func copyXAttrs(dst, src string) error {
+	xattrKeys, err := sysx.LListxattr(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
+	}
+	for _, xattr := range xattrKeys {
+		data, err := sysx.LGetxattr(src, xattr)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
+		}
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
+		}
+	}
+
+	return nil
+}
+
+func copyDevice(dst string, fi os.FileInfo) error {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return errors.New("unsupported stat type")
+	}
+	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
+}

components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go

@@ -0,0 +1,33 @@
+package fs
+
+import (
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	if err := os.Chmod(name, fi.Mode()); err != nil {
+		return errors.Wrapf(err, "failed to chmod %s", name)
+	}
+
+	// TODO: copy windows specific metadata
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	buf := bufferPool.Get().(*[]byte)
+	_, err := io.CopyBuffer(dst, src, *buf)
+	bufferPool.Put(buf)
+	return err
+}
+
+func copyXAttrs(dst, src string) error {
+	return nil
+}
+
+func copyDevice(dst string, fi os.FileInfo) error {
+	return errors.New("device copy not supported")
+}

components/engine/vendor/github.com/containerd/continuity/fs/diff.go

@@ -0,0 +1,310 @@
+package fs
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/sirupsen/logrus"
+)
+
+// ChangeKind is the type of modification that
+// a change is making.
+type ChangeKind int
+
+const (
+	// ChangeKindUnmodified represents an unmodified
+	// file
+	ChangeKindUnmodified = iota
+
+	// ChangeKindAdd represents an addition of
+	// a file
+	ChangeKindAdd
+
+	// ChangeKindModify represents a change to
+	// an existing file
+	ChangeKindModify
+
+	// ChangeKindDelete represents a delete of
+	// a file
+	ChangeKindDelete
+)
+
+func (k ChangeKind) String() string {
+	switch k {
+	case ChangeKindUnmodified:
+		return "unmodified"
+	case ChangeKindAdd:
+		return "add"
+	case ChangeKindModify:
+		return "modify"
+	case ChangeKindDelete:
+		return "delete"
+	default:
+		return ""
+	}
+}
+
+// Change represents single change between a diff and its parent.
+type Change struct {
+	Kind ChangeKind
+	Path string
+}
+
+// ChangeFunc is the type of function called for each change
+// computed during a directory changes calculation.
+type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error
+
+// Changes computes changes between two directories calling the
+// given change function for each computed change. The first
+// directory is intended to the base directory and second
+// directory the changed directory.
+//
+// The change callback is called by the order of path names and
+// should be appliable in that order.
+//  Due to this apply ordering, the following is true
+//  - Removed directory trees only create a single change for the root
+//    directory removed. Remaining changes are implied.
+//  - A directory which is modified to become a file will not have
+//    delete entries for sub-path items, their removal is implied
+//    by the removal of the parent directory.
+//
+// Opaque directories will not be treated specially and each file
+// removed from the base directory will show up as a removal.
+//
+// File content comparisons will be done on files which have timestamps
+// which may have been truncated. If either of the files being compared
+// has a zero value nanosecond value, each byte will be compared for
+// differences. If 2 files have the same seconds value but different
+// nanosecond values where one of those values is zero, the files will
+// be considered unchanged if the content is the same. This behavior
+// is to account for timestamp truncation during archiving.
+func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error {
+	if a == "" {
+		logrus.Debugf("Using single walk diff for %s", b)
+		return addDirChanges(ctx, changeFn, b)
+	} else if diffOptions := detectDirDiff(b, a); diffOptions != nil {
+		logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a)
+		return diffDirChanges(ctx, changeFn, a, diffOptions)
+	}
+
+	logrus.Debugf("Using double walk diff for %s from %s", b, a)
+	return doubleWalkDiff(ctx, changeFn, a, b)
+}
+
+func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error {
+	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(string(os.PathSeparator), path)
+
+		// Skip root
+		if path == string(os.PathSeparator) {
+			return nil
+		}
+
+		return changeFn(ChangeKindAdd, path, f, nil)
+	})
+}
+
+// diffDirOptions is used when the diff can be directly calculated from
+// a diff directory to its base, without walking both trees.
+type diffDirOptions struct {
+	diffDir      string
+	skipChange   func(string) (bool, error)
+	deleteChange func(string, string, os.FileInfo) (string, error)
+}
+
+// diffDirChanges walks the diff directory and compares changes against the base.
+func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error {
+	changedDirs := make(map[string]struct{})
+	return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(o.diffDir, path)
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(string(os.PathSeparator), path)
+
+		// Skip root
+		if path == string(os.PathSeparator) {
+			return nil
+		}
+
+		// TODO: handle opaqueness, start new double walker at this
+		// location to get deletes, and skip tree in single walker
+
+		if o.skipChange != nil {
+			if skip, err := o.skipChange(path); skip {
+				return err
+			}
+		}
+
+		var kind ChangeKind
+
+		deletedFile, err := o.deleteChange(o.diffDir, path, f)
+		if err != nil {
+			return err
+		}
+
+		// Find out what kind of modification happened
+		if deletedFile != "" {
+			path = deletedFile
+			kind = ChangeKindDelete
+			f = nil
+		} else {
+			// Otherwise, the file was added
+			kind = ChangeKindAdd
+
+			// ...Unless it already existed in a base, in which case, it's a modification
+			stat, err := os.Stat(filepath.Join(base, path))
+			if err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			if err == nil {
+				// The file existed in the base, so that's a modification
+
+				// However, if it's a directory, maybe it wasn't actually modified.
+				// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
+				if stat.IsDir() && f.IsDir() {
+					if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
+						// Both directories are the same, don't record the change
+						return nil
+					}
+				}
+				kind = ChangeKindModify
+			}
+		}
+
+		// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
+		// This block is here to ensure the change is recorded even if the
+		// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
+		// Check https://github.com/docker/docker/pull/13590 for details.
+		if f.IsDir() {
+			changedDirs[path] = struct{}{}
+		}
+		if kind == ChangeKindAdd || kind == ChangeKindDelete {
+			parent := filepath.Dir(path)
+			if _, ok := changedDirs[parent]; !ok && parent != "/" {
+				pi, err := os.Stat(filepath.Join(o.diffDir, parent))
+				if err := changeFn(ChangeKindModify, parent, pi, err); err != nil {
+					return err
+				}
+				changedDirs[parent] = struct{}{}
+			}
+		}
+
+		return changeFn(kind, path, f, nil)
+	})
+}
+
+// doubleWalkDiff walks both directories to create a diff
+func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) {
+	g, ctx := errgroup.WithContext(ctx)
+
+	var (
+		c1 = make(chan *currentPath)
+		c2 = make(chan *currentPath)
+
+		f1, f2 *currentPath
+		rmdir  string
+	)
+	g.Go(func() error {
+		defer close(c1)
+		return pathWalk(ctx, a, c1)
+	})
+	g.Go(func() error {
+		defer close(c2)
+		return pathWalk(ctx, b, c2)
+	})
+	g.Go(func() error {
+		for c1 != nil || c2 != nil {
+			if f1 == nil && c1 != nil {
+				f1, err = nextPath(ctx, c1)
+				if err != nil {
+					return err
+				}
+				if f1 == nil {
+					c1 = nil
+				}
+			}
+
+			if f2 == nil && c2 != nil {
+				f2, err = nextPath(ctx, c2)
+				if err != nil {
+					return err
+				}
+				if f2 == nil {
+					c2 = nil
+				}
+			}
+			if f1 == nil && f2 == nil {
+				continue
+			}
+
+			var f os.FileInfo
+			k, p := pathChange(f1, f2)
+			switch k {
+			case ChangeKindAdd:
+				if rmdir != "" {
+					rmdir = ""
+				}
+				f = f2.f
+				f2 = nil
+			case ChangeKindDelete:
+				// Check if this file is already removed by being
+				// under of a removed directory
+				if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
+					f1 = nil
+					continue
+				} else if f1.f.IsDir() {
+					rmdir = f1.path + string(os.PathSeparator)
+				} else if rmdir != "" {
+					rmdir = ""
+				}
+				f1 = nil
+			case ChangeKindModify:
+				same, err := sameFile(f1, f2)
+				if err != nil {
+					return err
+				}
+				if f1.f.IsDir() && !f2.f.IsDir() {
+					rmdir = f1.path + string(os.PathSeparator)
+				} else if rmdir != "" {
+					rmdir = ""
+				}
+				f = f2.f
+				f1 = nil
+				f2 = nil
+				if same {
+					if !isLinked(f) {
+						continue
+					}
+					k = ChangeKindUnmodified
+				}
+			}
+			if err := changeFn(k, p, f, nil); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	return g.Wait()
+}

components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go

@@ -0,0 +1,58 @@
+// +build !windows
+
+package fs
+
+import (
+	"bytes"
+	"os"
+	"syscall"
+
+	"github.com/containerd/continuity/sysx"
+	"github.com/pkg/errors"
+)
+
+// detectDirDiff returns diff dir options if a directory could
+// be found in the mount info for upper which is the direct
+// diff with the provided lower directory
+func detectDirDiff(upper, lower string) *diffDirOptions {
+	// TODO: get mount options for upper
+	// TODO: detect AUFS
+	// TODO: detect overlay
+	return nil
+}
+
+// compareSysStat returns whether the stats are equivalent,
+// whether the files are considered the same file, and
+// an error
+func compareSysStat(s1, s2 interface{}) (bool, error) {
+	ls1, ok := s1.(*syscall.Stat_t)
+	if !ok {
+		return false, nil
+	}
+	ls2, ok := s2.(*syscall.Stat_t)
+	if !ok {
+		return false, nil
+	}
+
+	return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil
+}
+
+func compareCapabilities(p1, p2 string) (bool, error) {
+	c1, err := sysx.LGetxattr(p1, "security.capability")
+	if err != nil && err != sysx.ENODATA {
+		return false, errors.Wrapf(err, "failed to get xattr for %s", p1)
+	}
+	c2, err := sysx.LGetxattr(p2, "security.capability")
+	if err != nil && err != sysx.ENODATA {
+		return false, errors.Wrapf(err, "failed to get xattr for %s", p2)
+	}
+	return bytes.Equal(c1, c2), nil
+}
+
+func isLinked(f os.FileInfo) bool {
+	s, ok := f.Sys().(*syscall.Stat_t)
+	if !ok {
+		return false
+	}
+	return !f.IsDir() && s.Nlink > 1
+}

components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go

@@ -0,0 +1,32 @@
+package fs
+
+import (
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+func detectDirDiff(upper, lower string) *diffDirOptions {
+	return nil
+}
+
+func compareSysStat(s1, s2 interface{}) (bool, error) {
+	f1, ok := s1.(windows.Win32FileAttributeData)
+	if !ok {
+		return false, nil
+	}
+	f2, ok := s2.(windows.Win32FileAttributeData)
+	if !ok {
+		return false, nil
+	}
+	return f1.FileAttributes == f2.FileAttributes, nil
+}
+
+func compareCapabilities(p1, p2 string) (bool, error) {
+	// TODO: Use windows equivalent
+	return true, nil
+}
+
+func isLinked(os.FileInfo) bool {
+	return false
+}

components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go

@@ -0,0 +1,87 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+func locateDummyIfEmpty(path string) (string, error) {
+	children, err := ioutil.ReadDir(path)
+	if err != nil {
+		return "", err
+	}
+	if len(children) != 0 {
+		return "", nil
+	}
+	dummyFile, err := ioutil.TempFile(path, "fsutils-dummy")
+	if err != nil {
+		return "", err
+	}
+	name := dummyFile.Name()
+	err = dummyFile.Close()
+	return name, err
+}
+
+// SupportsDType returns whether the filesystem mounted on path supports d_type
+func SupportsDType(path string) (bool, error) {
+	// locate dummy so that we have at least one dirent
+	dummy, err := locateDummyIfEmpty(path)
+	if err != nil {
+		return false, err
+	}
+	if dummy != "" {
+		defer os.Remove(dummy)
+	}
+
+	visited := 0
+	supportsDType := true
+	fn := func(ent *syscall.Dirent) bool {
+		visited++
+		if ent.Type == syscall.DT_UNKNOWN {
+			supportsDType = false
+			// stop iteration
+			return true
+		}
+		// continue iteration
+		return false
+	}
+	if err = iterateReadDir(path, fn); err != nil {
+		return false, err
+	}
+	if visited == 0 {
+		return false, fmt.Errorf("did not hit any dirent during iteration %s", path)
+	}
+	return supportsDType, nil
+}
+
+func iterateReadDir(path string, fn func(*syscall.Dirent) bool) error {
+	d, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	fd := int(d.Fd())
+	buf := make([]byte, 4096)
+	for {
+		nbytes, err := syscall.ReadDirent(fd, buf)
+		if err != nil {
+			return err
+		}
+		if nbytes == 0 {
+			break
+		}
+		for off := 0; off < nbytes; {
+			ent := (*syscall.Dirent)(unsafe.Pointer(&buf[off]))
+			if stop := fn(ent); stop {
+				return nil
+			}
+			off += int(ent.Reclen)
+		}
+	}
+	return nil
+}

components/engine/vendor/github.com/containerd/continuity/fs/du.go

@@ -0,0 +1,22 @@
+package fs
+
+import "context"
+
+// Usage of disk information
+type Usage struct {
+	Inodes int64
+	Size   int64
+}
+
+// DiskUsage counts the number of inodes and disk usage for the resources under
+// path.
+func DiskUsage(roots ...string) (Usage, error) {
+	return diskUsage(roots...)
+}
+
+// DiffUsage counts the numbers of inodes and disk usage in the
+// diff between the 2 directories. The first path is intended
+// as the base directory and the second as the changed directory.
+func DiffUsage(ctx context.Context, a, b string) (Usage, error) {
+	return diffUsage(ctx, a, b)
+}

components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go

@@ -0,0 +1,88 @@
+// +build !windows
+
+package fs
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+type inode struct {
+	// TODO(stevvooe): Can probably reduce memory usage by not tracking
+	// device, but we can leave this right for now.
+	dev, ino uint64
+}
+
+func newInode(stat *syscall.Stat_t) inode {
+	return inode{
+		// Dev is uint32 on darwin/bsd, uint64 on linux/solaris
+		dev: uint64(stat.Dev), // nolint: unconvert
+		// Ino is uint32 on bsd, uint64 on darwin/linux/solaris
+		ino: uint64(stat.Ino), // nolint: unconvert
+	}
+}
+
+func diskUsage(roots ...string) (Usage, error) {
+
+	var (
+		size   int64
+		inodes = map[inode]struct{}{} // expensive!
+	)
+
+	for _, root := range roots {
+		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			inoKey := newInode(fi.Sys().(*syscall.Stat_t))
+			if _, ok := inodes[inoKey]; !ok {
+				inodes[inoKey] = struct{}{}
+				size += fi.Size()
+			}
+
+			return nil
+		}); err != nil {
+			return Usage{}, err
+		}
+	}
+
+	return Usage{
+		Inodes: int64(len(inodes)),
+		Size:   size,
+	}, nil
+}
+
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
+	var (
+		size   int64
+		inodes = map[inode]struct{}{} // expensive!
+	)
+
+	if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if kind == ChangeKindAdd || kind == ChangeKindModify {
+			inoKey := newInode(fi.Sys().(*syscall.Stat_t))
+			if _, ok := inodes[inoKey]; !ok {
+				inodes[inoKey] = struct{}{}
+				size += fi.Size()
+			}
+
+			return nil
+
+		}
+		return nil
+	}); err != nil {
+		return Usage{}, err
+	}
+
+	return Usage{
+		Inodes: int64(len(inodes)),
+		Size:   size,
+	}, nil
+}

components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go

@@ -0,0 +1,60 @@
+// +build windows
+
+package fs
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+)
+
+func diskUsage(roots ...string) (Usage, error) {
+	var (
+		size int64
+	)
+
+	// TODO(stevvooe): Support inodes (or equivalent) for windows.
+
+	for _, root := range roots {
+		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			size += fi.Size()
+			return nil
+		}); err != nil {
+			return Usage{}, err
+		}
+	}
+
+	return Usage{
+		Size: size,
+	}, nil
+}
+
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
+	var (
+		size int64
+	)
+
+	if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if kind == ChangeKindAdd || kind == ChangeKindModify {
+			size += fi.Size()
+
+			return nil
+
+		}
+		return nil
+	}); err != nil {
+		return Usage{}, err
+	}
+
+	return Usage{
+		Size: size,
+	}, nil
+}

components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go

@@ -0,0 +1,27 @@
+package fs
+
+import "os"
+
+// GetLinkInfo returns an identifier representing the node a hardlink is pointing
+// to. If the file is not hard linked then 0 will be returned.
+func GetLinkInfo(fi os.FileInfo) (uint64, bool) {
+	return getLinkInfo(fi)
+}
+
+// getLinkSource returns a path for the given name and
+// file info to its link source in the provided inode
+// map. If the given file name is not in the map and
+// has other links, it is added to the inode map
+// to be a source for other link locations.
+func getLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) {
+	inode, isHardlink := getLinkInfo(fi)
+	if !isHardlink {
+		return "", nil
+	}
+
+	path, ok := inodes[inode]
+	if !ok {
+		inodes[inode] = name
+	}
+	return path, nil
+}

components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go

@@ -0,0 +1,18 @@
+// +build !windows
+
+package fs
+
+import (
+	"os"
+	"syscall"
+)
+
+func getLinkInfo(fi os.FileInfo) (uint64, bool) {
+	s, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, false
+	}
+
+	// Ino is uint32 on bsd, uint64 on darwin/linux/solaris
+	return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 // nolint: unconvert
+}

components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go

@@ -0,0 +1,7 @@
+package fs
+
+import "os"
+
+func getLinkInfo(fi os.FileInfo) (uint64, bool) {
+	return 0, false
+}

components/engine/vendor/github.com/containerd/continuity/fs/path.go

@@ -0,0 +1,276 @@
+package fs
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	errTooManyLinks = errors.New("too many links")
+)
+
+type currentPath struct {
+	path     string
+	f        os.FileInfo
+	fullPath string
+}
+
+func pathChange(lower, upper *currentPath) (ChangeKind, string) {
+	if lower == nil {
+		if upper == nil {
+			panic("cannot compare nil paths")
+		}
+		return ChangeKindAdd, upper.path
+	}
+	if upper == nil {
+		return ChangeKindDelete, lower.path
+	}
+	// TODO: compare by directory
+
+	switch i := strings.Compare(lower.path, upper.path); {
+	case i < 0:
+		// File in lower that is not in upper
+		return ChangeKindDelete, lower.path
+	case i > 0:
+		// File in upper that is not in lower
+		return ChangeKindAdd, upper.path
+	default:
+		return ChangeKindModify, upper.path
+	}
+}
+
+func sameFile(f1, f2 *currentPath) (bool, error) {
+	if os.SameFile(f1.f, f2.f) {
+		return true, nil
+	}
+
+	equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys())
+	if err != nil || !equalStat {
+		return equalStat, err
+	}
+
+	if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq {
+		return eq, err
+	}
+
+	// If not a directory also check size, modtime, and content
+	if !f1.f.IsDir() {
+		if f1.f.Size() != f2.f.Size() {
+			return false, nil
+		}
+		t1 := f1.f.ModTime()
+		t2 := f2.f.ModTime()
+
+		if t1.Unix() != t2.Unix() {
+			return false, nil
+		}
+
+		// If the timestamp may have been truncated in both of the
+		// files, check content of file to determine difference
+		if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 {
+			var eq bool
+			if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink {
+				eq, err = compareSymlinkTarget(f1.fullPath, f2.fullPath)
+			} else if f1.f.Size() > 0 {
+				eq, err = compareFileContent(f1.fullPath, f2.fullPath)
+			}
+			if err != nil || !eq {
+				return eq, err
+			}
+		} else if t1.Nanosecond() != t2.Nanosecond() {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func compareSymlinkTarget(p1, p2 string) (bool, error) {
+	t1, err := os.Readlink(p1)
+	if err != nil {
+		return false, err
+	}
+	t2, err := os.Readlink(p2)
+	if err != nil {
+		return false, err
+	}
+	return t1 == t2, nil
+}
+
+const compareChuckSize = 32 * 1024
+
+// compareFileContent compares the content of 2 same sized files
+// by comparing each byte.
+func compareFileContent(p1, p2 string) (bool, error) {
+	f1, err := os.Open(p1)
+	if err != nil {
+		return false, err
+	}
+	defer f1.Close()
+	f2, err := os.Open(p2)
+	if err != nil {
+		return false, err
+	}
+	defer f2.Close()
+
+	b1 := make([]byte, compareChuckSize)
+	b2 := make([]byte, compareChuckSize)
+	for {
+		n1, err1 := f1.Read(b1)
+		if err1 != nil && err1 != io.EOF {
+			return false, err1
+		}
+		n2, err2 := f2.Read(b2)
+		if err2 != nil && err2 != io.EOF {
+			return false, err2
+		}
+		if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, nil
+		}
+		if err1 == io.EOF && err2 == io.EOF {
+			return true, nil
+		}
+	}
+}
+
+func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error {
+	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(string(os.PathSeparator), path)
+
+		// Skip root
+		if path == string(os.PathSeparator) {
+			return nil
+		}
+
+		p := &currentPath{
+			path:     path,
+			f:        f,
+			fullPath: filepath.Join(root, path),
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case pathC <- p:
+			return nil
+		}
+	})
+}
+
+func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case p := <-pathC:
+		return p, nil
+	}
+}
+
+// RootPath joins a path with a root, evaluating and bounding any
+// symlink to the root directory.
+func RootPath(root, path string) (string, error) {
+	if path == "" {
+		return root, nil
+	}
+	var linksWalked int // to protect against cycles
+	for {
+		i := linksWalked
+		newpath, err := walkLinks(root, path, &linksWalked)
+		if err != nil {
+			return "", err
+		}
+		path = newpath
+		if i == linksWalked {
+			newpath = filepath.Join("/", newpath)
+			if path == newpath {
+				return filepath.Join(root, newpath), nil
+			}
+			path = newpath
+		}
+	}
+}
+
+func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) {
+	if *linksWalked > 255 {
+		return "", false, errTooManyLinks
+	}
+
+	path = filepath.Join("/", path)
+	if path == "/" {
+		return path, false, nil
+	}
+	realPath := filepath.Join(root, path)
+
+	fi, err := os.Lstat(realPath)
+	if err != nil {
+		// If path does not yet exist, treat as non-symlink
+		if os.IsNotExist(err) {
+			return path, false, nil
+		}
+		return "", false, err
+	}
+	if fi.Mode()&os.ModeSymlink == 0 {
+		return path, false, nil
+	}
+	newpath, err = os.Readlink(realPath)
+	if err != nil {
+		return "", false, err
+	}
+	if filepath.IsAbs(newpath) && strings.HasPrefix(newpath, root) {
+		newpath = newpath[:len(root)]
+		if !strings.HasPrefix(newpath, "/") {
+			newpath = "/" + newpath
+		}
+	}
+	*linksWalked++
+	return newpath, true, nil
+}
+
+func walkLinks(root, path string, linksWalked *int) (string, error) {
+	switch dir, file := filepath.Split(path); {
+	case dir == "":
+		newpath, _, err := walkLink(root, file, linksWalked)
+		return newpath, err
+	case file == "":
+		if os.IsPathSeparator(dir[len(dir)-1]) {
+			if dir == "/" {
+				return dir, nil
+			}
+			return walkLinks(root, dir[:len(dir)-1], linksWalked)
+		}
+		newpath, _, err := walkLink(root, dir, linksWalked)
+		return newpath, err
+	default:
+		newdir, err := walkLinks(root, dir, linksWalked)
+		if err != nil {
+			return "", err
+		}
+		newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked)
+		if err != nil {
+			return "", err
+		}
+		if !islink {
+			return newpath, nil
+		}
+		if filepath.IsAbs(newpath) {
+			return newpath, nil
+		}
+		return filepath.Join(newdir, newpath), nil
+	}
+}

components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go

@@ -0,0 +1,28 @@
+// +build darwin freebsd
+
+package fs
+
+import (
+	"syscall"
+	"time"
+)
+
+// StatAtime returns the access time from a stat struct
+func StatAtime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Atimespec
+}
+
+// StatCtime returns the created time from a stat struct
+func StatCtime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Ctimespec
+}
+
+// StatMtime returns the modified time from a stat struct
+func StatMtime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Mtimespec
+}
+
+// StatATimeAsTime returns the access time as a time.Time
+func StatATimeAsTime(st *syscall.Stat_t) time.Time {
+	return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) // nolint: unconvert
+}

components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go

@@ -0,0 +1,26 @@
+package fs
+
+import (
+	"syscall"
+	"time"
+)
+
+// StatAtime returns the Atim
+func StatAtime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Atim
+}
+
+// StatCtime returns the Ctim
+func StatCtime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Ctim
+}
+
+// StatMtime returns the Mtim
+func StatMtime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Mtim
+}
+
+// StatATimeAsTime returns st.Atim as a time.Time
+func StatATimeAsTime(st *syscall.Stat_t) time.Time {
+	return time.Unix(st.Atim.Sec, st.Atim.Nsec)
+}

components/engine/vendor/github.com/containerd/continuity/fs/time.go

@@ -0,0 +1,13 @@
+package fs
+
+import "time"
+
+// Gnu tar and the go tar writer don't have sub-second mtime
+// precision, which is problematic when we apply changes via tar
+// files, we handle this by comparing for exact times, *or* same
+// second count and either a or b having exactly 0 nanoseconds
+func sameFsTime(a, b time.Time) bool {
+	return a == b ||
+		(a.Unix() == b.Unix() &&
+			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
+}

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go

@@ -1,11 +0,0 @@
-package sysx
-
-// These functions will be generated by generate.sh
-//    $ GOOS=linux GOARCH=386 ./generate.sh copy
-//    $ GOOS=linux GOARCH=amd64 ./generate.sh copy
-//    $ GOOS=linux GOARCH=arm ./generate.sh copy
-//    $ GOOS=linux GOARCH=arm64 ./generate.sh copy
-//    $ GOOS=linux GOARCH=ppc64le ./generate.sh copy
-//    $ GOOS=linux GOARCH=s390x ./generate.sh copy
-
-//sys CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error)

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go

@@ -1,20 +0,0 @@
-// mksyscall.pl -l32 copy_linux.go
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
-
-package sysx
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go

@@ -1,20 +0,0 @@
-// mksyscall.pl copy_linux.go
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
-
-package sysx
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go

@@ -1,20 +0,0 @@
-// mksyscall.pl -l32 copy_linux.go
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
-
-package sysx
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go

@@ -1,20 +0,0 @@
-// mksyscall.pl copy_linux.go
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
-
-package sysx
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go

@@ -1,20 +0,0 @@
-// mksyscall.pl copy_linux.go
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
-
-package sysx
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}

components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go

@@ -1,20 +0,0 @@
-// mksyscall.pl copy_linux.go
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
-
-package sysx
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
-
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
-	n = int(r0)
-	if e1 != 0 {
-		err = errnoErr(e1)
-	}
-	return
-}

components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go

@@ -1,7 +0,0 @@
-package sysx
-
-const (
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
-	// Number defined in /usr/include/asm/unistd_32.h
-	SYS_COPY_FILE_RANGE = 377
-)

components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go

@@ -1,7 +0,0 @@
-package sysx
-
-const (
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
-	// Number defined in /usr/include/asm/unistd_64.h
-	SYS_COPY_FILE_RANGE = 326
-)

components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go

@@ -1,7 +0,0 @@
-package sysx
-
-const (
-	// SYS_COPY_FILE_RANGE defined in Kernel 4.5+
-	// Number defined in /usr/include/arm-linux-gnueabihf/asm/unistd.h
-	SYS_COPY_FILE_RANGE = 391
-)

components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go

@@ -1,7 +0,0 @@
-package sysx
-
-const (
-	// SYS_COPY_FILE_RANGE defined in Kernel 4.5+
-	// Number defined in /usr/include/asm-generic/unistd.h
-	SYS_COPY_FILE_RANGE = 285
-)

components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go

@@ -1,7 +0,0 @@
-package sysx
-
-const (
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
-	// Number defined in /usr/include/asm/unistd_64.h
-	SYS_COPY_FILE_RANGE = 379
-)

components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go

@@ -1,7 +0,0 @@
-package sysx
-
-const (
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
-	// Number defined in /usr/include/asm/unistd_64.h
-	SYS_COPY_FILE_RANGE = 375
-)

components/engine/vendor/github.com/containerd/continuity/vendor.conf

@@ -0,0 +1,13 @@
+bazil.org/fuse 371fbbdaa8987b715bdd21d6adc4c9b20155f748
+github.com/dustin/go-humanize bb3d318650d48840a39aa21a027c6630e198e626
+github.com/golang/protobuf 1e59b77b52bf8e4b449a57e6f79f21226d571845
+github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
+github.com/opencontainers/go-digest 279bed98673dd5bef374d3b6e4b09e2af76183bf
+github.com/pkg/errors f15c970de5b76fac0b59abb32d62c17cc7bed265
+github.com/sirupsen/logrus 89742aefa4b206dcf400792f3bd35b542998eb3b
+github.com/spf13/cobra 2da4a54c5ceefcee7ca5dd0eea1e18a3b6366489
+github.com/spf13/pflag 4c012f6dcd9546820e378d0bdda4d8fc772cdfea
+golang.org/x/crypto 9f005a07e0d31d45e6656d241bb5c0f2efd4bc94
+golang.org/x/net a337091b0525af65de94df2eb7e98bd9962dcbe2
+golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
+golang.org/x/sys 665f6529cca930e27b831a0d1dafffbe1c172924

components/engine/vendor/github.com/containerd/go-runc/runc.go

@@ -1,7 +1,6 @@
 package runc
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -532,7 +531,9 @@ func (r *Runc) Restore(context context.Context, id, bundle string, opts *Restore
 
 // Update updates the current container with the provided resource spec
 func (r *Runc) Update(context context.Context, id string, resources *specs.LinuxResources) error {
-	buf := bytes.NewBuffer(nil)
+	buf := getBuf()
+	defer putBuf(buf)
+
 	if err := json.NewEncoder(buf).Encode(resources); err != nil {
 		return err
 	}
@@ -638,11 +639,12 @@ func (r *Runc) runOrError(cmd *exec.Cmd) error {
 }
 
 func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) {
-	var b bytes.Buffer
+	b := getBuf()
+	defer putBuf(b)
 
-	cmd.Stdout = &b
+	cmd.Stdout = b
 	if combined {
-		cmd.Stderr = &b
+		cmd.Stderr = b
 	}
 	ec, err := Monitor.Start(cmd)
 	if err != nil {

components/engine/vendor/github.com/containerd/go-runc/utils.go

@@ -1,8 +1,10 @@
 package runc
 
 import (
+	"bytes"
 	"io/ioutil"
 	"strconv"
+	"sync"
 	"syscall"
 )
 
@@ -26,3 +28,18 @@ func exitStatus(status syscall.WaitStatus) int {
 	}
 	return status.ExitStatus()
 }
+
+var bytesBufferPool = sync.Pool{
+	New: func() interface{} {
+		return bytes.NewBuffer(nil)
+	},
+}
+
+func getBuf() *bytes.Buffer {
+	return bytesBufferPool.Get().(*bytes.Buffer)
+}
+
+func putBuf(b *bytes.Buffer) {
+	b.Reset()
+	bytesBufferPool.Put(b)
+}

components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c

@@ -22,7 +22,6 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 
-
 #include <linux/limits.h>
 #include <linux/netlink.h>
 #include <linux/types.h>
@@ -32,15 +31,15 @@
 
 /* Synchronisation values. */
 enum sync_t {
-	SYNC_USERMAP_PLS = 0x40, /* Request parent to map our users. */
-	SYNC_USERMAP_ACK = 0x41, /* Mapping finished by the parent. */
-	SYNC_RECVPID_PLS = 0x42, /* Tell parent we're sending the PID. */
-	SYNC_RECVPID_ACK = 0x43, /* PID was correctly received by parent. */
-	SYNC_GRANDCHILD  = 0x44, /* The grandchild is ready to run. */
-	SYNC_CHILD_READY = 0x45, /* The child or grandchild is ready to return. */
+	SYNC_USERMAP_PLS = 0x40,	/* Request parent to map our users. */
+	SYNC_USERMAP_ACK = 0x41,	/* Mapping finished by the parent. */
+	SYNC_RECVPID_PLS = 0x42,	/* Tell parent we're sending the PID. */
+	SYNC_RECVPID_ACK = 0x43,	/* PID was correctly received by parent. */
+	SYNC_GRANDCHILD = 0x44,	/* The grandchild is ready to run. */
+	SYNC_CHILD_READY = 0x45,	/* The child or grandchild is ready to return. */
 
 	/* XXX: This doesn't help with segfaults and other such issues. */
-	SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */
+	SYNC_ERR = 0xFF,	/* Fatal error, no turning back. The error code follows. */
 };
 
 /* longjmp() arguments. */
@@ -73,7 +72,7 @@ struct nlconfig_t {
 	char *oom_score_adj;
 	size_t oom_score_adj_len;
 
-	/* User namespace settings.*/
+	/* User namespace settings. */
 	char *uidmap;
 	size_t uidmap_len;
 	char *gidmap;
@@ -82,7 +81,7 @@ struct nlconfig_t {
 	size_t namespaces_len;
 	uint8_t is_setgroup;
 
-	/* Rootless container settings.*/
+	/* Rootless container settings. */
 	uint8_t is_rootless;
 	char *uidmappath;
 	size_t uidmappath_len;
@@ -167,7 +166,7 @@ static int write_file(char *data, size_t data_len, char *pathfmt, ...)
 		goto out;
 	}
 
-out:
+ out:
 	close(fd);
 	return ret;
 }
@@ -184,16 +183,16 @@ static void update_setgroups(int pid, enum policy_t setgroup)
 	char *policy;
 
 	switch (setgroup) {
-		case SETGROUPS_ALLOW:
-			policy = "allow";
-			break;
-		case SETGROUPS_DENY:
-			policy = "deny";
-			break;
-		case SETGROUPS_DEFAULT:
-		default:
-			/* Nothing to do. */
-			return;
+	case SETGROUPS_ALLOW:
+		policy = "allow";
+		break;
+	case SETGROUPS_DENY:
+		policy = "deny";
+		break;
+	case SETGROUPS_DEFAULT:
+	default:
+		/* Nothing to do. */
+		return;
 	}
 
 	if (write_file(policy, strlen(policy), "/proc/%d/setgroups", pid) < 0) {
@@ -226,14 +225,14 @@ static int try_mapping_tool(const char *app, int pid, char *map, size_t map_len)
 	if (!child) {
 #define MAX_ARGV 20
 		char *argv[MAX_ARGV];
-		char *envp[] = {NULL};
+		char *envp[] = { NULL };
 		char pid_fmt[16];
 		int argc = 0;
 		char *next;
 
 		snprintf(pid_fmt, 16, "%d", pid);
 
-		argv[argc++] = (char *) app;
+		argv[argc++] = (char *)app;
 		argv[argc++] = pid_fmt;
 		/*
 		 * Convert the map string into a list of argument that
@@ -319,7 +318,7 @@ static int clone_parent(jmp_buf *env, int jmpval) __attribute__ ((noinline));
 static int clone_parent(jmp_buf *env, int jmpval)
 {
 	struct clone_t ca = {
-		.env    = env,
+		.env = env,
 		.jmpval = jmpval,
 	};
 
@@ -533,7 +532,7 @@ void nsexec(void)
 	int pipenum;
 	jmp_buf env;
 	int sync_child_pipe[2], sync_grandchild_pipe[2];
-	struct nlconfig_t config = {0};
+	struct nlconfig_t config = { 0 };
 
 	/*
 	 * If we don't have an init pipe, just return to the go routine.
@@ -630,21 +629,21 @@ void nsexec(void)
 	 */
 
 	switch (setjmp(env)) {
-	/*
-	 * Stage 0: We're in the parent. Our job is just to create a new child
-	 *          (stage 1: JUMP_CHILD) process and write its uid_map and
-	 *          gid_map. That process will go on to create a new process, then
-	 *          it will send us its PID which we will send to the bootstrap
-	 *          process.
-	 */
-	case JUMP_PARENT: {
+		/*
+		 * Stage 0: We're in the parent. Our job is just to create a new child
+		 *          (stage 1: JUMP_CHILD) process and write its uid_map and
+		 *          gid_map. That process will go on to create a new process, then
+		 *          it will send us its PID which we will send to the bootstrap
+		 *          process.
+		 */
+	case JUMP_PARENT:{
 			int len;
 			pid_t child, first_child = -1;
 			char buf[JSON_MAX];
 			bool ready = false;
 
 			/* For debugging. */
-			prctl(PR_SET_NAME, (unsigned long) "runc:[0:PARENT]", 0, 0, 0);
+			prctl(PR_SET_NAME, (unsigned long)"runc:[0:PARENT]", 0, 0, 0);
 
 			/* Start the process of getting a container. */
 			child = clone_parent(&env, JUMP_CHILD);
@@ -702,7 +701,7 @@ void nsexec(void)
 						bail("failed to sync with child: write(SYNC_USERMAP_ACK)");
 					}
 					break;
-				case SYNC_RECVPID_PLS: {
+				case SYNC_RECVPID_PLS:{
 						first_child = child;
 
 						/* Get the init_func pid. */
@@ -781,16 +780,16 @@ void nsexec(void)
 			exit(0);
 		}
 
-	/*
-	 * Stage 1: We're in the first child process. Our job is to join any
-	 *          provided namespaces in the netlink payload and unshare all
-	 *          of the requested namespaces. If we've been asked to
-	 *          CLONE_NEWUSER, we will ask our parent (stage 0) to set up
-	 *          our user mappings for us. Then, we create a new child
-	 *          (stage 2: JUMP_INIT) for PID namespace. We then send the
-	 *          child's PID to our parent (stage 0).
-	 */
-	case JUMP_CHILD: {
+		/*
+		 * Stage 1: We're in the first child process. Our job is to join any
+		 *          provided namespaces in the netlink payload and unshare all
+		 *          of the requested namespaces. If we've been asked to
+		 *          CLONE_NEWUSER, we will ask our parent (stage 0) to set up
+		 *          our user mappings for us. Then, we create a new child
+		 *          (stage 2: JUMP_INIT) for PID namespace. We then send the
+		 *          child's PID to our parent (stage 0).
+		 */
+	case JUMP_CHILD:{
 			pid_t child;
 			enum sync_t s;
 
@@ -799,7 +798,7 @@ void nsexec(void)
 			close(sync_child_pipe[1]);
 
 			/* For debugging. */
-			prctl(PR_SET_NAME, (unsigned long) "runc:[1:CHILD]", 0, 0, 0);
+			prctl(PR_SET_NAME, (unsigned long)"runc:[1:CHILD]", 0, 0, 0);
 
 			/*
 			 * We need to setns first. We cannot do this earlier (in stage 0)
@@ -901,13 +900,13 @@ void nsexec(void)
 			exit(0);
 		}
 
-	/*
-	 * Stage 2: We're the final child process, and the only process that will
-	 *          actually return to the Go runtime. Our job is to just do the
-	 *          final cleanup steps and then return to the Go runtime to allow
-	 *          init_linux.go to run.
-	 */
-	case JUMP_INIT: {
+		/*
+		 * Stage 2: We're the final child process, and the only process that will
+		 *          actually return to the Go runtime. Our job is to just do the
+		 *          final cleanup steps and then return to the Go runtime to allow
+		 *          init_linux.go to run.
+		 */
+	case JUMP_INIT:{
 			/*
 			 * We're inside the child now, having jumped from the
 			 * start_child() code after forking in the parent.
@@ -921,7 +920,7 @@ void nsexec(void)
 			close(sync_child_pipe[1]);
 
 			/* For debugging. */
-			prctl(PR_SET_NAME, (unsigned long) "runc:[2:INIT]", 0, 0, 0);
+			prctl(PR_SET_NAME, (unsigned long)"runc:[2:INIT]", 0, 0, 0);
 
 			if (read(syncfd, &s, sizeof(s)) != sizeof(s))
 				bail("failed to sync with parent: read(SYNC_GRANDCHILD)");

components/engine/vendor/github.com/stevvooe/ttrpc/channel.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/binary"
 	"io"
+	"net"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -60,16 +61,18 @@ func writeMessageHeader(w io.Writer, p []byte, mh messageHeader) error {
 var buffers sync.Pool
 
 type channel struct {
+	conn  net.Conn
 	bw    *bufio.Writer
 	br    *bufio.Reader
 	hrbuf [messageHeaderLength]byte // avoid alloc when reading header
 	hwbuf [messageHeaderLength]byte
 }
 
-func newChannel(w io.Writer, r io.Reader) *channel {
+func newChannel(conn net.Conn) *channel {
 	return &channel{
-		bw: bufio.NewWriter(w),
-		br: bufio.NewReader(r),
+		conn: conn,
+		bw:   bufio.NewWriter(conn),
+		br:   bufio.NewReader(conn),
 	}
 }
 

components/engine/vendor/github.com/stevvooe/ttrpc/client.go

@@ -2,8 +2,12 @@ package ttrpc
 
 import (
 	"context"
+	"io"
 	"net"
+	"os"
+	"strings"
 	"sync"
+	"syscall"
 
 	"github.com/containerd/containerd/log"
 	"github.com/gogo/protobuf/proto"
@@ -11,6 +15,10 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+// ErrClosed is returned by client methods when the underlying connection is
+// closed.
+var ErrClosed = errors.New("ttrpc: closed")
+
 type Client struct {
 	codec   codec
 	conn    net.Conn
@@ -19,18 +27,20 @@ type Client struct {
 
 	closed    chan struct{}
 	closeOnce sync.Once
+	closeFunc func()
 	done      chan struct{}
 	err       error
 }
 
 func NewClient(conn net.Conn) *Client {
 	c := &Client{
-		codec:   codec{},
-		conn:    conn,
-		channel: newChannel(conn, conn),
-		calls:   make(chan *callRequest),
-		closed:  make(chan struct{}),
-		done:    make(chan struct{}),
+		codec:     codec{},
+		conn:      conn,
+		channel:   newChannel(conn),
+		calls:     make(chan *callRequest),
+		closed:    make(chan struct{}),
+		done:      make(chan struct{}),
+		closeFunc: func() {},
 	}
 
 	go c.run()
@@ -91,7 +101,7 @@ func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) err
 
 	select {
 	case err := <-errs:
-		return err
+		return filterCloseErr(err)
 	case <-c.done:
 		return c.err
 	}
@@ -105,6 +115,11 @@ func (c *Client) Close() error {
 	return nil
 }
 
+// OnClose allows a close func to be called when the server is closed
+func (c *Client) OnClose(closer func()) {
+	c.closeFunc = closer
+}
+
 type message struct {
 	messageHeader
 	p   []byte
@@ -150,6 +165,7 @@ func (c *Client) run() {
 
 	defer c.conn.Close()
 	defer close(c.done)
+	defer c.closeFunc()
 
 	for {
 		select {
@@ -171,7 +187,14 @@ func (c *Client) run() {
 			call.errs <- c.recv(call.resp, msg)
 			delete(waiters, msg.StreamID)
 		case <-shutdown:
+			if shutdownErr != nil {
+				shutdownErr = filterCloseErr(shutdownErr)
+			} else {
+				shutdownErr = ErrClosed
+			}
+
 			shutdownErr = errors.Wrapf(shutdownErr, "ttrpc: client shutting down")
+
 			c.err = shutdownErr
 			for _, waiter := range waiters {
 				waiter.errs <- shutdownErr
@@ -179,9 +202,12 @@ func (c *Client) run() {
 			c.Close()
 			return
 		case <-c.closed:
+			if c.err == nil {
+				c.err = ErrClosed
+			}
 			// broadcast the shutdown error to the remaining waiters.
 			for _, waiter := range waiters {
-				waiter.errs <- shutdownErr
+				waiter.errs <- c.err
 			}
 			return
 		}
@@ -209,3 +235,30 @@ func (c *Client) recv(resp *Response, msg *message) error {
 	defer c.channel.putmbuf(msg.p)
 	return proto.Unmarshal(msg.p, resp)
 }
+
+// filterCloseErr rewrites EOF and EPIPE errors to ErrClosed. Use when
+// returning from call or handling errors from main read loop.
+//
+// This purposely ignores errors with a wrapped cause.
+func filterCloseErr(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	if err == io.EOF {
+		return ErrClosed
+	}
+
+	if strings.Contains(err.Error(), "use of closed network connection") {
+		return ErrClosed
+	}
+
+	// if we have an epipe on a write, we cast to errclosed
+	if oerr, ok := err.(*net.OpError); ok && oerr.Op == "write" {
+		if serr, ok := oerr.Err.(*os.SyscallError); ok && serr.Err == syscall.EPIPE {
+			return ErrClosed
+		}
+	}
+
+	return err
+}

components/engine/vendor/github.com/stevvooe/ttrpc/server.go

@@ -16,7 +16,7 @@ import (
 )
 
 var (
-	ErrServerClosed = errors.New("ttrpc: server close")
+	ErrServerClosed = errors.New("ttrpc: server closed")
 )
 
 type Server struct {
@@ -281,7 +281,7 @@ func (c *serverConn) run(sctx context.Context) {
 	)
 
 	var (
-		ch          = newChannel(c.conn, c.conn)
+		ch          = newChannel(c.conn)
 		ctx, cancel = context.WithCancel(sctx)
 		active      int
 		state       connState = connStateIdle