Merge component 'engine' from git@github.com:moby/moby master

components/engine/Dockerfile

@@ -187,11 +187,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
-# Install tomlv, vndr, runc, containerd, tini, docker-proxy dockercli
-# Please edit hack/dockerfile/install-binaries.sh to update them.
-COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
-COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
-RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter
+# Install tomlv, vndr, runc, containerd, tini, proxy dockercli
+# Please edit hack/dockerfile/install/<name>.installer to update them.
+COPY hack/dockerfile/install hack/dockerfile/install
+RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \
+		do hack/dockerfile/install/install.sh $i; \
+	done
 ENV PATH=/usr/local/cli:$PATH
 
 # Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH

components/engine/Dockerfile.aarch64

@@ -151,14 +151,17 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
+#
 
-# Install tomlv, vndr, runc, containerd, tini, docker-proxy
-# Please edit hack/dockerfile/install-binaries.sh to update them.
-COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
-COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
-RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter
+# Install tomlv, vndr, runc, containerd, tini, proxy dockercli
+# Please edit hack/dockerfile/install/<name>.installer to update them.
+COPY hack/dockerfile/install hack/dockerfile/install
+RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \
+		do hack/dockerfile/install/install.sh $i; \
+	done
 ENV PATH=/usr/local/cli:$PATH
 
+
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 ENTRYPOINT ["hack/dind"]
 

components/engine/Dockerfile.armhf

@@ -138,11 +138,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
-# Install tomlv, vndr, runc, containerd, tini, docker-proxy
-# Please edit hack/dockerfile/install-binaries.sh to update them.
-COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
-COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
-RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter
+# Install tomlv, vndr, runc, containerd, tini, proxy dockercli
+# Please edit hack/dockerfile/install/<name>.installer to update them.
+COPY hack/dockerfile/install hack/dockerfile/install
+RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \
+		do hack/dockerfile/install/install.sh $i; \
+	done
 ENV PATH=/usr/local/cli:$PATH
 
 ENTRYPOINT ["hack/dind"]

components/engine/Dockerfile.e2e

@@ -23,7 +23,7 @@ RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \
 
 # Download Docker CLI binary
 COPY hack/dockerfile hack/dockerfile
-RUN hack/dockerfile/install-binaries.sh dockercli
+RUN hack/dockerfile/install.sh dockercli
 
 # Set tag and add sources
 ARG DOCKER_GITCOMMIT

components/engine/Dockerfile.ppc64le

@@ -136,11 +136,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
-# Install tomlv, vndr, runc, containerd, tini, docker-proxy
-# Please edit hack/dockerfile/install-binaries.sh to update them.
-COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
-COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
-RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter
+# Install tomlv, vndr, runc, containerd, tini, proxy dockercli
+# Please edit hack/dockerfile/install/<name>.installer to update them.
+COPY hack/dockerfile/install hack/dockerfile/install
+RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \
+		do hack/dockerfile/install/install.sh $i; \
+	done
 ENV PATH=/usr/local/cli:$PATH
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers

components/engine/Dockerfile.s390x

@@ -130,11 +130,12 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
 # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
-# Install tomlv, vndr, runc, containerd, tini, docker-proxy
-# Please edit hack/dockerfile/install-binaries.sh to update them.
-COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
-COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
-RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli gometalinter
+# Install tomlv, vndr, runc, containerd, tini, proxy dockercli
+# Please edit hack/dockerfile/install/<name>.installer to update them.
+COPY hack/dockerfile/install hack/dockerfile/install
+RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \
+		do hack/dockerfile/install/install.sh $i; \
+	done
 ENV PATH=/usr/local/cli:$PATH
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers

components/engine/Dockerfile.simple

@@ -50,10 +50,11 @@ ENV GOPATH /go
 ENV CGO_LDFLAGS -L/lib
 
 # Install runc, containerd, tini and docker-proxy
-# Please edit hack/dockerfile/install-binaries.sh to update them.
-COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
-COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
-RUN /tmp/install-binaries.sh runc containerd tini proxy dockercli
+# Please edit hack/dockerfile/install/<name>.installer to update them.
+COPY hack/dockerfile/install hack/dockerfile/install
+RUN for i in runc containerd tini proxy dockercli; \
+		do hack/dockerfile/install/install.sh $i; \
+	done
 ENV PATH=/usr/local/cli:$PATH
 
 ENV AUTO_GOPATH 1

components/engine/daemon/commit.go

@@ -1,9 +1,7 @@
 package daemon // import "github.com/docker/docker/daemon"
 
 import (
-	"encoding/json"
 	"fmt"
-	"io"
 	"runtime"
 	"strings"
 	"time"
@@ -12,10 +10,6 @@ import (
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder/dockerfile"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/layer"
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
 )
 
@@ -190,115 +184,3 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma
 	containerActions.WithValues("commit").UpdateSince(start)
 	return id.String(), nil
 }
-
-func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
-	layerStore, ok := daemon.layerStores[c.ContainerOS]
-	if !ok {
-		return "", system.ErrNotSupportedOperatingSystem
-	}
-	rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		if rwTar != nil {
-			rwTar.Close()
-		}
-	}()
-
-	var parent *image.Image
-	if c.ParentImageID == "" {
-		parent = new(image.Image)
-		parent.RootFS = image.NewRootFS()
-	} else {
-		parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID))
-		if err != nil {
-			return "", err
-		}
-	}
-
-	l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
-	if err != nil {
-		return "", err
-	}
-	defer layer.ReleaseAndLog(layerStore, l)
-
-	cc := image.ChildConfig{
-		ContainerID:     c.ContainerID,
-		Author:          c.Author,
-		Comment:         c.Comment,
-		ContainerConfig: c.ContainerConfig,
-		Config:          c.Config,
-		DiffID:          l.DiffID(),
-	}
-	config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS))
-	if err != nil {
-		return "", err
-	}
-
-	id, err := daemon.imageStore.Create(config)
-	if err != nil {
-		return "", err
-	}
-
-	if c.ParentImageID != "" {
-		if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil {
-			return "", err
-		}
-	}
-	return id, nil
-}
-
-func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
-	rwlayer, err := layerStore.GetRWLayer(id)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if err != nil {
-			layerStore.ReleaseRWLayer(rwlayer)
-		}
-	}()
-
-	// TODO: this mount call is not necessary as we assume that TarStream() should
-	// mount the layer if needed. But the Diff() function for windows requests that
-	// the layer should be mounted when calling it. So we reserve this mount call
-	// until windows driver can implement Diff() interface correctly.
-	_, err = rwlayer.Mount(mountLabel)
-	if err != nil {
-		return nil, err
-	}
-
-	archive, err := rwlayer.TarStream()
-	if err != nil {
-		rwlayer.Unmount()
-		return nil, err
-	}
-	return ioutils.NewReadCloserWrapper(archive, func() error {
-			archive.Close()
-			err = rwlayer.Unmount()
-			layerStore.ReleaseRWLayer(rwlayer)
-			return err
-		}),
-		nil
-}
-
-// CommitBuildStep is used by the builder to create an image for each step in
-// the build.
-//
-// This method is different from CreateImageFromContainer:
-//   * it doesn't attempt to validate container state
-//   * it doesn't send a commit action to metrics
-//   * it doesn't log a container commit event
-//
-// This is a temporary shim. Should be removed when builder stops using commit.
-func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
-	container, err := daemon.GetContainer(c.ContainerID)
-	if err != nil {
-		return "", err
-	}
-	c.ContainerMountLabel = container.MountLabel
-	c.ContainerOS = container.OS
-	c.ParentImageID = string(container.ImageID)
-	return daemon.commitImage(c)
-}

components/engine/daemon/events.go

@@ -44,30 +44,6 @@ func (daemon *Daemon) LogContainerEventWithAttributes(container *container.Conta
 	daemon.EventsService.Log(action, events.ContainerEventType, actor)
 }
 
-// LogImageEvent generates an event related to an image with only the default attributes.
-func (daemon *Daemon) LogImageEvent(imageID, refName, action string) {
-	daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{})
-}
-
-// LogImageEventWithAttributes generates an event related to an image with specific given attributes.
-func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) {
-	img, err := daemon.GetImage(imageID)
-	if err == nil && img.Config != nil {
-		// image has not been removed yet.
-		// it could be missing if the event is `delete`.
-		copyAttributes(attributes, img.Config.Labels)
-	}
-	if refName != "" {
-		attributes["name"] = refName
-	}
-	actor := events.Actor{
-		ID:         imageID,
-		Attributes: attributes,
-	}
-
-	daemon.EventsService.Log(action, events.ImageEventType, actor)
-}
-
 // LogPluginEvent generates an event related to a plugin with only the default attributes.
 func (daemon *Daemon) LogPluginEvent(pluginID, refName, action string) {
 	daemon.LogPluginEventWithAttributes(pluginID, refName, action, map[string]string{})

components/engine/daemon/image_commit.go

@@ -0,0 +1,124 @@
+package daemon // import "github.com/docker/docker/daemon"
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/docker/docker/api/types/backend"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/system"
+)
+
+func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
+	layerStore, ok := daemon.layerStores[c.ContainerOS]
+	if !ok {
+		return "", system.ErrNotSupportedOperatingSystem
+	}
+	rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		if rwTar != nil {
+			rwTar.Close()
+		}
+	}()
+
+	var parent *image.Image
+	if c.ParentImageID == "" {
+		parent = new(image.Image)
+		parent.RootFS = image.NewRootFS()
+	} else {
+		parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID))
+		if err != nil {
+			return "", err
+		}
+	}
+
+	l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
+	if err != nil {
+		return "", err
+	}
+	defer layer.ReleaseAndLog(layerStore, l)
+
+	cc := image.ChildConfig{
+		ContainerID:     c.ContainerID,
+		Author:          c.Author,
+		Comment:         c.Comment,
+		ContainerConfig: c.ContainerConfig,
+		Config:          c.Config,
+		DiffID:          l.DiffID(),
+	}
+	config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS))
+	if err != nil {
+		return "", err
+	}
+
+	id, err := daemon.imageStore.Create(config)
+	if err != nil {
+		return "", err
+	}
+
+	if c.ParentImageID != "" {
+		if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil {
+			return "", err
+		}
+	}
+	return id, nil
+}
+
+func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
+	rwlayer, err := layerStore.GetRWLayer(id)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			layerStore.ReleaseRWLayer(rwlayer)
+		}
+	}()
+
+	// TODO: this mount call is not necessary as we assume that TarStream() should
+	// mount the layer if needed. But the Diff() function for windows requests that
+	// the layer should be mounted when calling it. So we reserve this mount call
+	// until windows driver can implement Diff() interface correctly.
+	_, err = rwlayer.Mount(mountLabel)
+	if err != nil {
+		return nil, err
+	}
+
+	archive, err := rwlayer.TarStream()
+	if err != nil {
+		rwlayer.Unmount()
+		return nil, err
+	}
+	return ioutils.NewReadCloserWrapper(archive, func() error {
+			archive.Close()
+			err = rwlayer.Unmount()
+			layerStore.ReleaseRWLayer(rwlayer)
+			return err
+		}),
+		nil
+}
+
+// CommitBuildStep is used by the builder to create an image for each step in
+// the build.
+//
+// This method is different from CreateImageFromContainer:
+//   * it doesn't attempt to validate container state
+//   * it doesn't send a commit action to metrics
+//   * it doesn't log a container commit event
+//
+// This is a temporary shim. Should be removed when builder stops using commit.
+func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
+	container, err := daemon.GetContainer(c.ContainerID)
+	if err != nil {
+		return "", err
+	}
+	c.ContainerMountLabel = container.MountLabel
+	c.ContainerOS = container.OS
+	c.ParentImageID = string(container.ImageID)
+	return daemon.commitImage(c)
+}

components/engine/daemon/image_events.go

@@ -0,0 +1,29 @@
+package daemon // import "github.com/docker/docker/daemon"
+
+import (
+	"github.com/docker/docker/api/types/events"
+)
+
+// LogImageEvent generates an event related to an image with only the default attributes.
+func (daemon *Daemon) LogImageEvent(imageID, refName, action string) {
+	daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{})
+}
+
+// LogImageEventWithAttributes generates an event related to an image with specific given attributes.
+func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) {
+	img, err := daemon.GetImage(imageID)
+	if err == nil && img.Config != nil {
+		// image has not been removed yet.
+		// it could be missing if the event is `delete`.
+		copyAttributes(attributes, img.Config.Labels)
+	}
+	if refName != "" {
+		attributes["name"] = refName
+	}
+	actor := events.Actor{
+		ID:         imageID,
+		Attributes: attributes,
+	}
+
+	daemon.EventsService.Log(action, events.ImageEventType, actor)
+}

components/engine/daemon/image_prune.go

@@ -0,0 +1,168 @@
+package daemon // import "github.com/docker/docker/daemon"
+
+import (
+	"sync/atomic"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+)
+
+var imagesAcceptedFilters = map[string]bool{
+	"dangling": true,
+	"label":    true,
+	"label!":   true,
+	"until":    true,
+}
+
+// ImagesPrune removes unused images
+func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) {
+	if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) {
+		return nil, errPruneRunning
+	}
+	defer atomic.StoreInt32(&daemon.pruneRunning, 0)
+
+	// make sure that only accepted filters have been received
+	err := pruneFilters.Validate(imagesAcceptedFilters)
+	if err != nil {
+		return nil, err
+	}
+
+	rep := &types.ImagesPruneReport{}
+
+	danglingOnly := true
+	if pruneFilters.Contains("dangling") {
+		if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") {
+			danglingOnly = false
+		} else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") {
+			return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")}
+		}
+	}
+
+	until, err := getUntilFromPruneFilters(pruneFilters)
+	if err != nil {
+		return nil, err
+	}
+
+	var allImages map[image.ID]*image.Image
+	if danglingOnly {
+		allImages = daemon.imageStore.Heads()
+	} else {
+		allImages = daemon.imageStore.Map()
+	}
+
+	// Filter intermediary images and get their unique size
+	allLayers := make(map[layer.ChainID]layer.Layer)
+	for _, ls := range daemon.layerStores {
+		for k, v := range ls.Map() {
+			allLayers[k] = v
+		}
+	}
+	topImages := map[image.ID]*image.Image{}
+	for id, img := range allImages {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+			dgst := digest.Digest(id)
+			if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 {
+				continue
+			}
+			if !until.IsZero() && img.Created.After(until) {
+				continue
+			}
+			if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) {
+				continue
+			}
+			topImages[id] = img
+		}
+	}
+
+	canceled := false
+deleteImagesLoop:
+	for id := range topImages {
+		select {
+		case <-ctx.Done():
+			// we still want to calculate freed size and return the data
+			canceled = true
+			break deleteImagesLoop
+		default:
+		}
+
+		deletedImages := []types.ImageDeleteResponseItem{}
+		refs := daemon.referenceStore.References(id.Digest())
+		if len(refs) > 0 {
+			shouldDelete := !danglingOnly
+			if !shouldDelete {
+				hasTag := false
+				for _, ref := range refs {
+					if _, ok := ref.(reference.NamedTagged); ok {
+						hasTag = true
+						break
+					}
+				}
+
+				// Only delete if it's untagged (i.e. repo:<none>)
+				shouldDelete = !hasTag
+			}
+
+			if shouldDelete {
+				for _, ref := range refs {
+					imgDel, err := daemon.ImageDelete(ref.String(), false, true)
+					if imageDeleteFailed(ref.String(), err) {
+						continue
+					}
+					deletedImages = append(deletedImages, imgDel...)
+				}
+			}
+		} else {
+			hex := id.Digest().Hex()
+			imgDel, err := daemon.ImageDelete(hex, false, true)
+			if imageDeleteFailed(hex, err) {
+				continue
+			}
+			deletedImages = append(deletedImages, imgDel...)
+		}
+
+		rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...)
+	}
+
+	// Compute how much space was freed
+	for _, d := range rep.ImagesDeleted {
+		if d.Deleted != "" {
+			chid := layer.ChainID(d.Deleted)
+			if l, ok := allLayers[chid]; ok {
+				diffSize, err := l.DiffSize()
+				if err != nil {
+					logrus.Warnf("failed to get layer %s size: %v", chid, err)
+					continue
+				}
+				rep.SpaceReclaimed += uint64(diffSize)
+			}
+		}
+	}
+
+	if canceled {
+		logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep)
+	}
+
+	return rep, nil
+}
+
+func imageDeleteFailed(ref string, err error) bool {
+	switch {
+	case err == nil:
+		return false
+	case errdefs.IsConflict(err):
+		return true
+	default:
+		logrus.Warnf("failed to prune image %s: %v", ref, err)
+		return true
+	}
+}

components/engine/daemon/image_windows.go

@@ -0,0 +1,35 @@
+package daemon // import "github.com/docker/docker/daemon"
+
+import (
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/system"
+	"github.com/pkg/errors"
+)
+
+// GetLayerFolders returns the layer folders from an image RootFS
+func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) {
+	folders := []string{}
+	max := len(img.RootFS.DiffIDs)
+	for index := 1; index <= max; index++ {
+		// FIXME: why does this mutate the RootFS?
+		img.RootFS.DiffIDs = img.RootFS.DiffIDs[:index]
+		if !system.IsOSSupported(img.OperatingSystem()) {
+			return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID())
+		}
+		layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
+		}
+		// Reverse order, expecting parent first
+		folders = append([]string{layerPath}, folders...)
+	}
+	if rwLayer == nil {
+		return nil, errors.New("RWLayer is unexpectedly nil")
+	}
+	m, err := rwLayer.Metadata()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get layer metadata")
+	}
+	return append(folders, m["dir"]), nil
+}

components/engine/daemon/oci_windows.go

@@ -1,7 +1,6 @@
 package daemon // import "github.com/docker/docker/daemon"
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
@@ -10,11 +9,11 @@ import (
 
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
-	"github.com/docker/docker/layer"
 	"github.com/docker/docker/oci"
 	"github.com/docker/docker/pkg/sysinfo"
 	"github.com/docker/docker/pkg/system"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/registry"
 )
@@ -139,29 +138,10 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 		}
 	}
 	s.Process.User.Username = c.Config.User
-
-	// Get the layer path for each layer.
-	max := len(img.RootFS.DiffIDs)
-	for i := 1; i <= max; i++ {
-		img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
-		if !system.IsOSSupported(img.OperatingSystem()) {
-			return nil, fmt.Errorf("cannot get layerpath for ImageID %s: %s ", img.RootFS.ChainID(), system.ErrNotSupportedOperatingSystem)
-		}
-		layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
-		if err != nil {
-			return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID(), err)
-		}
-		// Reverse order, expecting parent most first
-		s.Windows.LayerFolders = append([]string{layerPath}, s.Windows.LayerFolders...)
-	}
-	if c.RWLayer == nil {
-		return nil, errors.New("RWLayer of container " + c.ID + " is unexpectedly nil")
-	}
-	m, err := c.RWLayer.Metadata()
+	s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get layer metadata - %s", err)
+		return nil, errors.Wrapf(err, "container %s", c.ID)
 	}
-	s.Windows.LayerFolders = append(s.Windows.LayerFolders, m["dir"])
 
 	dnsSearch := daemon.getDNSSearchSettings(c)
 

components/engine/daemon/prune.go

@@ -6,18 +6,13 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	timetypes "github.com/docker/docker/api/types/time"
-	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/directory"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/volume"
 	"github.com/docker/libnetwork"
-	digest "github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 )
@@ -36,12 +31,7 @@ var (
 		"label":  true,
 		"label!": true,
 	}
-	imagesAcceptedFilters = map[string]bool{
-		"dangling": true,
-		"label":    true,
-		"label!":   true,
-		"until":    true,
-	}
+
 	networksAcceptedFilters = map[string]bool{
 		"label":  true,
 		"label!": true,
@@ -159,152 +149,6 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg
 	return rep, err
 }
 
-// ImagesPrune removes unused images
-func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) {
-	if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) {
-		return nil, errPruneRunning
-	}
-	defer atomic.StoreInt32(&daemon.pruneRunning, 0)
-
-	// make sure that only accepted filters have been received
-	err := pruneFilters.Validate(imagesAcceptedFilters)
-	if err != nil {
-		return nil, err
-	}
-
-	rep := &types.ImagesPruneReport{}
-
-	danglingOnly := true
-	if pruneFilters.Contains("dangling") {
-		if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") {
-			danglingOnly = false
-		} else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") {
-			return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")}
-		}
-	}
-
-	until, err := getUntilFromPruneFilters(pruneFilters)
-	if err != nil {
-		return nil, err
-	}
-
-	var allImages map[image.ID]*image.Image
-	if danglingOnly {
-		allImages = daemon.imageStore.Heads()
-	} else {
-		allImages = daemon.imageStore.Map()
-	}
-
-	// Filter intermediary images and get their unique size
-	allLayers := make(map[layer.ChainID]layer.Layer)
-	for _, ls := range daemon.layerStores {
-		for k, v := range ls.Map() {
-			allLayers[k] = v
-		}
-	}
-	topImages := map[image.ID]*image.Image{}
-	for id, img := range allImages {
-		select {
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		default:
-			dgst := digest.Digest(id)
-			if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 {
-				continue
-			}
-			if !until.IsZero() && img.Created.After(until) {
-				continue
-			}
-			if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) {
-				continue
-			}
-			topImages[id] = img
-		}
-	}
-
-	canceled := false
-deleteImagesLoop:
-	for id := range topImages {
-		select {
-		case <-ctx.Done():
-			// we still want to calculate freed size and return the data
-			canceled = true
-			break deleteImagesLoop
-		default:
-		}
-
-		deletedImages := []types.ImageDeleteResponseItem{}
-		refs := daemon.referenceStore.References(id.Digest())
-		if len(refs) > 0 {
-			shouldDelete := !danglingOnly
-			if !shouldDelete {
-				hasTag := false
-				for _, ref := range refs {
-					if _, ok := ref.(reference.NamedTagged); ok {
-						hasTag = true
-						break
-					}
-				}
-
-				// Only delete if it's untagged (i.e. repo:<none>)
-				shouldDelete = !hasTag
-			}
-
-			if shouldDelete {
-				for _, ref := range refs {
-					imgDel, err := daemon.ImageDelete(ref.String(), false, true)
-					if imageDeleteFailed(ref.String(), err) {
-						continue
-					}
-					deletedImages = append(deletedImages, imgDel...)
-				}
-			}
-		} else {
-			hex := id.Digest().Hex()
-			imgDel, err := daemon.ImageDelete(hex, false, true)
-			if imageDeleteFailed(hex, err) {
-				continue
-			}
-			deletedImages = append(deletedImages, imgDel...)
-		}
-
-		rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...)
-	}
-
-	// Compute how much space was freed
-	for _, d := range rep.ImagesDeleted {
-		if d.Deleted != "" {
-			chid := layer.ChainID(d.Deleted)
-			if l, ok := allLayers[chid]; ok {
-				diffSize, err := l.DiffSize()
-				if err != nil {
-					logrus.Warnf("failed to get layer %s size: %v", chid, err)
-					continue
-				}
-				rep.SpaceReclaimed += uint64(diffSize)
-			}
-		}
-	}
-
-	if canceled {
-		logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep)
-	}
-
-	return rep, nil
-}
-
-func imageDeleteFailed(ref string, err error) bool {
-	switch {
-	case err == nil:
-		return false
-	case errdefs.IsConflict(err):
-		return true
-	default:
-		logrus.Warnf("failed to prune image %s: %v", ref, err)
-		return true
-	}
-}
-
 // localNetworksPrune removes unused local networks
 func (daemon *Daemon) localNetworksPrune(ctx context.Context, pruneFilters filters.Args) *types.NetworksPruneReport {
 	rep := &types.NetworksPruneReport{}

components/engine/hack/dockerfile/binaries-commits

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# When updating TOMLV_COMMIT, consider updating github.com/BurntSushi/toml
-# in vendor.conf accordingly
-TOMLV_COMMIT=a368813c5e648fee92e5f6c30e3944ff9d5e8895
-
-# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
-RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5
-
-# containerd is also pinned in vendor.conf. When updating the binary
-# version you may also need to update the vendor version to pick up bug
-# fixes or new APIs.
-CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2
-TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574
-
-# LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
-# updating the binary version, consider updating github.com/docker/libnetwork
-# in vendor.conf accordingly
-LIBNETWORK_COMMIT=ed2130d117c11c542327b4d5216a5db36770bc65
-VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384
-
-# Linting
-GOMETALINTER_COMMIT=bfcc1d6942136fd86eb6f1a6fb328de8398fbd80

components/engine/hack/dockerfile/install-binaries.sh

@@ -1,176 +0,0 @@
-#!/usr/bin/env bash
-set -e
-set -x
-
-. $(dirname "$0")/binaries-commits
-
-RM_GOPATH=0
-
-TMP_GOPATH=${TMP_GOPATH:-""}
-
-if [ -z "$TMP_GOPATH" ]; then
-	export GOPATH="$(mktemp -d)"
-	RM_GOPATH=1
-else
-	export GOPATH="$TMP_GOPATH"
-fi
-
-# Do not build with ambient capabilities support
-RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}"
-
-install_runc() {
-	echo "Install runc version $RUNC_COMMIT"
-	git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc"
-	cd "$GOPATH/src/github.com/opencontainers/runc"
-	git checkout -q "$RUNC_COMMIT"
-	make BUILDTAGS="$RUNC_BUILDTAGS" $1
-	cp runc /usr/local/bin/docker-runc
-}
-
-install_containerd() {
-	echo "Install containerd version $CONTAINERD_COMMIT"
-	git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd"
-	cd "$GOPATH/src/github.com/containerd/containerd"
-	git checkout -q "$CONTAINERD_COMMIT"
-	(
-		export GOPATH
-		make
-	)
-	cp bin/containerd /usr/local/bin/docker-containerd
-	cp bin/containerd-shim /usr/local/bin/docker-containerd-shim
-	cp bin/ctr /usr/local/bin/docker-containerd-ctr
-}
-
-install_containerd_static() {
-	echo "Install containerd version $CONTAINERD_COMMIT"
-	git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd"
-	cd "$GOPATH/src/github.com/containerd/containerd"
-	git checkout -q "$CONTAINERD_COMMIT"
-	(
-		export GOPATH
-		make BUILDTAGS='static_build netgo' EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"'
-	)
-	cp bin/containerd /usr/local/bin/docker-containerd
-	cp bin/containerd-shim /usr/local/bin/docker-containerd-shim
-	cp bin/ctr /usr/local/bin/docker-containerd-ctr
-}
-
-install_proxy() {
-	echo "Install docker-proxy version $LIBNETWORK_COMMIT"
-	git clone https://github.com/docker/libnetwork.git "$GOPATH/src/github.com/docker/libnetwork"
-	cd "$GOPATH/src/github.com/docker/libnetwork"
-	git checkout -q "$LIBNETWORK_COMMIT"
-	go build -buildmode=pie -ldflags="$PROXY_LDFLAGS" -o /usr/local/bin/docker-proxy github.com/docker/libnetwork/cmd/proxy
-}
-
-install_dockercli() {
-	DOCKERCLI_CHANNEL=${DOCKERCLI_CHANNEL:-edge}
-	DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce}
-	echo "Install docker/cli version $DOCKERCLI_VERSION from $DOCKERCLI_CHANNEL"
-
-	arch=$(uname -m)
-	# No official release of these platforms
-	if [[ "$arch" != "x86_64" ]] && [[ "$arch" != "s390x" ]]; then
-		build_dockercli
-		return
-	fi
-
-	url=https://download.docker.com/linux/static
-	curl -Ls $url/$DOCKERCLI_CHANNEL/$arch/docker-$DOCKERCLI_VERSION.tgz | \
-	tar -xz docker/docker
-	mv docker/docker /usr/local/bin/
-	rmdir docker
-}
-
-build_dockercli() {
-	DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce}
-	git clone https://github.com/docker/docker-ce "$GOPATH/tmp/docker-ce"
-	cd "$GOPATH/tmp/docker-ce"
-	git checkout -q "v$DOCKERCLI_VERSION"
-	mkdir -p "$GOPATH/src/github.com/docker"
-	mv components/cli "$GOPATH/src/github.com/docker/cli"
-	go build -buildmode=pie -o /usr/local/bin/docker github.com/docker/cli/cmd/docker
-}
-
-install_gometalinter() {
-	echo "Installing gometalinter version $GOMETALINTER_COMMIT"
-	go get -d github.com/alecthomas/gometalinter
-	cd "$GOPATH/src/github.com/alecthomas/gometalinter"
-	git checkout -q "$GOMETALINTER_COMMIT"
-	go build -buildmode=pie -o /usr/local/bin/gometalinter github.com/alecthomas/gometalinter
-	GOBIN=/usr/local/bin gometalinter --install
-}
-
-for prog in "$@"
-do
-	case $prog in
-		tomlv)
-			echo "Install tomlv version $TOMLV_COMMIT"
-			git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml"
-			cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT"
-			go build -buildmode=pie -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv
-			;;
-
-		runc)
-			install_runc static
-			;;
-
-		runc-dynamic)
-			install_runc
-			;;
-
-		containerd)
-			install_containerd_static
-			;;
-
-		containerd-dynamic)
-			install_containerd
-			;;
-
-		gometalinter)
-			install_gometalinter
-			;;
-
-		tini)
-			echo "Install tini version $TINI_COMMIT"
-			git clone https://github.com/krallin/tini.git "$GOPATH/tini"
-			cd "$GOPATH/tini"
-			git checkout -q "$TINI_COMMIT"
-			cmake .
-			make tini-static
-			cp tini-static /usr/local/bin/docker-init
-			;;
-
-		proxy)
-			(
-				export CGO_ENABLED=0
-				install_proxy
-			)
-			;;
-
-		proxy-dynamic)
-			PROXY_LDFLAGS="-linkmode=external" install_proxy
-			;;
-
-		vndr)
-			echo "Install vndr version $VNDR_COMMIT"
-			git clone https://github.com/LK4D4/vndr.git "$GOPATH/src/github.com/LK4D4/vndr"
-			cd "$GOPATH/src/github.com/LK4D4/vndr"
-			git checkout -q "$VNDR_COMMIT"
-			go build -buildmode=pie -v -o /usr/local/bin/vndr .
-			;;
-
-		dockercli)
-			install_dockercli
-			;;
-
-		*)
-			echo echo "Usage: $0 [tomlv|runc|runc-dynamic|containerd|containerd-dynamic|tini|proxy|proxy-dynamic|vndr|dockercli|gometalinter]"
-			exit 1
-
-	esac
-done
-
-if [ $RM_GOPATH -eq 1 ]; then
-	rm -rf "$GOPATH"
-fi

components/engine/hack/dockerfile/install/containerd.installer

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+
+# containerd is also pinned in vendor.conf. When updating the binary
+# version you may also need to update the vendor version to pick up bug
+# fixes or new APIs.
+CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2
+
+install_containerd() {
+	echo "Install containerd version $CONTAINERD_COMMIT"
+	git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd"
+	cd "$GOPATH/src/github.com/containerd/containerd"
+	git checkout -q "$CONTAINERD_COMMIT"
+
+	(
+
+		if [ "$1" == "static" ]; then
+			export BUILDTAGS='static_build netgo'
+			export EXTRA_FLAGS='-buildmod pie'
+			export EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"'
+		fi
+
+		make
+	)
+
+	mkdir -p ${PREFIX}
+
+	cp bin/containerd ${PREFIX}/docker-containerd
+	cp bin/containerd-shim ${PREFIX}/docker-containerd-shim
+	cp bin/ctr ${PREFIX}/docker-containerd-ctr
+}

components/engine/hack/dockerfile/install/dockercli.installer

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+DOCKERCLI_CHANNEL=${DOCKERCLI_CHANNEL:-edge}
+DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce}
+
+install_dockercli() {
+	echo "Install docker/cli version $DOCKERCLI_VERSION from $DOCKERCLI_CHANNEL"
+
+	arch=$(uname -m)
+	# No official release of these platforms
+	if [[ "$arch" != "x86_64" ]] && [[ "$arch" != "s390x" ]]; then
+		build_dockercli
+		return
+	fi
+
+	url=https://download.docker.com/linux/static
+	curl -Ls $url/$DOCKERCLI_CHANNEL/$arch/docker-$DOCKERCLI_VERSION.tgz | \
+	tar -xz docker/docker
+	mkdir -p ${PREFIX}
+	mv docker/docker ${PREFIX}/
+	rmdir docker
+}
+
+build_dockercli() {
+	git clone https://github.com/docker/docker-ce "$GOPATH/tmp/docker-ce"
+	cd "$GOPATH/tmp/docker-ce"
+	git checkout -q "v$DOCKERCLI_VERSION"
+	mkdir -p "$GOPATH/src/github.com/docker"
+	mv components/cli "$GOPATH/src/github.com/docker/cli"
+	go build -buildmode=pie -o ${PREFIX}/docker github.com/docker/cli/cmd/docker
+}

components/engine/hack/dockerfile/install/gometalinter.installer

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+GOMETALINTER_COMMIT=bfcc1d6942136fd86eb6f1a6fb328de8398fbd80
+
+install_gometalinter() {
+	echo "Installing gometalinter version $GOMETALINTER_COMMIT"
+	go get -d github.com/alecthomas/gometalinter
+	cd "$GOPATH/src/github.com/alecthomas/gometalinter"
+	git checkout -q "$GOMETALINTER_COMMIT"
+	go build -buildmode=pie -o ${PREFIX}/gometalinter github.com/alecthomas/gometalinter
+	GOBIN=${PREFIX} ${PREFIX}/gometalinter --install
+}

components/engine/hack/dockerfile/install/install.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+set -x
+
+RM_GOPATH=0
+
+TMP_GOPATH=${TMP_GOPATH:-""}
+
+: ${PREFIX:="/usr/local/bin"}
+
+if [ -z "$TMP_GOPATH" ]; then
+	export GOPATH="$(mktemp -d)"
+	RM_GOPATH=1
+else
+	export GOPATH="$TMP_GOPATH"
+fi
+
+dir="$(dirname $0)"
+
+bin=$1
+shift
+
+if [ ! -f "${dir}/${bin}.installer" ]; then
+	echo "Could not find installer for \"$bin\""
+	exit 1
+fi
+
+. $dir/$bin.installer
+install_$bin "$@"

components/engine/hack/dockerfile/install/proxy.installer

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
+# updating the binary version, consider updating github.com/docker/libnetwork
+# in vendor.conf accordingly
+LIBNETWORK_COMMIT=ed2130d117c11c542327b4d5216a5db36770bc65
+
+install_proxy() {
+	case "$1" in
+		"dynamic")
+			install_proxy_dynamic
+			return
+			;;
+		"")
+			export CGO_ENABLED=0
+			_install_proxy
+			;;
+		*)
+			echo 'Usage: $0 [dynamic]'
+			;;
+	esac
+}
+
+install_proxy_dynamic() {
+	export PROXY_LDFLAGS="-linkmode=external" install_proxy
+	_install_proxy
+}
+
+_install_proxy() {
+	echo "Install docker-proxy version $LIBNETWORK_COMMIT"
+	git clone https://github.com/docker/libnetwork.git "$GOPATH/src/github.com/docker/libnetwork"
+	cd "$GOPATH/src/github.com/docker/libnetwork"
+	git checkout -q "$LIBNETWORK_COMMIT"
+	go build -buildmode=pie -ldflags="$PROXY_LDFLAGS" -o ${PREFIX}/docker-proxy github.com/docker/libnetwork/cmd/proxy
+}
+
+

components/engine/hack/dockerfile/install/runc.installer

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
+RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5
+
+install_runc() {
+	# Do not build with ambient capabilities support
+	RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}"
+
+	echo "Install runc version $RUNC_COMMIT"
+	git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc"
+	cd "$GOPATH/src/github.com/opencontainers/runc"
+	git checkout -q "$RUNC_COMMIT"
+	make BUILDTAGS="$RUNC_BUILDTAGS" $1
+	mkdir -p ${PREFIX}
+	cp runc ${PREFIX}/docker-runc
+}

components/engine/hack/dockerfile/install/tini.installer

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574
+
+install_tini() {
+	echo "Install tini version $TINI_COMMIT"
+	git clone https://github.com/krallin/tini.git "$GOPATH/tini"
+	cd "$GOPATH/tini"
+	git checkout -q "$TINI_COMMIT"
+	cmake .
+	make tini-static
+	mkdir -p ${PREFIX}
+	cp tini-static ${PREFIX}/docker-init
+}

components/engine/hack/dockerfile/install/tomlv.installer

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# When updating TOMLV_COMMIT, consider updating github.com/BurntSushi/toml
+# in vendor.conf accordingly
+TOMLV_COMMIT=a368813c5e648fee92e5f6c30e3944ff9d5e8895
+
+install_tomlv() {
+	echo "Install tomlv version $TOMLV_COMMIT"
+	git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml"
+	cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT"
+	go build -v -buildmode=pie -o ${PREFIX}/tomlv github.com/BurntSushi/toml/cmd/tomlv
+}

components/engine/hack/dockerfile/install/vndr.installer

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384
+
+install_vndr() {
+	echo "Install vndr version $VNDR_COMMIT"
+	git clone https://github.com/LK4D4/vndr.git "$GOPATH/src/github.com/LK4D4/vndr"
+	cd "$GOPATH/src/github.com/LK4D4/vndr"
+	git checkout -q "$VNDR_COMMIT"
+	go build -buildmode=pie -v -o ${PREFIX}/vndr .
+}

components/engine/hack/make/.go-autogen

@@ -2,7 +2,9 @@
 
 rm -rf autogen
 
-source hack/dockerfile/binaries-commits
+source hack/dockerfile/install/runc.installer
+source hack/dockerfile/install/tini.installer
+source hack/dockerfile/install/containerd.installer
 
 cat > dockerversion/version_autogen.go <<DVEOF
 // +build autogen

components/engine/integration-cli/docker_cli_attach_unix_test.go

@@ -69,10 +69,10 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) {
 	cmd.Stdout = tty
 	cmd.Stderr = tty
 
-	errChan := make(chan error)
+	cmdExit := make(chan error)
 	go func() {
-		errChan <- cmd.Run()
-		close(errChan)
+		cmdExit <- cmd.Run()
+		close(cmdExit)
 	}()
 
 	c.Assert(waitRun(name), check.IsNil)
@@ -82,12 +82,7 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) {
 	cpty.Write([]byte{17})
 
 	select {
-	case err := <-errChan:
-		if err != nil {
-			buff := make([]byte, 200)
-			tty.Read(buff)
-			c.Fatalf("%s: %s", err, buff)
-		}
+	case <-cmdExit:
 	case <-time.After(5 * time.Second):
 		c.Fatal("timeout while detaching")
 	}
@@ -102,6 +97,7 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) {
 
 	err = cmd.Start()
 	c.Assert(err, checker.IsNil)
+	defer cmd.Process.Kill()
 
 	bytes := make([]byte, 10)
 	var nBytes int
@@ -124,11 +120,7 @@ func (s *DockerSuite) TestAttachAfterDetach(c *check.C) {
 		c.Fatal("timeout waiting for attach read")
 	}
 
-	err = cmd.Wait()
-	c.Assert(err, checker.IsNil)
-
 	c.Assert(string(bytes[:nBytes]), checker.Contains, "/ #")
-
 }
 
 // TestAttachDetach checks that attach in tty mode can be detached using the long container ID

components/engine/vendor.conf

@@ -32,7 +32,7 @@ github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2
 
 #get libnetwork packages
 
-# When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/binaries-commits accordingly
+# When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
 github.com/docker/libnetwork ed2130d117c11c542327b4d5216a5db36770bc65
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
@@ -47,7 +47,7 @@ github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef
 github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25
 github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e
 
-# When updating, consider updating TOMLV_COMMIT in hack/dockerfile/binaries-commits accordingly
+# When updating, consider updating TOMLV_COMMIT in hack/dockerfile/install/tomlv accordingly
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374
 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d
@@ -70,7 +70,7 @@ github.com/pborman/uuid v1.0
 
 google.golang.org/grpc v1.3.0
 
-# When updating, also update RUNC_COMMIT in hack/dockerfile/binaries-commits accordingly
+# When updating, also update RUNC_COMMIT in hack/dockerfile/install/runc accordingly
 github.com/opencontainers/runc 6c55f98695e902427906eed2c799e566e3d3dfb5
 github.com/opencontainers/runtime-spec v1.0.1
 github.com/opencontainers/image-spec v1.0.1
@@ -120,7 +120,7 @@ github.com/dmcgowan/go-tar go1.10
 github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577
 
 # cluster
-github.com/docker/swarmkit f74983e7c015a38a81c8642803a78b8322cf7eac
+github.com/docker/swarmkit 49a9d7f6ba3c1925262641e694c18eb43575f74b
 github.com/gogo/protobuf v0.4
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
 github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e

components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go

@@ -125,12 +125,17 @@ type clusterUpdate struct {
 
 // Dispatcher is responsible for dispatching tasks and tracking agent health.
 type Dispatcher struct {
-	// mu is a lock to provide  mutually exclusive access to dispatcher fields
-	// e.g. lastSeenManagers, networkBootstrapKeys, lastSeenRootCert etc.
+	// Mutex to synchronize access to dispatcher shared state e.g. nodes,
+	// lastSeenManagers, networkBootstrapKeys etc.
+	// TODO(anshul): This can potentially be removed and rpcRW used in its place.
 	mu sync.Mutex
-	// shutdownWait is used by stop() to wait for existing operations to finish.
-	shutdownWait sync.WaitGroup
-
+	// WaitGroup to handle the case when Stop() gets called before Run()
+	// has finished initializing the dispatcher.
+	wg sync.WaitGroup
+	// This RWMutex synchronizes RPC handlers and the dispatcher stop().
+	// The RPC handlers use the read lock while stop() uses the write lock
+	// and acts as a barrier to shutdown.
+	rpcRW                sync.RWMutex
 	nodes                *nodeStore
 	store                *store.MemoryStore
 	lastSeenManagers     []*api.WeightedPeer
@@ -253,11 +258,8 @@ func (d *Dispatcher) Run(ctx context.Context) error {
 	defer cancel()
 	d.ctx, d.cancel = context.WithCancel(ctx)
 	ctx = d.ctx
-
-	// If Stop() is called, it should wait
-	// for Run() to complete.
-	d.shutdownWait.Add(1)
-	defer d.shutdownWait.Done()
+	d.wg.Add(1)
+	defer d.wg.Done()
 	d.mu.Unlock()
 
 	publishManagers := func(peers []*api.Peer) {
@@ -320,15 +322,18 @@ func (d *Dispatcher) Stop() error {
 		return errors.New("dispatcher is already stopped")
 	}
 
-	// Cancel dispatcher context.
-	// This should also close the the streams in Tasks(), Assignments().
+	log := log.G(d.ctx).WithField("method", "(*Dispatcher).Stop")
+	log.Info("dispatcher stopping")
 	d.cancel()
 	d.mu.Unlock()
 
-	// Wait for the RPCs that are in-progress to finish.
-	d.shutdownWait.Wait()
-
+	// The active nodes list can be cleaned out only when all
+	// existing RPCs have finished.
+	// RPCs that start after rpcRW.Unlock() should find the context
+	// cancelled and should fail organically.
+	d.rpcRW.Lock()
 	d.nodes.Clean()
+	d.rpcRW.Unlock()
 
 	d.processUpdatesLock.Lock()
 	// In case there are any waiters. There is no chance of any starting
@@ -338,6 +343,14 @@ func (d *Dispatcher) Stop() error {
 	d.processUpdatesLock.Unlock()
 
 	d.clusterUpdateQueue.Close()
+
+	// TODO(anshul): This use of Wait() could be unsafe.
+	// According to go's documentation on WaitGroup,
+	// Add() with a positive delta that occur when the counter is zero
+	// must happen before a Wait().
+	// As is, dispatcher Stop() can race with Run().
+	d.wg.Wait()
+
 	return nil
 }
 
@@ -485,13 +498,13 @@ func nodeIPFromContext(ctx context.Context) (string, error) {
 
 // register is used for registration of node with particular dispatcher.
 func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, error) {
+	logLocal := log.G(ctx).WithField("method", "(*Dispatcher).register")
+	// prevent register until we're ready to accept it
 	dctx, err := d.isRunningLocked()
 	if err != nil {
 		return "", err
 	}
 
-	logLocal := log.G(ctx).WithField("method", "(*Dispatcher).register")
-
 	if err := d.nodes.CheckRateLimit(nodeID); err != nil {
 		return "", err
 	}
@@ -539,15 +552,8 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a
 // UpdateTaskStatus updates status of task. Node should send such updates
 // on every status change of its tasks.
 func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStatusRequest) (*api.UpdateTaskStatusResponse, error) {
-	// shutdownWait.Add() followed by isRunning() to ensures that
-	// if this rpc sees the dispatcher running,
-	// it will already have called Add() on the shutdownWait wait,
-	// which ensures that Stop() will wait for this rpc to complete.
-	// Note that Stop() first does Dispatcher.ctx.cancel() followed by
-	// shutdownWait.Wait() to make sure new rpc's don't start before waiting
-	// for existing ones to finish.
-	d.shutdownWait.Add(1)
-	defer d.shutdownWait.Done()
+	d.rpcRW.RLock()
+	defer d.rpcRW.RUnlock()
 
 	dctx, err := d.isRunningLocked()
 	if err != nil {
@@ -740,15 +746,8 @@ func (d *Dispatcher) processUpdates(ctx context.Context) {
 // of tasks which should be run on node, if task is not present in that list,
 // it should be terminated.
 func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServer) error {
-	// shutdownWait.Add() followed by isRunning() to ensures that
-	// if this rpc sees the dispatcher running,
-	// it will already have called Add() on the shutdownWait wait,
-	// which ensures that Stop() will wait for this rpc to complete.
-	// Note that Stop() first does Dispatcher.ctx.cancel() followed by
-	// shutdownWait.Wait() to make sure new rpc's don't start before waiting
-	// for existing ones to finish.
-	d.shutdownWait.Add(1)
-	defer d.shutdownWait.Done()
+	d.rpcRW.RLock()
+	defer d.rpcRW.RUnlock()
 
 	dctx, err := d.isRunningLocked()
 	if err != nil {
@@ -873,15 +872,8 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
 // Assignments is a stream of assignments for a node. Each message contains
 // either full list of tasks and secrets for the node, or an incremental update.
 func (d *Dispatcher) Assignments(r *api.AssignmentsRequest, stream api.Dispatcher_AssignmentsServer) error {
-	// shutdownWait.Add() followed by isRunning() to ensures that
-	// if this rpc sees the dispatcher running,
-	// it will already have called Add() on the shutdownWait wait,
-	// which ensures that Stop() will wait for this rpc to complete.
-	// Note that Stop() first does Dispatcher.ctx.cancel() followed by
-	// shutdownWait.Wait() to make sure new rpc's don't start before waiting
-	// for existing ones to finish.
-	d.shutdownWait.Add(1)
-	defer d.shutdownWait.Done()
+	d.rpcRW.RLock()
+	defer d.rpcRW.RUnlock()
 
 	dctx, err := d.isRunningLocked()
 	if err != nil {
@@ -1140,20 +1132,13 @@ func (d *Dispatcher) markNodeNotReady(id string, state api.NodeStatus_State, mes
 // Node should send new heartbeat earlier than now + TTL, otherwise it will
 // be deregistered from dispatcher and its status will be updated to NodeStatus_DOWN
 func (d *Dispatcher) Heartbeat(ctx context.Context, r *api.HeartbeatRequest) (*api.HeartbeatResponse, error) {
-	// shutdownWait.Add() followed by isRunning() to ensures that
-	// if this rpc sees the dispatcher running,
-	// it will already have called Add() on the shutdownWait wait,
-	// which ensures that Stop() will wait for this rpc to complete.
-	// Note that Stop() first does Dispatcher.ctx.cancel() followed by
-	// shutdownWait.Wait() to make sure new rpc's don't start before waiting
-	// for existing ones to finish.
-	d.shutdownWait.Add(1)
-	defer d.shutdownWait.Done()
+	d.rpcRW.RLock()
+	defer d.rpcRW.RUnlock()
 
-	// isRunningLocked() is not needed since its OK if
-	// the dispatcher context is cancelled while this call is in progress
-	// since Stop() which cancels the dispatcher context will wait for
-	// Heartbeat() to complete.
+	// Its OK to call isRunning() here instead of isRunningLocked()
+	// because of the rpcRW readlock above.
+	// TODO(anshul) other uses of isRunningLocked() can probably
+	// also be removed.
 	if !d.isRunning() {
 		return nil, status.Errorf(codes.Aborted, "dispatcher is stopped")
 	}
@@ -1192,15 +1177,8 @@ func (d *Dispatcher) getRootCACert() []byte {
 // a special boolean field Disconnect which if true indicates that node should
 // reconnect to another Manager immediately.
 func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_SessionServer) error {
-	// shutdownWait.Add() followed by isRunning() to ensures that
-	// if this rpc sees the dispatcher running,
-	// it will already have called Add() on the shutdownWait wait,
-	// which ensures that Stop() will wait for this rpc to complete.
-	// Note that Stop() first does Dispatcher.ctx.cancel() followed by
-	// shutdownWait.Wait() to make sure new rpc's don't start before waiting
-	// for existing ones to finish.
-	d.shutdownWait.Add(1)
-	defer d.shutdownWait.Done()
+	d.rpcRW.RLock()
+	defer d.rpcRW.RUnlock()
 
 	dctx, err := d.isRunningLocked()
 	if err != nil {
@@ -1208,6 +1186,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio
 	}
 
 	ctx := stream.Context()
+
 	nodeInfo, err := ca.RemoteNode(ctx)
 	if err != nil {
 		return err