Merge pull request #357 from jose-bigio/17.12_version

[17.12] bump version to 17.12-ce-rc4

CHANGELOG.md

@@ -5,6 +5,11 @@ information on the list of deprecated flags and APIs please have a look at
 https://docs.docker.com/engine/deprecated/ where target removal dates can also
 be found.
 
+IMPORTANT: 
+
+You should stop all containers and plugins **BEFORE** upgrading to Docker CE 17.12.
+See related PR: [moby/moby#35812](https://github.com/moby/moby/pull/35812) 
+
 ## 17.12.0-ce (2017-12-DD)
 
 
@@ -25,6 +30,8 @@ be found.
 - Fix behaviour of `rmi -f` with unexpected errors [docker/cli#654](https://github.com/docker/cli/pull/654)
 * Integrated Generic resource in service create [docker/cli#429](https://github.com/docker/cli/pull/429)
 - Fix external networks in stacks [docker/cli#743](https://github.com/docker/cli/pull/743)
+* Remove support for referencing images by image shortid [docker/cli#753](https://github.com/docker/cli/pull/753) and [moby/moby#35790](https://github.com/moby/moby/pull/35790)
+* Use commit-sha instead of tag for containerd [moby/moby#35770](https://github.com/moby/moby/pull/35770)
 
 ### Documentation
 
@@ -38,6 +45,7 @@ be found.
 + Add gelf log driver plugin to Windows build [moby/moby#35073](https://github.com/moby/moby/pull/35073)
 * Set timeout on splunk batch send [moby/moby#35496](https://github.com/moby/moby/pull/35496)
 * Update Graylog2/go-gelf [moby/moby#35765](https://github.com/moby/moby/pull/35765)
+- Fix aws logs batch size calculation [moby/moby#35726](https://github.com/moby/moby/pull/35726)
 
 ### Networking
 
@@ -49,6 +57,7 @@ be found.
 - Fix timeout on netlink sockets and watchmiss leak [moby/moby#35677](https://github.com/moby/moby/pull/35677)
 + New daemon config for networking diagnosis [moby/moby#35677](https://github.com/moby/moby/pull/35677)
 - Clean up node management logic [docker/libnetwork#2036](https://github.com/docker/libnetwork/pull/2036)
+- Allocate VIPs when endpoints are restored [docker/swarmkit#2474](https://github.com/docker/swarmkit/pull/2474)
 
 ### Runtime
 
@@ -81,10 +90,15 @@ be found.
 * Create labels when volume exists only remotely [moby/moby#34896](https://github.com/moby/moby/pull/34896)
 - Fix leaking container/exec state [moby/moby#35484](https://github.com/moby/moby/pull/35484)
 * Disallow using legacy (v1) registries [moby/moby#35751](https://github.com/moby/moby/pull/35751) and [docker/cli#747](https://github.com/docker/cli/pull/747)
+- Windows: Fix case insensitive filename matching against builder cache [moby/moby#35793](https://github.com/moby/moby/pull/35793)
+- Fix race conditions around process handling and error checks [moby/moby#35809](https://github.com/moby/moby/pull/35809)
+* Ensure containers are stopped on daemon startup [moby/moby#35805](https://github.com/moby/moby/pull/35805)
+* Follow containerd namespace conventions [moby/moby#35812](https://github.com/moby/moby/pull/35812)
 
 ### Swarm Mode
 
 + Added support for swarm service isolation mode [moby/moby#34424](https://github.com/moby/moby/pull/34424)
+- Fix task clean up for tasks that are complete [docker/swarmkit#2477](https://github.com/docker/swarmkit/pull/2477)
 
 ### Packaging
 

VERSION

@@ -1 +1 @@
-17.12.0-ce-rc3
+17.12.0-ce-rc4

components/cli/VERSION

@@ -1 +1 @@
-17.12.0-ce-rc3
+17.12.0-ce-rc4

components/cli/contrib/completion/bash/docker

@@ -2220,6 +2220,7 @@ _docker_daemon() {
 		--metrics-addr
 		--mtu
 		--network-control-plane-mtu
+		--node-generic-resource
 		--oom-score-adjust
 		--pidfile -p
 		--registry-mirror
@@ -2865,6 +2866,7 @@ _docker_inspect() {
 						$(__docker_services)
 						$(__docker_volumes)
 					" -- "$cur" ) )
+					__ltrim_colon_completions "$cur"
 					;;
 				container)
 					__docker_complete_containers_all
@@ -3395,6 +3397,7 @@ _docker_service_update_and_create() {
 			--dns-option
 			--dns-search
 			--env-file
+			--generic-resource
 			--group
 			--host
 			--mode
@@ -3456,6 +3459,8 @@ _docker_service_update_and_create() {
 			--dns-rm
 			--dns-search-add
 			--dns-search-rm
+			--generic-resource-add
+			--generic-resource-rm
 			--group-add
 			--group-rm
 			--host-add

components/cli/contrib/completion/zsh/_docker

@@ -745,7 +745,7 @@ __docker_container_subcommand() {
                 "($help)--privileged[Give extended Linux capabilities to the command]" \
                 "($help -t --tty)"{-t,--tty}"[Allocate a pseudo-tty]" \
                 "($help -u --user)"{-u=,--user=}"[Username or UID]:user:_users" \
-                "($help -w --workdir)"{-w=,--workdir=}"[Working directory inside the container]:directory:_directories"
+                "($help -w --workdir)"{-w=,--workdir=}"[Working directory inside the container]:directory:_directories" \
                 "($help -):containers:__docker_complete_running_containers" \
                 "($help -)*::command:->anycommand" && ret=0
             case $state in

components/cli/docs/deprecated.md

@@ -74,9 +74,13 @@ The `filter` param to filter the list of image by reference (name or name:tag) i
 ### `repository:shortid` image references
 **Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)**
 
-**Target For Removal In Release: v17.12**
+**Removed In Release: v17.12**
 
-`repository:shortid` syntax for referencing images is very little used, collides with tag references can be confused with digest references.
+The `repository:shortid` syntax for referencing images is very little used,
+collides with tag references, and can be confused with digest references.
+
+Support for the `repository:shortid` notation to reference images was removed
+in Docker 17.12.
 
 ### `docker daemon` subcommand
 **Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)**

components/cli/docs/reference/commandline/run.md

@@ -588,11 +588,11 @@ Use Docker's `--restart` to specify a container's *restart policy*. A restart
 policy controls whether the Docker daemon restarts a container after exit.
 Docker supports the following restart policies:
 
-| Policy    | Result                                                                                                                                                                                                                                                           |
-|:----------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `no`      | Do not automatically restart the container when it exits. This is the default.                                                                                                                                                                                   |
-| `failure` | Restart only if the container exits with a non-zero exit status. Optionally, limit the number of restart retries the Docker daemon attempts.                                                                                                                     |
-| `always`  | Always restart the container regardless of the exit status. When you specify always, the Docker daemon will try to restart the container indefinitely. The container will also always start on daemon startup, regardless of the current state of the container. |
+| Policy                     | Result                                                                                                                                                                                                                                                           |
+|:---------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `no`                       | Do not automatically restart the container when it exits. This is the default.                                                                                                                                                                                   |
+| `on-failure[:max-retries]` | Restart only if the container exits with a non-zero exit status. Optionally, limit the number of restart retries the Docker daemon attempts.                                                                                                                     |
+| `always`                   | Always restart the container regardless of the exit status. When you specify always, the Docker daemon will try to restart the container indefinitely. The container will also always start on daemon startup, regardless of the current state of the container. |
 
 ```bash
 $ docker run --restart=always redis

components/engine/daemon/daemon.go

@@ -62,8 +62,8 @@ import (
 	"github.com/pkg/errors"
 )
 
-// MainNamespace is the name of the namespace used for users containers
-const MainNamespace = "moby"
+// ContainersNamespace is the name of the namespace used for users containers
+const ContainersNamespace = "moby"
 
 var (
 	errSystemNotSupported = errors.New("the Docker daemon is not supported on this platform")
@@ -247,6 +247,11 @@ func (daemon *Daemon) restore() error {
 					logrus.WithError(err).Errorf("Failed to delete container %s from containerd", c.ID)
 					return
 				}
+			} else if !daemon.configStore.LiveRestoreEnabled {
+				if err := daemon.kill(c, c.StopSignal()); err != nil && !errdefs.IsNotFound(err) {
+					logrus.WithError(err).WithField("container", c.ID).Error("error shutting down container")
+					return
+				}
 			}
 
 			if c.IsRunning() || c.IsPaused() {
@@ -317,24 +322,24 @@ func (daemon *Daemon) restore() error {
 					activeSandboxes[c.NetworkSettings.SandboxID] = options
 					mapLock.Unlock()
 				}
-			} else {
-				// get list of containers we need to restart
+			}
 
-				// Do not autostart containers which
-				// has endpoints in a swarm scope
-				// network yet since the cluster is
-				// not initialized yet. We will start
-				// it after the cluster is
-				// initialized.
-				if daemon.configStore.AutoRestart && c.ShouldRestart() && !c.NetworkSettings.HasSwarmEndpoint {
-					mapLock.Lock()
-					restartContainers[c] = make(chan struct{})
-					mapLock.Unlock()
-				} else if c.HostConfig != nil && c.HostConfig.AutoRemove {
-					mapLock.Lock()
-					removeContainers[c.ID] = c
-					mapLock.Unlock()
-				}
+			// get list of containers we need to restart
+
+			// Do not autostart containers which
+			// has endpoints in a swarm scope
+			// network yet since the cluster is
+			// not initialized yet. We will start
+			// it after the cluster is
+			// initialized.
+			if daemon.configStore.AutoRestart && c.ShouldRestart() && !c.NetworkSettings.HasSwarmEndpoint {
+				mapLock.Lock()
+				restartContainers[c] = make(chan struct{})
+				mapLock.Unlock()
+			} else if c.HostConfig != nil && c.HostConfig.AutoRemove {
+				mapLock.Lock()
+				removeContainers[c.ID] = c
+				mapLock.Unlock()
 			}
 
 			c.Lock()
@@ -890,7 +895,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
 
 	go d.execCommandGC()
 
-	d.containerd, err = containerdRemote.NewClient(MainNamespace, d)
+	d.containerd, err = containerdRemote.NewClient(ContainersNamespace, d)
 	if err != nil {
 		return nil, err
 	}

components/engine/daemon/image.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/stringid"
 )
 
 // errImageDoesNotExist is error returned when no image can be found for a reference.
@@ -59,21 +58,6 @@ func (daemon *Daemon) GetImageIDAndOS(refOrID string) (image.ID, string, error)
 		return id, imageOS, nil
 	}
 
-	// deprecated: repo:shortid https://github.com/docker/docker/pull/799
-	if tagged, ok := namedRef.(reference.Tagged); ok {
-		if tag := tagged.Tag(); stringid.IsShortID(stringid.TruncateID(tag)) {
-			for platform := range daemon.stores {
-				if id, err := daemon.stores[platform].imageStore.Search(tag); err == nil {
-					for _, storeRef := range daemon.referenceStore.References(id.Digest()) {
-						if storeRef.Name() == namedRef.Name() {
-							return id, platform, nil
-						}
-					}
-				}
-			}
-		}
-	}
-
 	// Search based on ID
 	for os := range daemon.stores {
 		if id, err := daemon.stores[os].imageStore.Search(refOrID); err == nil {

components/engine/daemon/kill.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"runtime"
-	"strings"
 	"syscall"
 	"time"
 
+	"github.com/docker/docker/api/errdefs"
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/pkg/signal"
@@ -97,15 +97,11 @@ func (daemon *Daemon) killWithSignal(container *containerpkg.Container, sig int)
 	}
 
 	if err := daemon.kill(container, sig); err != nil {
-		err = errors.Wrapf(err, "Cannot kill container %s", container.ID)
-		// if container or process not exists, ignore the error
-		// TODO: we shouldn't have to parse error strings from containerd
-		if strings.Contains(err.Error(), "container not found") ||
-			strings.Contains(err.Error(), "no such process") {
-			logrus.Warnf("container kill failed because of 'container not found' or 'no such process': %s", err.Error())
+		if errdefs.IsNotFound(err) {
 			unpause = false
+			logrus.WithError(err).WithField("container", container.ID).WithField("action", "kill").Debug("container kill failed because of 'container not found' or 'no such process'")
 		} else {
-			return err
+			return errors.Wrapf(err, "Cannot kill container %s", container.ID)
 		}
 	}
 
@@ -171,7 +167,7 @@ func (daemon *Daemon) Kill(container *containerpkg.Container) error {
 // killPossibleDeadProcess is a wrapper around killSig() suppressing "no such process" error.
 func (daemon *Daemon) killPossiblyDeadProcess(container *containerpkg.Container, sig int) error {
 	err := daemon.killWithSignal(container, sig)
-	if err == syscall.ESRCH {
+	if errdefs.IsNotFound(err) {
 		e := errNoSuchProcess{container.GetPID(), sig}
 		logrus.Debug(e)
 		return e

components/engine/integration-cli/docker_cli_create_test.go

@@ -268,7 +268,6 @@ func (s *DockerSuite) TestCreateByImageID(c *check.C) {
 
 	dockerCmd(c, "create", imageID)
 	dockerCmd(c, "create", truncatedImageID)
-	dockerCmd(c, "create", fmt.Sprintf("%s:%s", imageName, truncatedImageID))
 
 	// Ensure this fails
 	out, exit, _ := dockerCmdWithError("create", fmt.Sprintf("%s:%s", imageName, imageID))
@@ -280,7 +279,10 @@ func (s *DockerSuite) TestCreateByImageID(c *check.C) {
 		c.Fatalf(`Expected %q in output; got: %s`, expected, out)
 	}
 
-	out, exit, _ = dockerCmdWithError("create", fmt.Sprintf("%s:%s", "wrongimage", truncatedImageID))
+	if i := strings.IndexRune(imageID, ':'); i >= 0 {
+		imageID = imageID[i+1:]
+	}
+	out, exit, _ = dockerCmdWithError("create", fmt.Sprintf("%s:%s", "wrongimage", imageID))
 	if exit == 0 {
 		c.Fatalf("expected non-zero exit code; received %d", exit)
 	}

components/engine/integration-cli/docker_cli_daemon_test.go

@@ -1451,7 +1451,7 @@ func (s *DockerDaemonSuite) TestCleanupMountsAfterDaemonAndContainerKill(c *chec
 
 	// kill the container
 	icmd.RunCommand(ctrBinary, "--address", "/var/run/docker/containerd/docker-containerd.sock",
-		"--namespace", moby_daemon.MainNamespace, "tasks", "kill", id).Assert(c, icmd.Success)
+		"--namespace", moby_daemon.ContainersNamespace, "tasks", "kill", id).Assert(c, icmd.Success)
 
 	// restart daemon.
 	d.Restart(c)
@@ -2011,7 +2011,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithKilledRunningContainer(t *check
 
 	// kill the container
 	icmd.RunCommand(ctrBinary, "--address", "/var/run/docker/containerd/docker-containerd.sock",
-		"--namespace", moby_daemon.MainNamespace, "tasks", "kill", cid).Assert(t, icmd.Success)
+		"--namespace", moby_daemon.ContainersNamespace, "tasks", "kill", cid).Assert(t, icmd.Success)
 
 	// Give time to containerd to process the command if we don't
 	// the exit event might be received after we do the inspect
@@ -2106,7 +2106,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithUnpausedRunningContainer(t *che
 	result := icmd.RunCommand(
 		ctrBinary,
 		"--address", "/var/run/docker/containerd/docker-containerd.sock",
-		"--namespace", moby_daemon.MainNamespace,
+		"--namespace", moby_daemon.ContainersNamespace,
 		"tasks", "resume", cid)
 	result.Assert(t, icmd.Success)
 

components/engine/integration-cli/docker_cli_tag_test.go

@@ -1,13 +1,10 @@
 package main
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/docker/docker/integration-cli/checker"
-	"github.com/docker/docker/integration-cli/cli/build"
 	"github.com/docker/docker/internal/testutil"
-	"github.com/docker/docker/pkg/stringid"
 	"github.com/go-check/check"
 )
 
@@ -140,29 +137,3 @@ func (s *DockerSuite) TestTagInvalidRepoName(c *check.C) {
 		c.Fatal("tagging with image named \"sha256\" should have failed")
 	}
 }
-
-// ensure tags cannot create ambiguity with image ids
-func (s *DockerSuite) TestTagTruncationAmbiguity(c *check.C) {
-	buildImageSuccessfully(c, "notbusybox:latest", build.WithDockerfile(`FROM busybox
-		MAINTAINER dockerio`))
-	imageID := getIDByName(c, "notbusybox:latest")
-	truncatedImageID := stringid.TruncateID(imageID)
-	truncatedTag := fmt.Sprintf("notbusybox:%s", truncatedImageID)
-
-	id := inspectField(c, truncatedTag, "Id")
-
-	// Ensure inspect by image id returns image for image id
-	c.Assert(id, checker.Equals, imageID)
-	c.Logf("Built image: %s", imageID)
-
-	// test setting tag fails
-	_, _, err := dockerCmdWithError("tag", "busybox:latest", truncatedTag)
-	if err != nil {
-		c.Fatalf("Error tagging with an image id: %s", err)
-	}
-
-	id = inspectField(c, truncatedTag, "Id")
-
-	// Ensure id is imageID and not busybox:latest
-	c.Assert(id, checker.Not(checker.Equals), imageID)
-}

components/engine/integration/container/restart_test.go

@@ -0,0 +1,112 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/integration-cli/daemon"
+)
+
+func TestDaemonRestartKillContainers(t *testing.T) {
+	type testCase struct {
+		desc       string
+		config     *container.Config
+		hostConfig *container.HostConfig
+
+		xRunning            bool
+		xRunningLiveRestore bool
+	}
+
+	for _, c := range []testCase{
+		{
+			desc:                "container without restart policy",
+			config:              &container.Config{Image: "busybox", Cmd: []string{"top"}},
+			xRunningLiveRestore: true,
+		},
+		{
+			desc:                "container with restart=always",
+			config:              &container.Config{Image: "busybox", Cmd: []string{"top"}},
+			hostConfig:          &container.HostConfig{RestartPolicy: container.RestartPolicy{Name: "always"}},
+			xRunning:            true,
+			xRunningLiveRestore: true,
+		},
+	} {
+		for _, liveRestoreEnabled := range []bool{false, true} {
+			for fnName, stopDaemon := range map[string]func(*testing.T, *daemon.Daemon){
+				"kill-daemon": func(t *testing.T, d *daemon.Daemon) {
+					if err := d.Kill(); err != nil {
+						t.Fatal(err)
+					}
+				},
+				"stop-daemon": func(t *testing.T, d *daemon.Daemon) {
+					d.Stop(t)
+				},
+			} {
+				t.Run(fmt.Sprintf("live-restore=%v/%s/%s", liveRestoreEnabled, c.desc, fnName), func(t *testing.T) {
+					c := c
+					liveRestoreEnabled := liveRestoreEnabled
+					stopDaemon := stopDaemon
+
+					t.Parallel()
+
+					d := daemon.New(t, "", "dockerd", daemon.Config{})
+					client, err := d.NewClient()
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					var args []string
+					if liveRestoreEnabled {
+						args = []string{"--live-restore"}
+					}
+
+					d.StartWithBusybox(t, args...)
+					defer d.Stop(t)
+					ctx := context.Background()
+
+					resp, err := client.ContainerCreate(ctx, c.config, c.hostConfig, nil, "")
+					if err != nil {
+						t.Fatal(err)
+					}
+					defer client.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
+
+					if err := client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
+						t.Fatal(err)
+					}
+
+					stopDaemon(t, d)
+					d.Start(t, args...)
+
+					expected := c.xRunning
+					if liveRestoreEnabled {
+						expected = c.xRunningLiveRestore
+					}
+
+					var running bool
+					for i := 0; i < 30; i++ {
+						inspect, err := client.ContainerInspect(ctx, resp.ID)
+						if err != nil {
+							t.Fatal(err)
+						}
+
+						running = inspect.State.Running
+						if running == expected {
+							break
+						}
+						time.Sleep(2 * time.Second)
+
+					}
+
+					if running != expected {
+						t.Fatalf("got unexpected running state, expected %v, got: %v", expected, running)
+					}
+					// TODO(cpuguy83): test pause states... this seems to be rather undefined currently
+				})
+			}
+		}
+	}
+}

components/engine/libcontainerd/client_daemon.go

@@ -27,6 +27,7 @@ import (
 	"github.com/containerd/containerd/archive"
 	"github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/linux/runctypes"
 	"github.com/containerd/typeurl"
@@ -42,7 +43,7 @@ import (
 const InitProcessName = "init"
 
 type container struct {
-	sync.Mutex
+	mu sync.Mutex
 
 	bundleDir string
 	ctr       containerd.Container
@@ -51,6 +52,54 @@ type container struct {
 	oomKilled bool
 }
 
+func (c *container) setTask(t containerd.Task) {
+	c.mu.Lock()
+	c.task = t
+	c.mu.Unlock()
+}
+
+func (c *container) getTask() containerd.Task {
+	c.mu.Lock()
+	t := c.task
+	c.mu.Unlock()
+	return t
+}
+
+func (c *container) addProcess(id string, p containerd.Process) {
+	c.mu.Lock()
+	if c.execs == nil {
+		c.execs = make(map[string]containerd.Process)
+	}
+	c.execs[id] = p
+	c.mu.Unlock()
+}
+
+func (c *container) deleteProcess(id string) {
+	c.mu.Lock()
+	delete(c.execs, id)
+	c.mu.Unlock()
+}
+
+func (c *container) getProcess(id string) containerd.Process {
+	c.mu.Lock()
+	p := c.execs[id]
+	c.mu.Unlock()
+	return p
+}
+
+func (c *container) setOOMKilled(killed bool) {
+	c.mu.Lock()
+	c.oomKilled = killed
+	c.mu.Unlock()
+}
+
+func (c *container) getOOMKilled() bool {
+	c.mu.Lock()
+	killed := c.oomKilled
+	c.mu.Unlock()
+	return killed
+}
+
 type client struct {
 	sync.RWMutex // protects containers map
 
@@ -160,10 +209,10 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
 // Start create and start a task for the specified containerd id
 func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio StdioCallback) (int, error) {
 	ctr := c.getContainer(id)
-	switch {
-	case ctr == nil:
+	if ctr == nil {
 		return -1, errors.WithStack(newNotFoundError("no such container"))
-	case ctr.task != nil:
+	}
+	if t := ctr.getTask(); t != nil {
 		return -1, errors.WithStack(newConflictError("container already started"))
 	}
 
@@ -227,9 +276,7 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 		return -1, err
 	}
 
-	c.Lock()
-	c.containers[id].task = t
-	c.Unlock()
+	ctr.setTask(t)
 
 	// Signal c.createIO that it can call CloseIO
 	close(stdinCloseSync)
@@ -239,9 +286,7 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 			c.logger.WithError(err).WithField("container", id).
 				Error("failed to delete task after fail start")
 		}
-		c.Lock()
-		c.containers[id].task = nil
-		c.Unlock()
+		ctr.setTask(nil)
 		return -1, err
 	}
 
@@ -250,12 +295,15 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 
 func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error) {
 	ctr := c.getContainer(containerID)
-	switch {
-	case ctr == nil:
+	if ctr == nil {
 		return -1, errors.WithStack(newNotFoundError("no such container"))
-	case ctr.task == nil:
+	}
+	t := ctr.getTask()
+	if t == nil {
 		return -1, errors.WithStack(newInvalidParameterError("container is not running"))
-	case ctr.execs != nil && ctr.execs[processID] != nil:
+	}
+
+	if p := ctr.getProcess(processID); p != nil {
 		return -1, errors.WithStack(newConflictError("id already in use"))
 	}
 
@@ -278,7 +326,7 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec *
 		}
 	}()
 
-	p, err = ctr.task.Exec(ctx, processID, spec, func(id string) (cio.IO, error) {
+	p, err = t.Exec(ctx, processID, spec, func(id string) (cio.IO, error) {
 		rio, err = c.createIO(fifos, containerID, processID, stdinCloseSync, attachStdio)
 		return rio, err
 	})
@@ -291,21 +339,14 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec *
 		return -1, err
 	}
 
-	ctr.Lock()
-	if ctr.execs == nil {
-		ctr.execs = make(map[string]containerd.Process)
-	}
-	ctr.execs[processID] = p
-	ctr.Unlock()
+	ctr.addProcess(processID, p)
 
 	// Signal c.createIO that it can call CloseIO
 	close(stdinCloseSync)
 
 	if err = p.Start(ctx); err != nil {
 		p.Delete(context.Background())
-		ctr.Lock()
-		delete(ctr.execs, processID)
-		ctr.Unlock()
+		ctr.deleteProcess(processID)
 		return -1, err
 	}
 
@@ -317,7 +358,7 @@ func (c *client) SignalProcess(ctx context.Context, containerID, processID strin
 	if err != nil {
 		return err
 	}
-	return p.Kill(ctx, syscall.Signal(signal))
+	return wrapError(p.Kill(ctx, syscall.Signal(signal)))
 }
 
 func (c *client) ResizeTerminal(ctx context.Context, containerID, processID string, width, height int) error {
@@ -431,12 +472,9 @@ func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, ti
 		return 255, time.Now(), nil
 	}
 
-	c.Lock()
-	if ctr, ok := c.containers[containerID]; ok {
-		ctr.task = nil
+	if ctr := c.getContainer(containerID); ctr != nil {
+		ctr.setTask(nil)
 	}
-	c.Unlock()
-
 	return status.ExitCode(), status.ExitTime(), nil
 }
 
@@ -470,7 +508,12 @@ func (c *client) Status(ctx context.Context, containerID string) (Status, error)
 		return StatusUnknown, errors.WithStack(newNotFoundError("no such container"))
 	}
 
-	s, err := ctr.task.Status(ctx)
+	t := ctr.getTask()
+	if t == nil {
+		return StatusUnknown, errors.WithStack(newNotFoundError("no such task"))
+	}
+
+	s, err := t.Status(ctx)
 	if err != nil {
 		return StatusUnknown, err
 	}
@@ -546,26 +589,22 @@ func (c *client) removeContainer(id string) {
 
 func (c *client) getProcess(containerID, processID string) (containerd.Process, error) {
 	ctr := c.getContainer(containerID)
-	switch {
-	case ctr == nil:
+	if ctr == nil {
 		return nil, errors.WithStack(newNotFoundError("no such container"))
-	case ctr.task == nil:
-		return nil, errors.WithStack(newNotFoundError("container is not running"))
-	case processID == InitProcessName:
-		return ctr.task, nil
-	default:
-		ctr.Lock()
-		defer ctr.Unlock()
-		if ctr.execs == nil {
-			return nil, errors.WithStack(newNotFoundError("no execs"))
-		}
 	}
 
-	p := ctr.execs[processID]
+	t := ctr.getTask()
+	if t == nil {
+		return nil, errors.WithStack(newNotFoundError("container is not running"))
+	}
+	if processID == InitProcessName {
+		return t, nil
+	}
+
+	p := ctr.getProcess(processID)
 	if p == nil {
 		return nil, errors.WithStack(newNotFoundError("no such exec"))
 	}
-
 	return p, nil
 }
 
@@ -623,12 +662,7 @@ func (c *client) processEvent(ctr *container, et EventType, ei EventInfo) {
 		}
 
 		if et == EventExit && ei.ProcessID != ei.ContainerID {
-			var p containerd.Process
-			ctr.Lock()
-			if ctr.execs != nil {
-				p = ctr.execs[ei.ProcessID]
-			}
-			ctr.Unlock()
+			p := ctr.getProcess(ei.ProcessID)
 			if p == nil {
 				c.logger.WithError(errors.New("no such process")).
 					WithFields(logrus.Fields{
@@ -644,9 +678,8 @@ func (c *client) processEvent(ctr *container, et EventType, ei EventInfo) {
 					"process":   ei.ProcessID,
 				}).Warn("failed to delete process")
 			}
-			c.Lock()
-			delete(ctr.execs, ei.ProcessID)
-			c.Unlock()
+			ctr.deleteProcess(ei.ProcessID)
+
 			ctr := c.getContainer(ei.ContainerID)
 			if ctr == nil {
 				c.logger.WithFields(logrus.Fields{
@@ -783,10 +816,10 @@ func (c *client) processEventStream(ctx context.Context) {
 		}
 
 		if oomKilled {
-			ctr.oomKilled = true
+			ctr.setOOMKilled(true)
 			oomKilled = false
 		}
-		ei.OOMKilled = ctr.oomKilled
+		ei.OOMKilled = ctr.getOOMKilled()
 
 		c.processEvent(ctr, et, ei)
 	}
@@ -816,12 +849,19 @@ func (c *client) writeContent(ctx context.Context, mediaType, ref string, r io.R
 }
 
 func wrapError(err error) error {
-	if err != nil {
-		msg := err.Error()
-		for _, s := range []string{"container does not exist", "not found", "no such container"} {
-			if strings.Contains(msg, s) {
-				return wrapNotFoundError(err)
-			}
+	if err == nil {
+		return nil
+	}
+
+	switch {
+	case errdefs.IsNotFound(err):
+		return wrapNotFoundError(err)
+	}
+
+	msg := err.Error()
+	for _, s := range []string{"container does not exist", "not found", "no such container"} {
+		if strings.Contains(msg, s) {
+			return wrapNotFoundError(err)
 		}
 	}
 	return err

components/engine/pkg/tarsum/fileinfosums.go

@@ -1,6 +1,10 @@
 package tarsum
 
-import "sort"
+import (
+	"runtime"
+	"sort"
+	"strings"
+)
 
 // FileInfoSumInterface provides an interface for accessing file checksum
 // information within a tar file. This info is accessed through interface
@@ -35,8 +39,11 @@ type FileInfoSums []FileInfoSumInterface
 
 // GetFile returns the first FileInfoSumInterface with a matching name.
 func (fis FileInfoSums) GetFile(name string) FileInfoSumInterface {
+	// We do case insensitive matching on Windows as c:\APP and c:\app are
+	// the same. See issue #33107.
 	for i := range fis {
-		if fis[i].Name() == name {
+		if (runtime.GOOS == "windows" && strings.EqualFold(fis[i].Name(), name)) ||
+			(runtime.GOOS != "windows" && fis[i].Name() == name) {
 			return fis[i]
 		}
 	}

components/engine/plugin/executor/containerd/containerd.go

@@ -16,7 +16,7 @@ import (
 )
 
 // PluginNamespace is the name used for the plugins namespace
-var PluginNamespace = "moby-plugins"
+var PluginNamespace = "plugins.moby"
 
 // ExitHandler represents an object that is called when the exit event is received from containerd
 type ExitHandler interface {

components/engine/vendor.conf

@@ -114,7 +114,7 @@ github.com/dmcgowan/go-tar go1.10
 github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f
 
 # cluster
-github.com/docker/swarmkit 4429c763170d9ca96929249353c3270c19e7d39e
+github.com/docker/swarmkit 7598f7a937de4ad0a856012bd548009ceeb0d10e
 github.com/gogo/protobuf v0.4
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
 github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e

components/engine/vendor/github.com/docker/swarmkit/manager/allocator/cnmallocator/networkallocator.go

@@ -404,6 +404,11 @@ func (na *cnmNetworkAllocator) IsServiceAllocated(s *api.Service, flags ...func(
 	vipLoop:
 		for _, vip := range s.Endpoint.VirtualIPs {
 			if na.IsVIPOnIngressNetwork(vip) && networkallocator.IsIngressNetworkNeeded(s) {
+				// This checks the condition when ingress network is needed
+				// but allocation has not been done.
+				if _, ok := na.services[s.ID]; !ok {
+					return false
+				}
 				continue vipLoop
 			}
 			for _, net := range specNetworks {

components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go

@@ -96,10 +96,10 @@ func (tr *TaskReaper) Run(ctx context.Context) {
 			// Serviceless tasks can be cleaned up right away since they are not attached to a service.
 			tr.cleanup = append(tr.cleanup, t.ID)
 		}
-		// tasks with desired state REMOVE that have progressed beyond SHUTDOWN can be cleaned up
+		// tasks with desired state REMOVE that have progressed beyond COMPLETE can be cleaned up
 		// right away
 		for _, t := range removeTasks {
-			if t.Status.State >= api.TaskStateShutdown {
+			if t.Status.State >= api.TaskStateCompleted {
 				tr.cleanup = append(tr.cleanup, t.ID)
 			}
 		}
@@ -138,10 +138,10 @@ func (tr *TaskReaper) Run(ctx context.Context) {
 				if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" {
 					tr.cleanup = append(tr.cleanup, t.ID)
 				}
-				// add tasks that have progressed beyond SHUTDOWN and have desired state REMOVE. These
+				// add tasks that have progressed beyond COMPLETE and have desired state REMOVE. These
 				// tasks are associated with slots that were removed as part of a service scale down
 				// or service removal.
-				if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateShutdown {
+				if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateCompleted {
 					tr.cleanup = append(tr.cleanup, t.ID)
 				}
 			case api.EventUpdateCluster:
@@ -282,6 +282,8 @@ func (tr *TaskReaper) tick() {
 
 // Stop stops the TaskReaper and waits for the main loop to exit.
 func (tr *TaskReaper) Stop() {
+	// TODO(dperny) calling stop on the task reaper twice will cause a panic
+	// because we try to close a channel that will already have been closed.
 	close(tr.stopChan)
 	<-tr.doneChan
 }