Merge pull request #268 from andrewhsu/v

[17.10] bump version to 17.10.0-ce-rc2

CHANGELOG.md

@@ -5,4 +5,42 @@ information on the list of deprecated flags and APIs please have a look at
 https://docs.docker.com/engine/deprecated/ where target removal dates can also
 be found.
 
-## 17.09.0-ce (2017-09-DD)
+## 17.10.0-ce (2017-10-DD)
+
+IMPORTANT: Starting with this release, `docker service create`, `docker service update`,
+`docker service scale` and `docker service rollback` use non-detached mode as default,
+use `--detach` to keep the old behaviour.
+
+### Builder
+
+* Reset uid/gid to 0 in uploaded build context to share build cache with other clients [docker/cli#513](https://github.com/docker/cli/pull/513)
++ Add support for `ADD` urls without any sub path [moby/moby#34217](https://github.com/moby/moby/pull/34217)
+
+### Client
+
+* Move output of `docker stack rm` to stdout [docker/cli#491](https://github.com/docker/cli/pull/491)
+* Use natural sort secrets and configs in cli [docker/cli#307](https://github.com/docker/cli/pull/307)
+* Use non-detached mode as default for `docker service` commands [docker/cli#525](https://github.com/docker/cli/pull/525)
+* Set APIVersion on the client, even when Ping fails [docker/cli#546](https://github.com/docker/cli/pull/546)
+- Fix loader error with different build syntax in `docker stack deploy` [docker/cli#544](https://github.com/docker/cli/pull/544)
+* Change the default output format for `docker container stats` to show `CONTAINER ID` and `NAME` [docker/cli#565](https://github.com/docker/cli/pull/565)
++ Add `--no-trunc` flag to `docker container stats` [docker/cli#565](https://github.com/docker/cli/pull/565)
++ Add experimental `docker trust`: `view`, `revoke`, `sign` subcommands [docker/cli#472](https://github.com/docker/cli/pull/472)
+
+### Networking
+
+* Enabling ILB/ELB on windows using per-node, per-network LB endpoint [moby/moby#34674](https://github.com/moby/moby/pull/34674)
+
+### Runtime
+
+* LCOW: Add UVM debugability by grabbing logs before tear-down [moby/moby#34846](https://github.com/moby/moby/pull/34846)
+* LCOW: Prepare work for bind mounts [moby/moby#34258](https://github.com/moby/moby/pull/34258)
+* LCOW: Support for docker cp, ADD/COPY on build [moby/moby#34252](https://github.com/moby/moby/pull/34252)
+* LCOW: VHDX boot to readonly [moby/moby#34754](https://github.com/moby/moby/pull/34754)
+* Volume: evaluate symlinks before relabeling mount source [moby/moby#34792](https://github.com/moby/moby/pull/34792)
+- Fixing ‘docker cp’ to allow new target file name in a host symlinked directory [moby/moby#31993](https://github.com/moby/moby/pull/31993)
+
+### Swarm Mode
+
+* Produce an error if `docker swarm init --force-new-cluster` is executed on worker nodes [moby/moby#34881](https://github.com/moby/moby/pull/34881)
++ Add support for `.Node.Hostname` templating in swarm services [moby/moby#34686](https://github.com/moby/moby/pull/34686)

VERSION

@@ -1 +1 @@
-17.09.0-dev
+17.10.0-ce-rc2

components/cli/VERSION

@@ -1 +1 @@
-17.10.0-dev
+17.10.0-ce-rc2

components/cli/cli/command/container/run.go

@@ -290,8 +290,11 @@ func attachContainer(
 		return nil, errAttach
 	}
 
+	ch := make(chan error, 1)
+	*errCh = ch
+
 	go func() {
-		*errCh <- func() error {
+		ch <- func() error {
 			streamer := hijackedIOStreamer{
 				streams:      dockerCli,
 				inputStream:  in,

components/cli/cli/command/image/pull.go

@@ -41,24 +41,25 @@ func NewPullCommand(dockerCli command.Cli) *cobra.Command {
 }
 
 func runPull(cli command.Cli, opts pullOptions) error {
-	ctx := context.Background()
-	imgRefAndAuth, err := trust.GetImageReferencesAndAuth(ctx, AuthResolver(cli), opts.remote)
-	if err != nil {
+	distributionRef, err := reference.ParseNormalizedNamed(opts.remote)
+	switch {
+	case err != nil:
 		return err
-	}
-
-	distributionRef := imgRefAndAuth.Reference()
-	if opts.all && !reference.IsNameOnly(distributionRef) {
+	case opts.all && !reference.IsNameOnly(distributionRef):
 		return errors.New("tag can't be used with --all-tags/-a")
-	}
-
-	if !opts.all && reference.IsNameOnly(distributionRef) {
+	case !opts.all && reference.IsNameOnly(distributionRef):
 		distributionRef = reference.TagNameOnly(distributionRef)
 		if tagged, ok := distributionRef.(reference.Tagged); ok {
 			fmt.Fprintf(cli.Out(), "Using default tag: %s\n", tagged.Tag())
 		}
 	}
 
+	ctx := context.Background()
+	imgRefAndAuth, err := trust.GetImageReferencesAndAuth(ctx, AuthResolver(cli), distributionRef.String())
+	if err != nil {
+		return err
+	}
+
 	// Check if reference has a digest
 	_, isCanonical := distributionRef.(reference.Canonical)
 	if command.IsTrusted() && !isCanonical {

components/cli/cli/command/image/pull_test.go

@@ -2,11 +2,14 @@ package image
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
+	"strings"
 	"testing"
 
 	"github.com/docker/cli/internal/test"
 	"github.com/docker/cli/internal/test/testutil"
+	"github.com/docker/docker/api/types"
 	"github.com/gotestyourself/gotestyourself/golden"
 	"github.com/stretchr/testify/assert"
 )
@@ -44,20 +47,28 @@ func TestNewPullCommandErrors(t *testing.T) {
 
 func TestNewPullCommandSuccess(t *testing.T) {
 	testCases := []struct {
-		name string
-		args []string
+		name        string
+		args        []string
+		expectedTag string
 	}{
 		{
-			name: "simple",
-			args: []string{"image:tag"},
+			name:        "simple",
+			args:        []string{"image:tag"},
+			expectedTag: "image:tag",
 		},
 		{
-			name: "simple-no-tag",
-			args: []string{"image"},
+			name:        "simple-no-tag",
+			args:        []string{"image"},
+			expectedTag: "image:latest",
 		},
 	}
 	for _, tc := range testCases {
-		cli := test.NewFakeCli(&fakeClient{})
+		cli := test.NewFakeCli(&fakeClient{
+			imagePullFunc: func(ref string, options types.ImagePullOptions) (io.ReadCloser, error) {
+				assert.Equal(t, tc.expectedTag, ref, tc.name)
+				return ioutil.NopCloser(strings.NewReader("")), nil
+			},
+		})
 		cmd := NewPullCommand(cli)
 		cmd.SetOutput(ioutil.Discard)
 		cmd.SetArgs(tc.args)

components/cli/cli/command/image/trust.go

@@ -200,7 +200,11 @@ func trustedPull(ctx context.Context, cli command.Cli, imgRefAndAuth trust.Image
 		if err != nil {
 			return err
 		}
-		if err := imagePullPrivileged(ctx, cli, imgRefAndAuth, false); err != nil {
+		updatedImgRefAndAuth, err := trust.GetImageReferencesAndAuth(ctx, AuthResolver(cli), trustedRef.String())
+		if err != nil {
+			return err
+		}
+		if err := imagePullPrivileged(ctx, cli, updatedImgRefAndAuth, false); err != nil {
 			return err
 		}
 

components/cli/contrib/completion/bash/docker

@@ -3224,7 +3224,7 @@ _docker_service_rm() {
 _docker_service_rollback() {
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--detach=false -d --help --quit -q" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--detach -d --help --quit -q" -- "$cur" ) )
 			;;
 		*)
 			local counter=$( __docker_pos_first_nonflag )
@@ -3238,7 +3238,7 @@ _docker_service_rollback() {
 _docker_service_scale() {
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--detach=false -d=false --help" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--detach -d --help" -- "$cur" ) )
 			;;
 		*)
 			__docker_complete_services
@@ -3339,7 +3339,7 @@ _docker_service_update_and_create() {
 	"
 
 	local boolean_options="
-		--detach=false -d=false
+		--detach -d
 		--help
 		--no-healthcheck
 		--read-only
@@ -4188,17 +4188,17 @@ _docker_secret() {
 
 _docker_secret_create() {
 	case "$prev" in
-		--label|-l)
+		--driver|-d|--label|-l)
 			return
 			;;
 	esac
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--help --label -l" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--driver -d --help --label -l" -- "$cur" ) )
 			;;
 		*)
-			local counter=$(__docker_pos_first_nonflag '--label|-l')
+			local counter=$(__docker_pos_first_nonflag '--driver|-d|--label|-l')
 			if [ "$cword" -eq "$((counter + 1))" ]; then
 				_filedir
 			fi
@@ -4345,16 +4345,20 @@ _docker_stack_deploy() {
 			_filedir yml
 			return
 			;;
+		--resolve-image)
+			COMPREPLY=( $( compgen -W "always changed never" -- "$cur" ) )
+			return
+			;;
 	esac
 
 	case "$cur" in
 		-*)
-			local options="--compose-file -c --help --prune --with-registry-auth"
+			local options="--compose-file -c --help --prune --resolve-image --with-registry-auth"
 			__docker_daemon_is_experimental && options+=" --bundle-file"
 			COMPREPLY=( $( compgen -W "$options" -- "$cur" ) )
 			;;
 		*)
-			local counter=$(__docker_pos_first_nonflag '--compose-file|-c|--bundle-file')
+			local counter=$(__docker_pos_first_nonflag '--bundle-file|--compose-file|-c|--resolve-image')
 			if [ "$cword" -eq "$counter" ]; then
 				__docker_complete_stacks
 			fi

components/cli/dockerfiles/Dockerfile.dev

@@ -1,7 +1,7 @@
 
 FROM    golang:1.8.3-alpine
 
-RUN     apk add -U git make bash coreutils
+RUN     apk add -U git make bash coreutils ca-certificates
 
 ARG     VNDR_SHA=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384
 RUN     go get -d github.com/LK4D4/vndr && \

components/cli/docs/deprecated.md

@@ -24,13 +24,16 @@ see [Feature Deprecation Policy](https://docs.docker.com/engine/#feature-depreca
 
 **Deprecated In Release: v17.05.0**
 
-**Disabled by default in release: [v17.09](https://github.com/docker/docker-ce/releases/tag/v17.09.0-ce)**
+**Disabled by default in release: [v17.10](https://github.com/docker/docker-ce/releases/tag/v17.10.0-ce)**
 
 Docker 17.05.0 added an optional `--detach=false` option to make the
 `docker service create` and `docker service update` work synchronously. This
-option will be enable by default in Docker 17.09, at which point the `--detach`
+option will be enabled by default in Docker 17.10, at which point the `--detach`
 flag can be used to use the previous (asynchronous) behavior.
 
+The default for this option will also be changed accordingly for `docker service rollback`
+and `docker service scale` in Docker 17.10.
+
 ### `-g` and `--graph` flags on `dockerd`
 
 **Deprecated In Release: v17.05.0**

components/cli/docs/reference/commandline/dockerd.md

@@ -967,7 +967,7 @@ Will make `hyperv` the default isolation technology on Windows. If no isolation
 value is specified on daemon start, on Windows client, the default is
 `hyperv`, and on Windows server, the default is `process`.
 
-#### Daemon DNS options
+### Daemon DNS options
 
 To set the DNS server for all Docker containers, use:
 
@@ -981,7 +981,7 @@ To set the DNS search domain for all Docker containers, use:
 $ sudo dockerd --dns-search example.com
 ```
 
-#### Allow push of nondistributable artifacts
+### Allow push of nondistributable artifacts
 
 Some images (e.g., Windows base images) contain artifacts whose distribution is
 restricted by license. When these images are pushed to a registry, restricted
@@ -1007,7 +1007,7 @@ images without connecting to another server.
 > artifacts to private registries and ensure that you are in compliance with
 > any terms that cover redistributing nondistributable artifacts.
 
-#### Insecure registries
+### Insecure registries
 
 Docker considers a private registry either secure or insecure. In the rest of
 this section, *registry* is used for *private registry*, and `myregistry:5000`
@@ -1051,7 +1051,7 @@ because its use creates security vulnerabilities it should ONLY be enabled for
 testing purposes.  For increased security, users should add their CA to their
 system's list of trusted CAs instead of enabling `--insecure-registry`.
 
-##### Legacy Registries
+#### Legacy Registries
 
 Operations against registries supporting only the legacy v1 protocol are
 disabled by default. Specifically, the daemon will not attempt `push`,
@@ -1066,7 +1066,7 @@ registries that have not yet migrated to the v2 protocol.
 Interaction v1 registries will no longer be supported in Docker v17.12,
 and the `disable-legacy-registry` configuration option will be removed.
 
-#### Running a Docker daemon behind an HTTPS_PROXY
+### Running a Docker daemon behind an HTTPS_PROXY
 
 When running inside a LAN that uses an `HTTPS` proxy, the Docker Hub
 certificates will be replaced by the proxy's certificates. These certificates
@@ -1083,7 +1083,7 @@ This will only add the proxy and authentication to the Docker daemon's requests
 your `docker build`s and running containers will need extra configuration to
 use the proxy
 
-#### Default `ulimit` settings
+### Default `ulimit` settings
 
 `--default-ulimit` allows you to set the default `ulimit` options to use for
 all containers. It takes the same options as `--ulimit` for `docker run`. If
@@ -1095,7 +1095,7 @@ Be careful setting `nproc` with the `ulimit` flag as `nproc` is designed by Linu
 set the maximum number of processes available to a user, not to a container. For details
 please check the [run](run.md) reference.
 
-#### Node discovery
+### Node discovery
 
 The `--cluster-advertise` option specifies the `host:port` or `interface:port`
 combination that this particular daemon instance should use when advertising
@@ -1130,7 +1130,7 @@ The currently supported cluster store options are:
 | `kv.keyfile`          | Specifies the path to a local file with a PEM encoded private key. This private key is used as the client key for communication with the Key/Value store.                                                                     |
 | `kv.path`             | Specifies the path in the Key/Value store. If not configured, the default value is 'docker/nodes'.                                                                                                                            |
 
-#### Access authorization
+### Access authorization
 
 Docker's access authorization can be extended by authorization plugins that your
 organization can purchase or build themselves. You can install one or more
@@ -1155,7 +1155,7 @@ For information about how to create an authorization plugin, see [authorization
 plugin](../../extend/plugins_authorization.md) section in the Docker extend section of this documentation.
 
 
-#### Daemon user namespace options
+### Daemon user namespace options
 
 The Linux kernel
 [user namespace support](http://man7.org/linux/man-pages/man7/user_namespaces.7.html)
@@ -1237,7 +1237,7 @@ Please note that this feature is still marked as experimental as metrics and met
 names could change while this feature is still in experimental.  Please provide
 feedback on what you would like to see collected in the API.
 
-#### Daemon configuration file
+### Daemon configuration file
 
 The `--config-file` option allows you to set any configuration option
 for the daemon in a JSON format. This file uses the same flag names as keys,

components/cli/docs/reference/commandline/rmi.md

@@ -29,7 +29,7 @@ Options:
 ## Examples
 
 You can remove an image using its short or long ID, its tag, or its digest. If
-an image has one or more tag referencing it, you must remove all of them before
+an image has one or more tags referencing it, you must remove all of them before
 the image is removed. Digest references are removed automatically when an image
 is removed by tag.
 

components/cli/docs/reference/commandline/service_create.md

@@ -294,8 +294,9 @@ volumes in a service:
     <td>
       <p>The type of mount, can be either <tt>volume</tt>, <tt>bind</tt>, or <tt>tmpfs</tt>. Defaults to <tt>volume</tt> if no type is specified.
       <ul>
-        <li><tt>volume</tt>: mounts a [managed volume](volume_create.md) into the container.</li>
-        <li><tt>bind</tt>: bind-mounts a directory or file from the host into the container.</li>
+        <li><tt>volume</tt>: mounts a <a href="https://docs.docker.com/engine/reference/commandline/volume_create/">managed volume</a>
+        into the container.</li> <li><tt>bind</tt>:
+        bind-mounts a directory or file from the host into the container.</li>
         <li><tt>tmpfs</tt>: mount a tmpfs in the container</li>
       </ul></p>
     </td>

components/cli/docs/reference/commandline/stack_deploy.md

@@ -28,6 +28,8 @@ Options:
   -c, --compose-file string   Path to a Compose file
       --help                  Print usage
       --prune                 Prune services that are no longer referenced
+      --resolve-image string  Query the registry to resolve image digest and supported platforms
+                              ("always"|"changed"|"never") (default "always")
       --with-registry-auth    Send registry authentication details to Swarm agents
 ```
 

components/cli/e2e/compose-env.yaml

@@ -8,3 +8,14 @@ services:
       image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}'
       privileged: true
       command: ['--insecure-registry=registry:5000']
+
+  notary-server:
+      image: 'notary:server-0.4.2'
+      ports:
+        - 4443:4443
+      volumes:
+       - notary-fixtures:/fixtures
+      command: ['notary-server', '-config=/fixtures/notary-config.json']
+
+volumes:
+  notary-fixtures: {}

components/cli/e2e/image/main_test.go

@@ -0,0 +1,17 @@
+package image
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/cli/internal/test/environment"
+)
+
+func TestMain(m *testing.M) {
+	if err := environment.Setup(); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(3)
+	}
+	os.Exit(m.Run())
+}

components/cli/e2e/image/pull_test.go

@@ -0,0 +1,71 @@
+package image
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/gotestyourself/gotestyourself/golden"
+	"github.com/gotestyourself/gotestyourself/icmd"
+)
+
+const notaryURL = "https://notary-server:4443"
+const registryPrefix = "registry:5000"
+
+const alpineImage = "registry:5000/alpine:3.6"
+const busyboxImage = "registry:5000/busybox:1.27.2"
+
+func TestPullWithContentTrust(t *testing.T) {
+	image := createMaskedTrustedRemoteImage(t, "trust", "latest")
+
+	result := icmd.RunCmd(icmd.Command("docker", "pull", image), withTrustNoPassphrase)
+	result.Assert(t, icmd.Expected{Err: icmd.None})
+	golden.Assert(t, result.Stdout(), "pull-with-content-trust.golden")
+}
+
+// createMaskedTrustedRemoteImage creates a remote image that is signed with
+// content trust, then pushes a different untrusted image at the same tag.
+func createMaskedTrustedRemoteImage(t *testing.T, repo, tag string) string {
+	image := createTrustedRemoteImage(t, repo, tag)
+	createNamedUnsignedImageFromBusyBox(t, image)
+	return image
+}
+
+func createTrustedRemoteImage(t *testing.T, repo, tag string) string {
+	image := fmt.Sprintf("%s/%s:%s", registryPrefix, repo, tag)
+	icmd.RunCommand("docker", "pull", alpineImage).Assert(t, icmd.Success)
+	icmd.RunCommand("docker", "tag", alpineImage, image).Assert(t, icmd.Success)
+	result := icmd.RunCmd(
+		icmd.Command("docker", "push", image),
+		withTrustAndPassphrase("root_password", "repo_password"))
+	result.Assert(t, icmd.Success)
+	icmd.RunCommand("docker", "rmi", image).Assert(t, icmd.Success)
+	return image
+}
+
+func createNamedUnsignedImageFromBusyBox(t *testing.T, image string) {
+	icmd.RunCommand("docker", "pull", busyboxImage).Assert(t, icmd.Success)
+	icmd.RunCommand("docker", "tag", busyboxImage, image).Assert(t, icmd.Success)
+	icmd.RunCommand("docker", "push", image).Assert(t, icmd.Success)
+	icmd.RunCommand("docker", "rmi", image).Assert(t, icmd.Success)
+}
+
+func withTrustAndPassphrase(rootPwd, repositoryPwd string) func(cmd *icmd.Cmd) {
+	return func(cmd *icmd.Cmd) {
+		env := append(os.Environ(),
+			"DOCKER_CONTENT_TRUST=1",
+			"DOCKER_CONTENT_TRUST_SERVER="+notaryURL,
+			"DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE="+rootPwd,
+			"DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE="+repositoryPwd,
+		)
+		cmd.Env = append(cmd.Env, env...)
+	}
+}
+
+func withTrustNoPassphrase(cmd *icmd.Cmd) {
+	env := append(os.Environ(),
+		"DOCKER_CONTENT_TRUST=1",
+		"DOCKER_CONTENT_TRUST_SERVER="+notaryURL,
+	)
+	cmd.Env = append(cmd.Env, env...)
+}

components/cli/e2e/image/testdata/pull-with-content-trust.golden

@@ -0,0 +1,5 @@
+Pull (1 of 1): registry:5000/trust:latest@sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d
+sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d: Pulling from trust
+Digest: sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d
+Status: Downloaded newer image for registry:5000/trust@sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d
+Tagging registry:5000/trust@sha256:641b95ddb2ea9dc2af1a0113b6b348ebc20872ba615204fbe12148e98fd6f23d as registry:5000/trust:latest

components/cli/e2e/testdata/notary/notary-config.json

@@ -0,0 +1,19 @@
+{
+    "server": {
+        "http_addr": "notary-server:4443",
+        "tls_key_file": "./notary-server.key",
+        "tls_cert_file": "./notary-server.cert"
+    },
+    "trust_service": {
+        "type": "local",
+        "hostname": "",
+        "port": "",
+        "key_algorithm": "ed25519"
+    },
+    "logging": {
+        "level": "debug"
+    },
+    "storage": {
+        "backend": "memory"
+    }
+}

components/cli/e2e/testdata/notary/notary-server.cert

@@ -0,0 +1,63 @@
+-----BEGIN CERTIFICATE-----
+MIIFBDCCAuygAwIBAgIJAPlHYZzp1daGMA0GCSqGSIb3DQEBCwUAMGwxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEPMA0G
+A1UEChMGRG9ja2VyMScwJQYDVQQDEx5Ob3RhcnkgSW50ZXJtZWRpYXRlIFRlc3Rp
+bmcgQ0EwHhcNMTcwMjE3MDA0MzE3WhcNMTkwMzA5MDA0MzE3WjBbMQswCQYDVQQG
+EwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xDzANBgNV
+BAoTBkRvY2tlcjEWMBQGA1UEAxMNbm90YXJ5LXNlcnZlcjCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAKjbeflOtVrOv0IOeJGKfi5LHH3Di0O2nlZu8AIT
+SJbDZPSXoYc+cprpoEWYncbFFC3C94z5xBW5vcAqMhLs50ml5ADl86umcLl2C/mX
+8NuZnlIevMCb0mBiavDtSPV3J5DqOk+trgKEXs9g4hyh5Onh5Y5InPO1lDJ+2cEt
+VGBMhhddfWRVlV9ZUWxPYVCTt6L0bD9SeyXJVB0dnFhr3xICayhDlhlvcjXVOTUs
+ewJLo/L2nq0ve93Jb2smKio27ZGE79bCGqJK213/FNqfAlGUPkhYTfYJTcgjhS1p
+lmtgN6KZF6RVXvOrCBMEDM2yZq1mEPWjoT0tn0MkWErDANcCAwEAAaOBuTCBtjAf
+BgNVHSMEGDAWgBS6l0MbzVfv/9OdgJ2V2t/f3oOJ3TAMBgNVHRMBAf8EAjAAMB0G
+A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAOBgNVHQ8BAf8EBAMCBaAwNwYD
+VR0RBDAwLoINbm90YXJ5LXNlcnZlcoIMbm90YXJ5c2VydmVygglsb2NhbGhvc3SH
+BH8AAAEwHQYDVR0OBBYEFBQcColyhey0o0RTLiiGAtaRhUIuMA0GCSqGSIb3DQEB
+CwUAA4ICAQB04WZaMeF90mQDqiRVhBUkp8HvfEqchP6QLwprZmgbaRi75JksK59x
+ynaqgQj61hvN2RzpA1V/YXagmD6dk+GqhgiR+O++k+wb26446qQTSP6jkYRQGUT6
+s2Qp0fhgV9eHHZ/27Cl4rEpjYtxd6yVN/DNQapj/h3qejuZ1UDIZhvswfEgiL57f
+0W0huPNS6LnSOwoKKgSlA38OGs993BwMJkc+1ikzEcpVcn4l+kjeefnDmguBrxFv
+5il7yQ45BxGwR/SLobpehV+XodjNUd8mpdoF9QWr8kibaDPNndhdJLHuzyYatnRe
+hDqFA5DqZ+uaSwPyixilDoAXFs81P6UTkGh3EjP7rMbZNYnIHYFYIKpYVu23vbh+
+eriCw61YvEcIxqfvtIAVfbxwnXExQWGIDXgkJlfskHh/c4hQ1CWHCgqmO8Hvix1u
+OMfhB5LygX+4QANoKMkUUlKv2MC5HXQ7Bg6rCfPioju2nzGIbbUK043UnfJ2yXIh
+5g0bKGGdWMr5Qw0at8A2NvR6WvXm6+9gu94rNDGoIPn6umTmFjJCbGhjcyyjxg+k
+DO0uhoilX2OkvQHeaBwiy1WM2ETMQBKvkUfq6EUoLsWQTT2NOZiwwEMywwJCb853
+LuQjsvxfOFuqEgXEWjrEnhjwDCJFEDqaJAgajmBZ9xU+yUco44U9zQ==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIF1TCCA72gAwIBAgIJAKfYxoqVGl7nMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV
+BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEPMA0G
+A1UEChMGRG9ja2VyMRowGAYDVQQDExFOb3RhcnkgVGVzdGluZyBDQTAeFw0xNzAy
+MTcwMDQzMTdaFw0yNzAyMTUwMDQzMTdaMGwxCzAJBgNVBAYTAlVTMQswCQYDVQQI
+EwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEPMA0GA1UEChMGRG9ja2VyMScw
+JQYDVQQDEx5Ob3RhcnkgSW50ZXJtZWRpYXRlIFRlc3RpbmcgQ0EwggIiMA0GCSqG
+SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/VjDUa9JQLhfnHvonnF+yxsbwJSkFWbad
+ci5boMGakJGsjgjDK+IfzzkRNbA3aYSd27UX5Vz+nPyt3BLJHIhlxOW9iDA6nqWb
+q/eJn5eV42eXoR+6ttrNzFWLZOTT+v5ZGQJYOKmk4vmZT/xoHTgHlsRshpj3EFRd
+PxgolcKMSsZpSD8I2sgUYwh+rmI/nbesZGmb7mjQxMb2jtZPhxlHpSwL79RlFw4f
+cS0x7qts0WsZtY8pa3HWxSG0x4uWuNkwivojq0vpsFvynpLY6t7jdb0Vu7iUgMAA
+t8AsbCt+uTv4KhyJw5rD7kg+Ad55QqVuTqtwoz0+SBREHm67q8gn/skTQT3ro4pB
+nQdlAQDNPBa3JnZvXmLDkgHdVCKZtaarm92L0P5byIUo+UtDA4j+FkkRXLyVC1FU
+O3awwbAOIK2/VznRpaKoxEV7p2sp7pkqFIFAX1ALUXQRjuxd2EPuTn4HgMdIta6e
+xnZjTxcehUzpSxMBEnlfSNmYvSWD6enMDr0ZuRU1L5ZnLU6RibBtMREqfTaHAygf
+kPBRO6AHB+K5OPK5yVwzp4G8yve5FAUo0YnNTnaIYs7ZYL14eOzPH3RIkRE+F8hB
+A8YB5vFEAbjto/ZxeikvCVk6jNuuvw5mbHN9yHIfD7qI1pPY54EIEGgxVZGwaJTj
+mRKvTbf6JQIDAQABo4GGMIGDMB8GA1UdIwQYMBaAFMXa5XX+ba5Jf0YRuL6UqjtU
+QS5ZMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG
+AQUFBwMCMA4GA1UdDwEB/wQEAwIBRjAdBgNVHQ4EFgQUupdDG81X7//TnYCdldrf
+396Did0wDQYJKoZIhvcNAQELBQADggIBAEKzd2/OInjHpFNmqJnuAvrFJPQHhwAQ
+CUv4HwLqMdRbKLhAF2Xc687K0rJaW6ZpvvCWP/bsgzDi7bcti+sbpRtutL98ollq
+NtqEBW7pW3ljvQteBHiXCPzHmSdKBG5B972mFvIi1k9NpeHcEz4y9q5s9BBW5hIu
+6UwKIQSZ/dQGsukgQXe3lXJ/MjRg/QK0U1Xn/ABknm75vMbdc7L+WLvmrEY11NAy
+3vso6qm7k2elDLUVmqIwMU+r9pfGZFi0nQCxfiRhW1hZyDxzpngBZFnRppiO+2CX
+u16yiJfipD5wHZc1NjZ1WQ+XuoUhDgV1yzGpgRGtUdX4fByEUqNZb1GbY6UdIjmx
+LvwT2SJnEcv9irGlQU2D4mm5oXgeZXJhxP3gMxEtw9bZS3tUDMWkMD5GCdo8T+FB
+tjlGIe8Uh8R4wn33EYDMq+qFeUe+NiMDsVcFIlKWSwCRvmsTB3gvTYYkssuyS65Y
+8Ngdsy5GIBhMEsLnZNdwmP7NJ41CECqlojK9tQeu8KAhB7xsKibbwPUbec7whQFl
+82vOLgag7Fn3V1eVonqxk7uBt0nriV39/hXifPwBRo3WLoVLUTs+IAOiLapLAb6o
+LCQV7ymBRptOyj9bPPYV+NJrfr/5IlkNamioWlbzH3SJdvyp5ldZH/2vciItlO3s
+ALNmRewqFkXD
+-----END CERTIFICATE-----

components/cli/e2e/testdata/notary/notary-server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCo23n5TrVazr9C
+DniRin4uSxx9w4tDtp5WbvACE0iWw2T0l6GHPnKa6aBFmJ3GxRQtwveM+cQVub3A
+KjIS7OdJpeQA5fOrpnC5dgv5l/DbmZ5SHrzAm9JgYmrw7Uj1dyeQ6jpPra4ChF7P
+YOIcoeTp4eWOSJzztZQyftnBLVRgTIYXXX1kVZVfWVFsT2FQk7ei9Gw/UnslyVQd
+HZxYa98SAmsoQ5YZb3I11Tk1LHsCS6Py9p6tL3vdyW9rJioqNu2RhO/WwhqiSttd
+/xTanwJRlD5IWE32CU3II4UtaZZrYDeimRekVV7zqwgTBAzNsmatZhD1o6E9LZ9D
+JFhKwwDXAgMBAAECggEAbqa0PV0IlqMYze6xr53zpd5uozM61XqcM8Oq35FHZhRQ
+2b9riDax3zXtYu3pplGLMZmrouQhTKNU5tI/0gsQXUCqMrR9gyQkhkQHAN5CZYU7
+LFEcG5OAvsx/i7XSs5gLg3kaERCdEOUxQ/AW+/BTE7iGN0D6KPH6VUSu6VoNCrTK
+PmYvgta7hwebnvo65/OAc4inp9C19FUkhcNbaCKduWBgUt348+IzVEw9H8+PrdVZ
+dYGfVXAsDFY3zz0ThUbaZ52XS1pCCQ1Df9bQnTgqJNc+u1xQHLYAageKS83uAbtS
+nYjBFFuxeRR2FA1n8echCWQV+16Kqq31U1E2yLfWcQKBgQDSoT73pO9h/yN5myqu
+XxhAC+Ndas0DTl4pmVPpybpenJerba/0KCfYpcSFHAdkXZ1DYL7U+9uh5NRul09f
+WdjayFjn0vv63rwX+PGi1yPHTIv5kLvjYXJtaxzxSzQivYMPmD/7QX4tEsUkpJ8k
+90vMSS/C5ieWbpFwWVvEjFbqHQKBgQDNOsTq6USE3R8p4lj9Ror/waUuWVdzZnm3
+uZGJw3WzvzaXmqLSfVHntUnD8TPHgk3WZxDhrF73POMADkl9IN/JPI150/Uo6YJo
+qYGoZr0mmnEZxVCkwODz5C9icnyjklcRdIRM6eljhFMQDVEacDkptsntHUyIdQZc
+L2eLNUfEgwKBgHxy7UNg3lemag110rgIU8mzvHj7m3oymYw2nc/qcwVnvG17d5Tp
+DPICr6R+NRfl//9JcDdjQBfdnm5hVHJgIbLS4UTH8j390GDRo+O0/dzJq4KfM4Rb
+lUJ1ITqoVnuYQZG7QUJxJd330yedZLJwswZWz7N2TTmixqf9BC2TRd85AoGAN+Qh
+bLhKaMSvkACMq61ifXSHP7AlGNB3pYlsEVCh5WnVvEPow9pNTAUbKbmumE7sU8+N
+0WfYFQ0H5SP+74zcZTmQbfVDdvjhAw/mt64DJVg6JQKPi87bdJBYNz9mokVgYOiS
+fz/Ux71pwZ1e0QxvBOU66NBp31+/c6uVT1wbR3ECgYAdye1+UPpS9Dn89g6Ks0kv
+UaFKykXu7vY2uxiNqhmWzze4iq5wmIHmEwc6+rVMluXQPAME7Iya3mBmto9AHQ/n
+/ka+fGoaUgAojCLZW5DZcelIETw+Dk+95vyyAUsWfAvn4nKo4/rkBXcSHlvgElzq
+SorPiBWYosFB6jqUTXew2w==
+-----END PRIVATE KEY-----

components/cli/e2e/testdata/notary/root-ca.cert

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFhjCCA26gAwIBAgIJAI/HWUuNSUQjMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV
+BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEPMA0G
+A1UEChMGRG9ja2VyMRowGAYDVQQDExFOb3RhcnkgVGVzdGluZyBDQTAeFw0xNzAy
+MTcwMDQzMTNaFw0yNzAyMTUwMDQzMTNaMF8xCzAJBgNVBAYTAlVTMQswCQYDVQQI
+EwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEPMA0GA1UEChMGRG9ja2VyMRow
+GAYDVQQDExFOb3RhcnkgVGVzdGluZyBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIP
+ADCCAgoCggIBAPPHnaVuLXmF0fQ2LY9CSf2HjZKofabwjt7b6gH7132dcDqBzWbJ
+BTiRo0oze9LHV6P1AT4rvahM93SnWVpn5gHHnprMnFyG/PRpB0NjvkexSGFqUH/Q
+3B9xXkczh0BYGQquR56qCQr3oQKsu5vlIhcvQb6QrOB4Vm/AO9BtYicPcI6O2c5p
+ZQgh9Ih62JQQa97dDQc8/5JbC+WcXudPO2o+uyU9f1P0OpXh5AWc/N4HGIwJGDzJ
+i24U4R04jq0HQ1BMT0Q3EuGc0pPt9XxIzBj5qKtCQC2sChGZV8uLHVKMW3vgdpi8
+ZFbRjYkiSfiQ8+FIV4+2+MRF6jPm8VIrpqxaZHS6/SLfsCv0+bhXC/3CcxQYXLe0
+DR3D5JZAMyqVCaUVVJBi3tPqgv3GCG5VuKSe8gAww9SNDVT2PMQ2L1Z3PL3+09I1
+sed56ftC/zrZY3NYaD8f+9mOjR/yWyRM3cO7/TIe3riY/G1RVHAeL0HU/QVcWAZN
+CPJziKH+hMwEjIDFiMf8nY43EUn/hKx39oqPnLdw0aQRSfg/+052P15wTFSdjjhQ
+t9Z5ofw8vD4jaB9dXCry0iJ+kiaBDRS74awRCLKn7WwuXREveMcRJlYGooZskQLE
+EgMnMOuE0W2dw0hLsgImC3resdAd2UKnfdNO+5Wc7SuaxLsD3Th1B543AgMBAAGj
+RTBDMBIGA1UdEwEB/wQIMAYBAf8CAQEwDgYDVR0PAQH/BAQDAgFGMB0GA1UdDgQW
+BBTF2uV1/m2uSX9GEbi+lKo7VEEuWTANBgkqhkiG9w0BAQsFAAOCAgEAecm3BGLW
+igmsNIeO+Pn01v+EiPFQDBS4LkRiY/OH1lOEbskT8bHOVbKKBzcTI/0i8oRtn8CX
+faGYv0xRCfmM0ZKy73HD6FteObWiUessdLKbXFRc9p02QBvzU8rJ/yZyZjb2Bn7g
+KDylhmNmygQNvpy0TBCMA9l2pgokN5RD4zbY88DTrdYetkdBV70QFTUn3Za1Y3z0
+ZAigKyA9U1FUnRIgZprkliVJiBzv+JVD5uFIxut5nVoKaEuQ/EoM89BMJXEmlm2T
+LDmfJ2pSm4rjvt+V+jfR/f8lCwsqJ/DqnbxCbpCoMegJSoaSGvL0yWvfCTuiDzRk
+tAcz4v/bZ6mtH4pYJLQ1xswrDUW+3loCjB+9bU1185X7hZo2nkan409zqQ2gPWKR
+0qaVC1KnvsQaVupd7j4mYr/AzBNugR7PZpNKmBomLVZx/HLRAW7Qkz9wrXl3pcW7
+rXU5R3Z8NygRbzRadG2pXcmZqOTEM/3El5LOQs2bxb2/Qr7YAz957xEZBtZbRlMt
+lhWyA0PnlewJ2NeIBwf1WAw8lYXJMQnibCCHXsPh0A864F95QJAopVrsx0w+Junz
+C5rBWBS2H5c9cDA+BrIEV6SE94AvPs7OxEMCFDrqybZh/Q0xD4ADlm6EJoRvgtN/
+rba7O3YGSuScQakjt9mw2Q5ISwImkRHF3qE=
+-----END CERTIFICATE-----

components/cli/scripts/test/e2e/load-busybox

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+src=busybox@sha256:3e8fa85ddfef1af9ca85a5cfb714148956984e02f00bec3f7f49d3925a91e0e7
+dest=registry:5000/busybox:1.27.2
+docker pull $src
+docker tag $src $dest
+docker push $dest

components/cli/scripts/test/e2e/run

@@ -23,6 +23,7 @@ function setup {
         export DOCKER_HOST="$engine_host"
         timeout -t 200 ./scripts/test/e2e/wait-on-daemon
         ./scripts/test/e2e/load-alpine
+        ./scripts/test/e2e/load-busybox
         is_swarm_enabled || docker swarm init
     ) >&2
     echo "$engine_host"
@@ -33,12 +34,14 @@ function is_swarm_enabled {
 }
 
 function cleanup {
-    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down >&2
+    COMPOSE_PROJECT_NAME=$1 COMPOSE_FILE=$2 docker-compose down -v >&2
 }
 
 function runtests {
     local engine_host=$1
 
+    # TODO: only run if inside a container
+    update-ca-certificates
     # shellcheck disable=SC2086
     env -i \
         TEST_DOCKER_HOST="$engine_host" \
@@ -77,6 +80,9 @@ case "$cmd" in
         cleanup "$unique_id" "$compose_env_file"
         exit $testexit
         ;;
+    shell)
+        $SHELL
+        ;;
     *)
         echo "Unknown command: $cmd"
         echo "Usage: "

components/cli/scripts/test/e2e/wrapper

@@ -22,13 +22,31 @@ docker build \
     -t "$dev_image" \
     -f dockerfiles/Dockerfile.dev .
 
+notary_volume="${unique_id}_notary-fixtures"
+docker volume create --name "$notary_volume"
+docker run --rm \
+    -v "$PWD:/go/src/github.com/docker/cli" \
+    -v "$notary_volume:/data" \
+    "$dev_image" \
+    cp -r ./e2e/testdata/notary/* /data/
+
 engine_host=$(run_in_env setup)
 testexit=0
+
+
+test_cmd="test"
+if [[ -n "${TEST_DEBUG-}" ]]; then
+    test_cmd="shell"
+fi
+
 docker run -i --rm \
     -v "$PWD:/go/src/github.com/docker/cli" \
+    -v "$PWD/e2e/testdata/notary/root-ca.cert:/usr/local/share/ca-certificates/notary.cert" \
     --network "${unique_id}_default" \
     -e TESTFLAGS \
+    -e ENGINE_HOST="$engine_host" \
     "$dev_image" \
-    ./scripts/test/e2e/run test "$engine_host" || testexit="$?"
+    ./scripts/test/e2e/run "$test_cmd" "$engine_host" || testexit="$?"
+
 run_in_env cleanup
 exit "$testexit"

components/engine/VERSION

@@ -1 +1 @@
-17.06.0-dev
+17.10.0-ce-rc2

components/engine/daemon/graphdriver/overlay2/check.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"syscall"
 
 	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
@@ -15,10 +16,11 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-// hasOpaqueCopyUpBug checks whether the filesystem has a bug
+// doesSupportNativeDiff checks whether the filesystem has a bug
 // which copies up the opaque flag when copying up an opaque
-// directory. When this bug exists naive diff should be used.
-func hasOpaqueCopyUpBug(d string) error {
+// directory or the kernel enable CONFIG_OVERLAY_FS_REDIRECT_DIR.
+// When these exist naive diff should be used.
+func doesSupportNativeDiff(d string) error {
 	td, err := ioutil.TempDir(d, "opaque-bug-check")
 	if err != nil {
 		return err
@@ -29,10 +31,13 @@ func hasOpaqueCopyUpBug(d string) error {
 		}
 	}()
 
-	// Make directories l1/d, l2/d, l3, work, merged
+	// Make directories l1/d, l1/d1, l2/d, l3, work, merged
 	if err := os.MkdirAll(filepath.Join(td, "l1", "d"), 0755); err != nil {
 		return err
 	}
+	if err := os.MkdirAll(filepath.Join(td, "l1", "d1"), 0755); err != nil {
+		return err
+	}
 	if err := os.MkdirAll(filepath.Join(td, "l2", "d"), 0755); err != nil {
 		return err
 	}
@@ -75,5 +80,23 @@ func hasOpaqueCopyUpBug(d string) error {
 		return errors.New("opaque flag erroneously copied up, consider update to kernel 4.8 or later to fix")
 	}
 
+	// rename "d1" to "d2"
+	if err := os.Rename(filepath.Join(td, "merged", "d1"), filepath.Join(td, "merged", "d2")); err != nil {
+		// if rename failed with syscall.EXDEV, the kernel doesn't have CONFIG_OVERLAY_FS_REDIRECT_DIR enabled
+		if err.(*os.LinkError).Err == syscall.EXDEV {
+			return nil
+		}
+		return errors.Wrap(err, "failed to rename dir in merged directory")
+	}
+	// get the xattr of "d2"
+	xattrRedirect, err := system.Lgetxattr(filepath.Join(td, "l3", "d2"), "trusted.overlay.redirect")
+	if err != nil {
+		return errors.Wrap(err, "failed to read redirect flag on upper layer")
+	}
+
+	if string(xattrRedirect) == "d1" {
+		return errors.New("kernel has CONFIG_OVERLAY_FS_REDIRECT_DIR enabled")
+	}
+
 	return nil
 }

components/engine/daemon/graphdriver/overlay2/overlay.go

@@ -267,8 +267,8 @@ func supportsOverlay() error {
 
 func useNaiveDiff(home string) bool {
 	useNaiveDiffLock.Do(func() {
-		if err := hasOpaqueCopyUpBug(home); err != nil {
-			logrus.Warnf("Not using native diff for overlay2: %v", err)
+		if err := doesSupportNativeDiff(home); err != nil {
+			logrus.Warnf("Not using native diff for overlay2, this may cause degraded performance for building images: %v", err)
 			useNaiveDiffOnly = true
 		}
 	})

components/engine/distribution/pull_v2.go

@@ -708,29 +708,20 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
 	}
 
 	logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a os/arch match", ref, len(mfstList.Manifests))
-	var manifestDigest digest.Digest
-	// TODO @jhowardmsft LCOW Support: Need to remove the hard coding in LCOW mode.
-	lookingForOS := runtime.GOOS
-	if system.LCOWSupported() {
-		lookingForOS = "linux"
-	}
-	for _, manifestDescriptor := range mfstList.Manifests {
-		// TODO(aaronl): The manifest list spec supports optional
-		// "features" and "variant" fields. These are not yet used.
-		// Once they are, their values should be interpreted here.
-		if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == lookingForOS {
-			manifestDigest = manifestDescriptor.Digest
-			logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDigest.String())
-			break
-		}
-	}
 
-	if manifestDigest == "" {
+	manifestMatches := filterManifests(mfstList.Manifests)
+
+	if len(manifestMatches) == 0 {
 		errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", runtime.GOOS, runtime.GOARCH)
 		logrus.Debugf(errMsg)
 		return "", "", errors.New(errMsg)
 	}
 
+	if len(manifestMatches) > 1 {
+		logrus.Debugf("found multiple matches in manifest list, choosing best match %s", manifestMatches[0].Digest.String())
+	}
+	manifestDigest := manifestMatches[0].Digest
+
 	manSvc, err := p.repo.Manifests(ctx)
 	if err != nil {
 		return "", "", err

components/engine/distribution/pull_v2_unix.go

@@ -3,11 +3,27 @@
 package distribution
 
 import (
+	"runtime"
+
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/sirupsen/logrus"
 )
 
 func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekCloser, error) {
 	blobs := ld.repo.Blobs(ctx)
 	return blobs.Open(ctx, ld.digest)
 }
+
+func filterManifests(manifests []manifestlist.ManifestDescriptor) []manifestlist.ManifestDescriptor {
+	var matches []manifestlist.ManifestDescriptor
+	for _, manifestDescriptor := range manifests {
+		if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS {
+			matches = append(matches, manifestDescriptor)
+
+			logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String())
+		}
+	}
+	return matches
+}

components/engine/distribution/pull_v2_windows.go

@@ -3,13 +3,19 @@
 package distribution
 
 import (
+	"fmt"
 	"net/http"
 	"os"
+	"runtime"
+	"sort"
+	"strings"
 
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest/manifestlist"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/registry/client/transport"
+	"github.com/docker/docker/pkg/system"
 	"github.com/sirupsen/logrus"
 )
 
@@ -55,3 +61,54 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo
 	}
 	return rsc, err
 }
+
+func filterManifests(manifests []manifestlist.ManifestDescriptor) []manifestlist.ManifestDescriptor {
+	version := system.GetOSVersion()
+
+	// TODO @jhowardmsft LCOW Support: Need to remove the hard coding in LCOW mode.
+	lookingForOS := runtime.GOOS
+	osVersion := fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build)
+	if system.LCOWSupported() {
+		lookingForOS = "linux"
+		osVersion = ""
+	}
+
+	var matches []manifestlist.ManifestDescriptor
+	for _, manifestDescriptor := range manifests {
+		// TODO: Consider filtering out greater versions, including only greater UBR
+		if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == lookingForOS {
+			matches = append(matches, manifestDescriptor)
+
+			logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String())
+		}
+	}
+	if lookingForOS == "windows" {
+		sort.Stable(manifestsByVersion{osVersion, matches})
+	}
+	return matches
+}
+
+func versionMatch(actual, expected string) bool {
+	// Check whether the version matches up to the build, ignoring UBR
+	return strings.HasPrefix(actual, expected+".")
+}
+
+type manifestsByVersion struct {
+	version string
+	list    []manifestlist.ManifestDescriptor
+}
+
+func (mbv manifestsByVersion) Less(i, j int) bool {
+	// TODO: Split version by parts and compare
+	// TODO: Prefer versions which have a greater version number
+	// Move compatible versions to the top, with no other ordering changes
+	return versionMatch(mbv.list[i].Platform.OSVersion, mbv.version) && !versionMatch(mbv.list[j].Platform.OSVersion, mbv.version)
+}
+
+func (mbv manifestsByVersion) Len() int {
+	return len(mbv.list)
+}
+
+func (mbv manifestsByVersion) Swap(i, j int) {
+	mbv.list[i], mbv.list[j] = mbv.list[j], mbv.list[i]
+}

components/engine/integration-cli/docker_cli_build_test.go

@@ -4211,6 +4211,7 @@ func (s *DockerTrustSuite) TestBuildContextDirIsSymlink(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedBuildTagFromReleasesRole(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 
 	latestTag := s.setupTrustedImage(c, "trusted-build-releases-role")
@@ -4242,6 +4243,7 @@ func (s *DockerTrustSuite) TestTrustedBuildTagFromReleasesRole(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedBuildTagIgnoresOtherDelegationRoles(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 
 	latestTag := s.setupTrustedImage(c, "trusted-build-releases-role")

components/engine/integration-cli/docker_cli_cp_from_container_test.go

@@ -64,7 +64,7 @@ func (s *DockerSuite) TestCpFromErrDstParentNotExists(c *check.C) {
 	// Try with a file source.
 	srcPath := containerCpPath(containerID, "/file1")
 	dstPath := cpPath(tmpDir, "notExists", "file1")
-	_, dstStatErr := os.Lstat(filepath.Dir(dstPath))
+	_, dstStatErr := os.Stat(filepath.Dir(dstPath))
 	c.Assert(os.IsNotExist(dstStatErr), checker.True)
 
 	err := runDockerCp(c, srcPath, dstPath, nil)

components/engine/integration-cli/docker_cli_pull_trusted_test.go

@@ -133,6 +133,7 @@ func (s *DockerTrustSuite) TestTrustedPullDelete(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedPullReadsFromReleasesRole(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 	repoName := fmt.Sprintf("%v/dockerclireleasesdelegationpulling/trusted", privateRegistryURL)
 	targetName := fmt.Sprintf("%s:latest", repoName)
@@ -188,6 +189,7 @@ func (s *DockerTrustSuite) TestTrustedPullReadsFromReleasesRole(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedPullIgnoresOtherDelegationRoles(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 	repoName := fmt.Sprintf("%v/dockerclipullotherdelegation/trusted", privateRegistryURL)
 	targetName := fmt.Sprintf("%s:latest", repoName)

components/engine/integration-cli/docker_cli_push_test.go

@@ -282,6 +282,7 @@ func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c
 }
 
 func (s *DockerTrustSuite) TestTrustedPush(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	repoName := fmt.Sprintf("%v/dockerclitrusted/pushtest:latest", privateRegistryURL)
 	// tag the image and upload it to the private registry
 	cli.DockerCmd(c, "tag", "busybox", repoName)
@@ -366,6 +367,7 @@ func (s *DockerTrustSuite) TestTrustedPushWithExistingSignedTag(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedPushWithIncorrectPassphraseForNonRoot(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	repoName := fmt.Sprintf("%v/dockercliincorretpwd/trusted:latest", privateRegistryURL)
 	// tag the image and upload it to the private registry
 	cli.DockerCmd(c, "tag", "busybox", repoName)

components/engine/integration-cli/docker_cli_rmi_test.go

@@ -152,7 +152,7 @@ func (s *DockerSuite) TestRmiImageIDForceWithRunningContainersAndMultipleTags(c
 
 	out, _, err := dockerCmdWithError("rmi", "-f", imgID)
 	// rmi -f should not delete image with running containers
-	c.Assert(err, checker.NotNil)
+	c.Assert(err, checker.IsNil) // NOTE: rmi -f fails silently on a real error see: https://github.com/docker/cli/issues/394
 	c.Assert(out, checker.Contains, "(cannot be forced) - image is being used by running container")
 }
 
@@ -245,7 +245,7 @@ func (s *DockerSuite) TestRmiContainerImageNotFound(c *check.C) {
 	// Try to remove the image of the running container and see if it fails as expected.
 	out, _, err := dockerCmdWithError("rmi", "-f", imageIds[0])
 	// The image of the running container should not be removed.
-	c.Assert(err, checker.NotNil)
+	c.Assert(err, checker.IsNil) // NOTE: rmi -f fails silently on a real error see: https://github.com/docker/cli/issues/394
 	c.Assert(out, checker.Contains, "image is being used by running container", check.Commentf("out: %s", out))
 }
 

components/engine/integration-cli/docker_cli_service_scale_test.go

@@ -27,10 +27,10 @@ func (s *DockerSwarmSuite) TestServiceScale(c *check.C) {
 	out, err = d.Cmd(service2Args...)
 	c.Assert(err, checker.IsNil)
 
-	out, err = d.Cmd("service", "scale", "TestService1=2")
+	out, err = d.Cmd("service", "scale", "--detach", "TestService1=2")
 	c.Assert(err, checker.IsNil)
 
-	out, err = d.Cmd("service", "scale", "TestService1=foobar")
+	out, err = d.Cmd("service", "scale", "--detach", "TestService1=foobar")
 	c.Assert(err, checker.NotNil)
 
 	str := fmt.Sprintf("%s: invalid replicas value %s", service1Name, "foobar")
@@ -38,7 +38,7 @@ func (s *DockerSwarmSuite) TestServiceScale(c *check.C) {
 		c.Errorf("got: %s, expected has sub string: %s", out, str)
 	}
 
-	out, err = d.Cmd("service", "scale", "TestService1=-1")
+	out, err = d.Cmd("service", "scale", "--detach", "TestService1=-1")
 	c.Assert(err, checker.NotNil)
 
 	str = fmt.Sprintf("%s: invalid replicas value %s", service1Name, "-1")
@@ -47,7 +47,7 @@ func (s *DockerSwarmSuite) TestServiceScale(c *check.C) {
 	}
 
 	// TestService2 is a global mode
-	out, err = d.Cmd("service", "scale", "TestService2=2")
+	out, err = d.Cmd("service", "scale", "--detach", "TestService2=2")
 	c.Assert(err, checker.NotNil)
 
 	str = fmt.Sprintf("%s: scale can only be used with replicated mode\n", service2Name)

components/engine/integration-cli/docker_cli_stats_test.go

@@ -34,7 +34,7 @@ func (s *DockerSuite) TestStatsNoStream(c *check.C) {
 	select {
 	case outerr := <-ch:
 		c.Assert(outerr.err, checker.IsNil, check.Commentf("Error running stats: %v", outerr.err))
-		c.Assert(string(outerr.out), checker.Contains, id) //running container wasn't present in output
+		c.Assert(string(outerr.out), checker.Contains, id[:12]) //running container wasn't present in output
 	case <-time.After(3 * time.Second):
 		statsCmd.Process.Kill()
 		c.Fatalf("stats did not return immediately when not streaming")

components/engine/integration-cli/docker_cli_swarm_test.go

@@ -1545,7 +1545,8 @@ func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) {
 
 	out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo")
 	c.Assert(err, checker.IsNil, check.Commentf(out))
-	c.Assert(strings.TrimSpace(out), checker.Equals, "map[foo:bar]")
+	c.Assert(strings.TrimSpace(out), checker.Contains, "foo:bar")
+        c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true")
 
 	out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--network=foo", "--name", "top", "busybox", "top")
 	c.Assert(err, checker.IsNil, check.Commentf(out))
@@ -1555,7 +1556,8 @@ func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) {
 
 	out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo")
 	c.Assert(err, checker.IsNil, check.Commentf(out))
-	c.Assert(strings.TrimSpace(out), checker.Equals, "map[foo:bar]")
+	c.Assert(strings.TrimSpace(out), checker.Contains, "foo:bar")
+        c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true")
 }
 
 func (s *DockerTrustedSwarmSuite) TestTrustedServiceCreate(c *check.C) {
@@ -2033,7 +2035,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterEventsService(c *check.C) {
 
 	// scale service
 	t2 := daemonUnixTime(c)
-	out, err = d.Cmd("service", "scale", "test=3")
+	out, err = d.Cmd("service", "scale", "--detach", "test=3")
 	c.Assert(err, checker.IsNil, check.Commentf(out))
 
 	out = waitForEvent(c, d, t2, "-f scope=swarm", "service update "+serviceID, defaultRetryCount)

components/engine/integration-cli/docker_cli_update_unix_test.go

@@ -180,7 +180,7 @@ func GetKernelVersion() *kernel.VersionInfo {
 // CheckKernelVersion checks if current kernel is newer than (or equal to)
 // the given version.
 func CheckKernelVersion(k, major, minor int) bool {
-	return kernel.CompareKernelVersion(*GetKernelVersion(), kernel.VersionInfo{Kernel: k, Major: major, Minor: minor}) > 0
+	return kernel.CompareKernelVersion(*GetKernelVersion(), kernel.VersionInfo{Kernel: k, Major: major, Minor: minor}) >= 0
 }
 
 func (s *DockerSuite) TestUpdateSwapMemoryOnly(c *check.C) {

components/engine/vendor.conf

@@ -31,7 +31,7 @@ github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8
 github.com/tonistiigi/fsutil 1dedf6e90084bd88c4c518a15e68a37ed1370203
 
 #get libnetwork packages
-github.com/docker/libnetwork 0f08d31bf0e640e0cdc6d5161227f87602d605c5
+github.com/docker/libnetwork 6c512920fef411945513d04a50740ebbf13f2fd8
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -110,7 +110,7 @@ github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 github.com/stevvooe/continuity cd7a8e21e2b6f84799f5dd4b65faf49c8d3ee02d
 
 # cluster
-github.com/docker/swarmkit 941a01844b89c56aa61086fecb167ab3af1de22b
+github.com/docker/swarmkit 1d2bc2e202bb963c402938e09366f37564f62f46
 github.com/gogo/protobuf v0.4
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
 github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e

components/engine/vendor/github.com/docker/libnetwork/agent.go

@@ -6,11 +6,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
-	"os"
 	"sort"
 	"sync"
 
-	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-events"
 	"github.com/docker/libnetwork/cluster"
 	"github.com/docker/libnetwork/datastore"
@@ -282,12 +280,8 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, d
 	}
 
 	keys, _ := c.getKeys(subsysGossip)
-	hostname, _ := os.Hostname()
-	nodeName := hostname + "-" + stringid.TruncateID(stringid.GenerateRandomID())
-	logrus.Info("Gossip cluster hostname ", nodeName)
 
 	netDBConf := networkdb.DefaultConfig()
-	netDBConf.NodeName = nodeName
 	netDBConf.BindAddr = listenAddr
 	netDBConf.AdvertiseAddr = advertiseAddr
 	netDBConf.Keys = keys

components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go

@@ -41,6 +41,7 @@ type Handle struct {
 	id         string
 	dbIndex    uint64
 	dbExists   bool
+	curr       uint64
 	store      datastore.DataStore
 	sync.Mutex
 }
@@ -193,26 +194,27 @@ func (h *Handle) getCopy() *Handle {
 		dbIndex:    h.dbIndex,
 		dbExists:   h.dbExists,
 		store:      h.store,
+		curr:       h.curr,
 	}
 }
 
 // SetAnyInRange atomically sets the first unset bit in the specified range in the sequence and returns the corresponding ordinal
-func (h *Handle) SetAnyInRange(start, end uint64) (uint64, error) {
+func (h *Handle) SetAnyInRange(start, end uint64, serial bool) (uint64, error) {
 	if end < start || end >= h.bits {
 		return invalidPos, fmt.Errorf("invalid bit range [%d, %d]", start, end)
 	}
 	if h.Unselected() == 0 {
 		return invalidPos, ErrNoBitAvailable
 	}
-	return h.set(0, start, end, true, false)
+	return h.set(0, start, end, true, false, serial)
 }
 
 // SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal
-func (h *Handle) SetAny() (uint64, error) {
+func (h *Handle) SetAny(serial bool) (uint64, error) {
 	if h.Unselected() == 0 {
 		return invalidPos, ErrNoBitAvailable
 	}
-	return h.set(0, 0, h.bits-1, true, false)
+	return h.set(0, 0, h.bits-1, true, false, serial)
 }
 
 // Set atomically sets the corresponding bit in the sequence
@@ -220,7 +222,7 @@ func (h *Handle) Set(ordinal uint64) error {
 	if err := h.validateOrdinal(ordinal); err != nil {
 		return err
 	}
-	_, err := h.set(ordinal, 0, 0, false, false)
+	_, err := h.set(ordinal, 0, 0, false, false, false)
 	return err
 }
 
@@ -229,7 +231,7 @@ func (h *Handle) Unset(ordinal uint64) error {
 	if err := h.validateOrdinal(ordinal); err != nil {
 		return err
 	}
-	_, err := h.set(ordinal, 0, 0, false, true)
+	_, err := h.set(ordinal, 0, 0, false, true, false)
 	return err
 }
 
@@ -298,7 +300,7 @@ func (h *Handle) CheckConsistency() error {
 }
 
 // set/reset the bit
-func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64, error) {
+func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) {
 	var (
 		bitPos  uint64
 		bytePos uint64
@@ -308,6 +310,7 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64
 
 	for {
 		var store datastore.DataStore
+		curr := uint64(0)
 		h.Lock()
 		store = h.store
 		h.Unlock()
@@ -318,15 +321,18 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64
 		}
 
 		h.Lock()
+		if serial {
+			curr = h.curr
+		}
 		// Get position if available
 		if release {
 			bytePos, bitPos = ordinalToPos(ordinal)
 		} else {
 			if any {
-				bytePos, bitPos, err = getFirstAvailable(h.head, start)
+				bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end)
 				ret = posToOrdinal(bytePos, bitPos)
-				if end < ret {
-					err = ErrNoBitAvailable
+				if err == nil {
+					h.curr = ret + 1
 				}
 			} else {
 				bytePos, bitPos, err = checkIfAvailable(h.head, ordinal)
@@ -515,6 +521,29 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {
 	return invalidPos, invalidPos, ErrNoBitAvailable
 }
 
+// getAvailableFromCurrent will look for available ordinal from the current ordinal.
+// If none found then it will loop back to the start to check of the available bit.
+// This can be further optimized to check from start till curr in case of a rollover
+func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) {
+	var bytePos, bitPos uint64
+	if curr != 0 && curr > start {
+		bytePos, bitPos, _ = getFirstAvailable(head, curr)
+		ret := posToOrdinal(bytePos, bitPos)
+		if end < ret {
+			goto begin
+		}
+		return bytePos, bitPos, nil
+	}
+
+begin:
+	bytePos, bitPos, _ = getFirstAvailable(head, start)
+	ret := posToOrdinal(bytePos, bitPos)
+	if end < ret {
+		return invalidPos, invalidPos, ErrNoBitAvailable
+	}
+	return bytePos, bitPos, nil
+}
+
 // checkIfAvailable checks if the bit correspondent to the specified ordinal is unset
 // If the ordinal is beyond the sequence limits, a negative response is returned
 func checkIfAvailable(head *sequence, ordinal uint64) (uint64, uint64, error) {

components/engine/vendor/github.com/docker/libnetwork/bitseq/store.go

@@ -87,6 +87,7 @@ func (h *Handle) CopyTo(o datastore.KVObject) error {
 	dstH.dbIndex = h.dbIndex
 	dstH.dbExists = h.dbExists
 	dstH.store = h.store
+	dstH.curr = h.curr
 	dstH.Unlock()
 
 	return nil

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go

@@ -21,7 +21,6 @@ import (
 
 const (
 	r            = 0xD0C4E3
-	timeout      = 30
 	pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8)
 )
 

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go

@@ -68,7 +68,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	ep.ifName = containerIfName
 
-	if err := d.writeEndpointToStore(ep); err != nil {
+	if err = d.writeEndpointToStore(ep); err != nil {
 		return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)
 	}
 
@@ -86,7 +86,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return err
 	}
 
-	if err := sbox.AddInterface(overlayIfName, "veth",
+	if err = sbox.AddInterface(overlayIfName, "veth",
 		sbox.InterfaceOptions().Master(s.brName)); err != nil {
 		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
 	}
@@ -100,7 +100,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return err
 	}
 
-	if err := nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil {
+	if err = nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil {
 		return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err)
 	}
 
@@ -108,7 +108,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		if sub == s {
 			continue
 		}
-		if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {
+		if err = jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {
 			logrus.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id)
 		}
 	}
@@ -122,7 +122,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true)
 
-	if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {
+	if err = d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {
 		logrus.Warn(err)
 	}
 
@@ -200,7 +200,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri
 	}
 
 	if etype == driverapi.Delete {
-		d.peerDelete(nid, eid, addr.IP, addr.Mask, mac, vtep)
+		d.peerDelete(nid, eid, addr.IP, addr.Mask, mac, vtep, false)
 		return
 	}
 
@@ -232,11 +232,9 @@ func (d *driver) Leave(nid, eid string) error {
 		}
 	}
 
-	n.leaveSandbox()
+	d.peerDelete(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true)
 
-	if err := d.checkEncryption(nid, nil, 0, true, false); err != nil {
-		logrus.Warn(err)
-	}
+	n.leaveSandbox()
 
 	return nil
 }

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go

@@ -119,7 +119,7 @@ func setDefaultVlan() {
 	data := []byte{'0', '\n'}
 
 	if err = ioutil.WriteFile(path, data, 0644); err != nil {
-		logrus.Errorf("endbling default vlan on bridge %s failed %v", brName, err)
+		logrus.Errorf("enabling default vlan on bridge %s failed %v", brName, err)
 		os.Exit(1)
 	}
 	os.Exit(0)
@@ -251,8 +251,9 @@ func (d *driver) DeleteNetwork(nid string) error {
 		if err := d.deleteEndpointFromStore(ep); err != nil {
 			logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
 		}
-
 	}
+	// flush the peerDB entries
+	d.peerFlush(nid)
 	d.deleteNetwork(nid)
 
 	vnis, err := n.releaseVxlanID()
@@ -505,11 +506,7 @@ func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) erro
 	vxlanIfaceOption := make([]osl.IfaceOption, 1)
 	vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))
 	Ifaces[vxlanName+"+vxlan"] = vxlanIfaceOption
-	err = sbox.Restore(Ifaces, nil, nil, nil)
-	if err != nil {
-		return err
-	}
-	return nil
+	return sbox.Restore(Ifaces, nil, nil, nil)
 }
 
 func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {
@@ -760,58 +757,38 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
 				continue
 			}
 
-			logrus.Debugf("miss notification: dest IP %v, dest MAC %v", ip, mac)
-
 			if neigh.State&(netlink.NUD_STALE|netlink.NUD_INCOMPLETE) == 0 {
 				continue
 			}
 
 			if n.driver.isSerfAlive() {
+				logrus.Debugf("miss notification: dest IP %v, dest MAC %v", ip, mac)
 				mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip)
 				if err != nil {
 					logrus.Errorf("could not resolve peer %q: %v", ip, err)
 					continue
 				}
 				n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, l2Miss, l3Miss, false)
-			} else {
-				// If the gc_thresh values are lower kernel might knock off the neighor entries.
-				// When we get a L3 miss check if its a valid peer and reprogram the neighbor
-				// entry again. Rate limit it to once attempt every 500ms, just in case a faulty
-				// container sends a flood of packets to invalid peers
-				if !l3Miss {
-					continue
-				}
-				if time.Since(t) > 500*time.Millisecond {
+			} else if l3Miss && time.Since(t) > time.Second {
+				// All the local peers will trigger a miss notification but this one is expected and the local container will reply
+				// autonomously to the ARP request
+				// In case the gc_thresh3 values is low kernel might reject new entries during peerAdd. This will trigger the following
+				// extra logs that will inform of the possible issue.
+				// Entries created would not be deleted see documentation http://man7.org/linux/man-pages/man7/arp.7.html:
+				// Entries which are marked as permanent are never deleted by the garbage-collector.
+				// The time limit here is to guarantee that the dbSearch is not
+				// done too frequently causing a stall of the peerDB operations.
+				pKey, pEntry, err := n.driver.peerDbSearch(n.id, ip)
+				if err == nil && !pEntry.isLocal {
 					t = time.Now()
-					n.programNeighbor(ip)
+					logrus.Warnf("miss notification for peer:%+v l3Miss:%t l2Miss:%t, if the problem persist check the gc_thresh on the host pKey:%+v pEntry:%+v err:%v",
+						neigh, l3Miss, l2Miss, *pKey, *pEntry, err)
 				}
 			}
 		}
 	}
 }
 
-func (n *network) programNeighbor(ip net.IP) {
-	peerMac, _, _, err := n.driver.peerDbSearch(n.id, ip)
-	if err != nil {
-		logrus.Errorf("Reprogramming on L3 miss failed for %s, no peer entry", ip)
-		return
-	}
-	s := n.getSubnetforIPAddr(ip)
-	if s == nil {
-		logrus.Errorf("Reprogramming on L3 miss failed for %s, not a valid subnet", ip)
-		return
-	}
-	sbox := n.sandbox()
-	if sbox == nil {
-		logrus.Errorf("Reprogramming on L3 miss failed for %s, overlay sandbox missing", ip)
-		return
-	}
-	if err := sbox.AddNeighbor(ip, peerMac, true, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
-		logrus.Errorf("Reprogramming on L3 miss failed for %s: %v", ip, err)
-		return
-	}
-}
-
 func (d *driver) addNetwork(n *network) {
 	d.Lock()
 	d.networks[n.id] = n
@@ -1058,7 +1035,7 @@ func (n *network) obtainVxlanID(s *subnet) error {
 		}
 
 		if s.vni == 0 {
-			vxlanID, err := n.driver.vxlanIdm.GetID()
+			vxlanID, err := n.driver.vxlanIdm.GetID(true)
 			if err != nil {
 				return fmt.Errorf("failed to allocate vxlan id: %v", err)
 			}
@@ -1090,15 +1067,6 @@ func (n *network) contains(ip net.IP) bool {
 	return false
 }
 
-func (n *network) getSubnetforIPAddr(ip net.IP) *subnet {
-	for _, s := range n.subnets {
-		if s.subnetIP.Contains(ip) {
-			return s
-		}
-	}
-	return nil
-}
-
 // getSubnetforIP returns the subnet to which the given IP belongs
 func (n *network) getSubnetforIP(ip *net.IPNet) *subnet {
 	for _, s := range n.subnets {

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go

@@ -122,7 +122,7 @@ func (d *driver) processEvent(u serf.UserEvent) {
 	case "join":
 		d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), false, false, false)
 	case "leave":
-		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr))
+		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), false)
 	}
 }
 
@@ -135,13 +135,13 @@ func (d *driver) processQuery(q *serf.Query) {
 		fmt.Printf("Failed to scan query payload string: %v\n", err)
 	}
 
-	peerMac, peerIPMask, vtep, err := d.peerDbSearch(nid, net.ParseIP(ipStr))
+	pKey, pEntry, err := d.peerDbSearch(nid, net.ParseIP(ipStr))
 	if err != nil {
 		return
 	}
 
-	logrus.Debugf("Sending peer query resp mac %s, mask %s, vtep %s", peerMac, net.IP(peerIPMask), vtep)
-	q.Respond([]byte(fmt.Sprintf("%s %s %s", peerMac.String(), net.IP(peerIPMask).String(), vtep.String())))
+	logrus.Debugf("Sending peer query resp mac %v, mask %s, vtep %s", pKey.peerMac, net.IP(pEntry.peerIPMask).String(), pEntry.vtep)
+	q.Respond([]byte(fmt.Sprintf("%s %s %s", pKey.peerMac.String(), net.IP(pEntry.peerIPMask).String(), pEntry.vtep.String())))
 }
 
 func (d *driver) resolvePeer(nid string, peerIP net.IP) (net.HardwareAddr, net.IPMask, net.IP, error) {

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go

@@ -262,7 +262,7 @@ func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) {
 		d.Unlock()
 
 		// If containers are already running on this network update the
-		// advertiseaddress in the peerDB
+		// advertise address in the peerDB
 		d.localJoinOnce.Do(func() {
 			d.peerDBUpdateSelf()
 		})

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.proto

@@ -24,4 +24,4 @@ message PeerRecord {
 	// which this container is running and can be reached by
 	// building a tunnel to that host IP.
 	string tunnel_endpoint_ip = 3 [(gogoproto.customname) = "TunnelEndpointIP"];
-}
+}

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go

@@ -165,7 +165,7 @@ func (n *network) obtainVxlanID(s *subnet) error {
 	n.Unlock()
 
 	if vni == 0 {
-		vni, err = n.driver.vxlanIdm.GetIDInRange(vxlanIDStart, vxlanIDEnd)
+		vni, err = n.driver.vxlanIdm.GetIDInRange(vxlanIDStart, vxlanIDEnd, true)
 		if err != nil {
 			return err
 		}

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go

@@ -8,6 +8,7 @@ import (
 	"syscall"
 
 	"github.com/docker/libnetwork/common"
+	"github.com/docker/libnetwork/osl"
 	"github.com/sirupsen/logrus"
 )
 
@@ -22,16 +23,48 @@ type peerEntry struct {
 	eid        string
 	vtep       net.IP
 	peerIPMask net.IPMask
-	inSandbox  bool
 	isLocal    bool
 }
 
+func (p *peerEntry) MarshalDB() peerEntryDB {
+	ones, bits := p.peerIPMask.Size()
+	return peerEntryDB{
+		eid:            p.eid,
+		vtep:           p.vtep.String(),
+		peerIPMaskOnes: ones,
+		peerIPMaskBits: bits,
+		isLocal:        p.isLocal,
+	}
+}
+
+// This the structure saved into the set (SetMatrix), due to the implementation of it
+// the value inserted in the set has to be Hashable so the []byte had to be converted into
+// strings
+type peerEntryDB struct {
+	eid            string
+	vtep           string
+	peerIPMaskOnes int
+	peerIPMaskBits int
+	isLocal        bool
+}
+
+func (p *peerEntryDB) UnMarshalDB() peerEntry {
+	return peerEntry{
+		eid:        p.eid,
+		vtep:       net.ParseIP(p.vtep),
+		peerIPMask: net.CIDRMask(p.peerIPMaskOnes, p.peerIPMaskBits),
+		isLocal:    p.isLocal,
+	}
+}
+
 type peerMap struct {
-	mp map[string]peerEntry
+	// set of peerEntry, note they have to be objects and not pointers to maintain the proper equality checks
+	mp common.SetMatrix
 	sync.Mutex
 }
 
 type peerNetworkMap struct {
+	// map with key peerKey
 	mp map[string]*peerMap
 	sync.Mutex
 }
@@ -54,11 +87,7 @@ func (pKey *peerKey) Scan(state fmt.ScanState, verb rune) error {
 	}
 
 	pKey.peerMac, err = net.ParseMAC(string(macB))
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 func (d *driver) peerDbWalk(f func(string, *peerKey, *peerEntry) bool) error {
@@ -87,10 +116,13 @@ func (d *driver) peerDbNetworkWalk(nid string, f func(*peerKey, *peerEntry) bool
 	}
 
 	mp := map[string]peerEntry{}
-
 	pMap.Lock()
-	for pKeyStr, pEntry := range pMap.mp {
-		mp[pKeyStr] = pEntry
+	for _, pKeyStr := range pMap.mp.Keys() {
+		entryDBList, ok := pMap.mp.Get(pKeyStr)
+		if ok {
+			peerEntryDB := entryDBList[0].(peerEntryDB)
+			mp[pKeyStr] = peerEntryDB.UnMarshalDB()
+		}
 	}
 	pMap.Unlock()
 
@@ -107,45 +139,38 @@ func (d *driver) peerDbNetworkWalk(nid string, f func(*peerKey, *peerEntry) bool
 	return nil
 }
 
-func (d *driver) peerDbSearch(nid string, peerIP net.IP) (net.HardwareAddr, net.IPMask, net.IP, error) {
-	var (
-		peerMac    net.HardwareAddr
-		vtep       net.IP
-		peerIPMask net.IPMask
-		found      bool
-	)
-
+func (d *driver) peerDbSearch(nid string, peerIP net.IP) (*peerKey, *peerEntry, error) {
+	var pKeyMatched *peerKey
+	var pEntryMatched *peerEntry
 	err := d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool {
 		if pKey.peerIP.Equal(peerIP) {
-			peerMac = pKey.peerMac
-			peerIPMask = pEntry.peerIPMask
-			vtep = pEntry.vtep
-			found = true
-			return found
+			pKeyMatched = pKey
+			pEntryMatched = pEntry
+			return true
 		}
 
-		return found
+		return false
 	})
 
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("peerdb search for peer ip %q failed: %v", peerIP, err)
+		return nil, nil, fmt.Errorf("peerdb search for peer ip %q failed: %v", peerIP, err)
 	}
 
-	if !found {
-		return nil, nil, nil, fmt.Errorf("peer ip %q not found in peerdb", peerIP)
+	if pKeyMatched == nil || pEntryMatched == nil {
+		return nil, nil, fmt.Errorf("peer ip %q not found in peerdb", peerIP)
 	}
 
-	return peerMac, peerIPMask, vtep, nil
+	return pKeyMatched, pEntryMatched, nil
 }
 
 func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) {
+	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) (bool, int) {
 
 	d.peerDb.Lock()
 	pMap, ok := d.peerDb.mp[nid]
 	if !ok {
 		d.peerDb.mp[nid] = &peerMap{
-			mp: make(map[string]peerEntry),
+			mp: common.NewSetMatrix(),
 		}
 
 		pMap = d.peerDb.mp[nid]
@@ -165,18 +190,24 @@ func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask
 	}
 
 	pMap.Lock()
-	pMap.mp[pKey.String()] = pEntry
-	pMap.Unlock()
+	defer pMap.Unlock()
+	b, i := pMap.mp.Insert(pKey.String(), pEntry.MarshalDB())
+	if i != 1 {
+		// Transient case, there is more than one endpoint that is using the same IP,MAC pair
+		s, _ := pMap.mp.String(pKey.String())
+		logrus.Warnf("peerDbAdd transient condition - Key:%s cardinality:%d db state:%s", pKey.String(), i, s)
+	}
+	return b, i
 }
 
 func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP) peerEntry {
+	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) (bool, int) {
 
 	d.peerDb.Lock()
 	pMap, ok := d.peerDb.mp[nid]
 	if !ok {
 		d.peerDb.Unlock()
-		return peerEntry{}
+		return false, 0
 	}
 	d.peerDb.Unlock()
 
@@ -185,22 +216,22 @@ func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPM
 		peerMac: peerMac,
 	}
 
-	pMap.Lock()
-
-	pEntry, ok := pMap.mp[pKey.String()]
-	if ok {
-		// Mismatched endpoint ID(possibly outdated). Do not
-		// delete peerdb
-		if pEntry.eid != eid {
-			pMap.Unlock()
-			return pEntry
-		}
+	pEntry := peerEntry{
+		eid:        eid,
+		vtep:       vtep,
+		peerIPMask: peerIPMask,
+		isLocal:    isLocal,
 	}
 
-	delete(pMap.mp, pKey.String())
-	pMap.Unlock()
-
-	return pEntry
+	pMap.Lock()
+	defer pMap.Unlock()
+	b, i := pMap.mp.Remove(pKey.String(), pEntry.MarshalDB())
+	if i != 0 {
+		// Transient case, there is more than one endpoint that is using the same IP,MAC pair
+		s, _ := pMap.mp.String(pKey.String())
+		logrus.Warnf("peerDbDelete transient condition - Key:%s cardinality:%d db state:%s", pKey.String(), i, s)
+	}
+	return b, i
 }
 
 // The overlay uses a lazy initialization approach, this means that when a network is created
@@ -224,6 +255,7 @@ const (
 	peerOperationINIT peerOperationType = iota
 	peerOperationADD
 	peerOperationDELETE
+	peerOperationFLUSH
 )
 
 type peerOperation struct {
@@ -253,7 +285,9 @@ func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) {
 			case peerOperationADD:
 				err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.l2Miss, op.l3Miss, true, op.localPeer)
 			case peerOperationDELETE:
-				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP)
+				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.localPeer)
+			case peerOperationFLUSH:
+				err = d.peerFlushOp(op.networkID)
 			}
 			if err != nil {
 				logrus.Warnf("Peer operation failed:%s op:%v", err, op)
@@ -286,7 +320,6 @@ func (d *driver) peerInitOp(nid string) error {
 
 func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 	peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, localPeer bool) {
-	callerName := common.CallerName(1)
 	d.peerOpCh <- &peerOperation{
 		opType:     peerOperationADD,
 		networkID:  nid,
@@ -298,24 +331,32 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 		l2Miss:     l2Miss,
 		l3Miss:     l3Miss,
 		localPeer:  localPeer,
-		callerName: callerName,
+		callerName: common.CallerName(1),
 	}
 }
 
 func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, updateDB, updateOnlyDB bool) error {
+	peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, updateDB, localPeer bool) error {
 
 	if err := validateID(nid, eid); err != nil {
 		return err
 	}
 
+	var dbEntries int
+	var inserted bool
 	if updateDB {
-		d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, false)
-		if updateOnlyDB {
-			return nil
+		inserted, dbEntries = d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, localPeer)
+		if !inserted {
+			logrus.Warnf("Entry already present in db: nid:%s eid:%s peerIP:%v peerMac:%v isLocal:%t vtep:%v",
+				nid, eid, peerIP, peerMac, localPeer, vtep)
 		}
 	}
 
+	// Local peers do not need any further configuration
+	if localPeer {
+		return nil
+	}
+
 	n := d.network(nid)
 	if n == nil {
 		return nil
@@ -353,21 +394,26 @@ func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask
 
 	// Add neighbor entry for the peer IP
 	if err := sbox.AddNeighbor(peerIP, peerMac, l3Miss, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
-		return fmt.Errorf("could not add neighbor entry into the sandbox: %v", err)
+		if _, ok := err.(osl.NeighborSearchError); ok && dbEntries > 1 {
+			// We are in the transient case so only the first configuration is programmed into the kernel
+			// Upon deletion if the active configuration is deleted the next one from the database will be restored
+			// Note we are skipping also the next configuration
+			return nil
+		}
+		return fmt.Errorf("could not add neighbor entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)
 	}
 
 	// Add fdb entry to the bridge for the peer mac
 	if err := sbox.AddNeighbor(vtep, peerMac, l2Miss, sbox.NeighborOptions().LinkName(s.vxlanName),
 		sbox.NeighborOptions().Family(syscall.AF_BRIDGE)); err != nil {
-		return fmt.Errorf("could not add fdb entry into the sandbox: %v", err)
+		return fmt.Errorf("could not add fdb entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)
 	}
 
 	return nil
 }
 
 func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP) {
-	callerName := common.CallerName(1)
+	peerMac net.HardwareAddr, vtep net.IP, localPeer bool) {
 	d.peerOpCh <- &peerOperation{
 		opType:     peerOperationDELETE,
 		networkID:  nid,
@@ -376,18 +422,23 @@ func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMas
 		peerIPMask: peerIPMask,
 		peerMac:    peerMac,
 		vtepIP:     vtep,
-		callerName: callerName,
+		callerName: common.CallerName(1),
+		localPeer:  localPeer,
 	}
 }
 
 func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP) error {
+	peerMac net.HardwareAddr, vtep net.IP, localPeer bool) error {
 
 	if err := validateID(nid, eid); err != nil {
 		return err
 	}
 
-	pEntry := d.peerDbDelete(nid, eid, peerIP, peerIPMask, peerMac, vtep)
+	deleted, dbEntries := d.peerDbDelete(nid, eid, peerIP, peerIPMask, peerMac, vtep, localPeer)
+	if !deleted {
+		logrus.Warnf("Entry was not in db: nid:%s eid:%s peerIP:%v peerMac:%v isLocal:%t vtep:%v",
+			nid, eid, peerIP, peerMac, localPeer, vtep)
+	}
 
 	n := d.network(nid)
 	if n == nil {
@@ -399,30 +450,59 @@ func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPM
 		return nil
 	}
 
-	// Delete fdb entry to the bridge for the peer mac only if the
-	// entry existed in local peerdb. If it is a stale delete
-	// request, still call DeleteNeighbor but only to cleanup any
-	// leftover sandbox neighbor cache and not actually delete the
-	// kernel state.
-	if (eid == pEntry.eid && vtep.Equal(pEntry.vtep)) ||
-		(eid != pEntry.eid && !vtep.Equal(pEntry.vtep)) {
-		if err := sbox.DeleteNeighbor(vtep, peerMac,
-			eid == pEntry.eid && vtep.Equal(pEntry.vtep)); err != nil {
-			return fmt.Errorf("could not delete fdb entry into the sandbox: %v", err)
-		}
-	}
-
-	// Delete neighbor entry for the peer IP
-	if eid == pEntry.eid {
-		if err := sbox.DeleteNeighbor(peerIP, peerMac, true); err != nil {
-			return fmt.Errorf("could not delete neighbor entry into the sandbox: %v", err)
-		}
-	}
-
-	if err := d.checkEncryption(nid, vtep, 0, false, false); err != nil {
+	if err := d.checkEncryption(nid, vtep, 0, localPeer, false); err != nil {
 		logrus.Warn(err)
 	}
 
+	// Local peers do not have any local configuration to delete
+	if !localPeer {
+		// Remove fdb entry to the bridge for the peer mac
+		if err := sbox.DeleteNeighbor(vtep, peerMac, true); err != nil {
+			if _, ok := err.(osl.NeighborSearchError); ok && dbEntries > 0 {
+				// We fall in here if there is a transient state and if the neighbor that is being deleted
+				// was never been configured into the kernel (we allow only 1 configuration at the time per <ip,mac> mapping)
+				return nil
+			}
+			return fmt.Errorf("could not delete fdb entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)
+		}
+
+		// Delete neighbor entry for the peer IP
+		if err := sbox.DeleteNeighbor(peerIP, peerMac, true); err != nil {
+			return fmt.Errorf("could not delete neighbor entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)
+		}
+	}
+
+	if dbEntries == 0 {
+		return nil
+	}
+
+	// If there is still an entry into the database and the deletion went through without errors means that there is now no
+	// configuration active in the kernel.
+	// Restore one configuration for the <ip,mac> directly from the database, note that is guaranteed that there is one
+	peerKey, peerEntry, err := d.peerDbSearch(nid, peerIP)
+	if err != nil {
+		logrus.Errorf("peerDeleteOp unable to restore a configuration for nid:%s ip:%v mac:%v err:%s", nid, peerIP, peerMac, err)
+		return err
+	}
+	return d.peerAddOp(nid, peerEntry.eid, peerIP, peerEntry.peerIPMask, peerKey.peerMac, peerEntry.vtep, false, false, false, peerEntry.isLocal)
+}
+
+func (d *driver) peerFlush(nid string) {
+	d.peerOpCh <- &peerOperation{
+		opType:     peerOperationFLUSH,
+		networkID:  nid,
+		callerName: common.CallerName(1),
+	}
+}
+
+func (d *driver) peerFlushOp(nid string) error {
+	d.peerDb.Lock()
+	defer d.peerDb.Unlock()
+	_, ok := d.peerDb.mp[nid]
+	if !ok {
+		return fmt.Errorf("Unable to find the peerDB for nid:%s", nid)
+	}
+	delete(d.peerDb.mp, nid)
 	return nil
 }
 

components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go

@@ -718,7 +718,7 @@ func (n *network) obtainVxlanID(s *subnet) error {
 		}
 
 		if s.vni == 0 {
-			vxlanID, err := n.driver.vxlanIdm.GetID()
+			vxlanID, err := n.driver.vxlanIdm.GetID(true)
 			if err != nil {
 				return fmt.Errorf("failed to allocate vxlan id: %v", err)
 			}

components/engine/vendor/github.com/docker/libnetwork/idm/idm.go

@@ -34,11 +34,11 @@ func New(ds datastore.DataStore, id string, start, end uint64) (*Idm, error) {
 }
 
 // GetID returns the first available id in the set
-func (i *Idm) GetID() (uint64, error) {
+func (i *Idm) GetID(serial bool) (uint64, error) {
 	if i.handle == nil {
 		return 0, errors.New("ID set is not initialized")
 	}
-	ordinal, err := i.handle.SetAny()
+	ordinal, err := i.handle.SetAny(serial)
 	return i.start + ordinal, err
 }
 
@@ -56,7 +56,7 @@ func (i *Idm) GetSpecificID(id uint64) error {
 }
 
 // GetIDInRange returns the first available id in the set within a [start,end] range
-func (i *Idm) GetIDInRange(start, end uint64) (uint64, error) {
+func (i *Idm) GetIDInRange(start, end uint64, serial bool) (uint64, error) {
 	if i.handle == nil {
 		return 0, errors.New("ID set is not initialized")
 	}
@@ -65,7 +65,7 @@ func (i *Idm) GetIDInRange(start, end uint64) (uint64, error) {
 		return 0, errors.New("Requested range does not belong to the set")
 	}
 
-	ordinal, err := i.handle.SetAnyInRange(start-i.start, end-i.start)
+	ordinal, err := i.handle.SetAnyInRange(start-i.start, end-i.start, serial)
 
 	return i.start + ordinal, err
 }

components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go

@@ -457,7 +457,15 @@ func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[s
 		return nil, nil, types.InternalErrorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v",
 			k.String(), prefAddress, poolID, err)
 	}
-	ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range)
+	// In order to request for a serial ip address allocation, callers can pass in the option to request
+	// IP allocation serially or first available IP in the subnet
+	var serial bool
+	if opts != nil {
+		if val, ok := opts[ipamapi.AllocSerialPrefix]; ok {
+			serial = (val == "true")
+		}
+	}
+	ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range, serial)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -522,7 +530,7 @@ func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error {
 	return bm.Unset(ipToUint64(h))
 }
 
-func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange) (net.IP, error) {
+func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange, serial bool) (net.IP, error) {
 	var (
 		ordinal uint64
 		err     error
@@ -535,7 +543,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres
 		return nil, ipamapi.ErrNoAvailableIPs
 	}
 	if ipr == nil && prefAddress == nil {
-		ordinal, err = bitmask.SetAny()
+		ordinal, err = bitmask.SetAny(serial)
 	} else if prefAddress != nil {
 		hostPart, e := types.GetHostPartIP(prefAddress, base.Mask)
 		if e != nil {
@@ -544,7 +552,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres
 		ordinal = ipToUint64(types.GetMinimalIP(hostPart))
 		err = bitmask.Set(ordinal)
 	} else {
-		ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End)
+		ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End, serial)
 	}
 
 	switch err {

components/engine/vendor/github.com/docker/libnetwork/ipamapi/labels.go

@@ -0,0 +1,10 @@
+package ipamapi
+
+const (
+	// Prefix constant marks the reserved label space for libnetwork
+	Prefix = "com.docker.network"
+
+	// AllocSerialPrefix constant marks the reserved label space for libnetwork ipam
+	// allocation ordering.(serial/first available)
+	AllocSerialPrefix = Prefix + ".ipam.serial"
+)

components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go

@@ -456,7 +456,7 @@ func RawCombinedOutputNative(args ...string) error {
 
 // ExistChain checks if a chain exists
 func ExistChain(chain string, table Table) bool {
-	if _, err := Raw("-t", string(table), "-L", chain); err == nil {
+	if _, err := Raw("-t", string(table), "-nL", chain); err == nil {
 		return true
 	}
 	return false

components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go

@@ -32,7 +32,7 @@ func (nDB *NetworkDB) sendNetworkEvent(nid string, event NetworkEvent_Type, ltim
 	nEvent := NetworkEvent{
 		Type:      event,
 		LTime:     ltime,
-		NodeName:  nDB.config.NodeName,
+		NodeName:  nDB.config.NodeID,
 		NetworkID: nid,
 	}
 
@@ -44,7 +44,7 @@ func (nDB *NetworkDB) sendNetworkEvent(nid string, event NetworkEvent_Type, ltim
 	nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
 		msg:  raw,
 		id:   nid,
-		node: nDB.config.NodeName,
+		node: nDB.config.NodeID,
 	})
 	return nil
 }
@@ -72,7 +72,7 @@ func (nDB *NetworkDB) sendNodeEvent(event NodeEvent_Type) error {
 	nEvent := NodeEvent{
 		Type:     event,
 		LTime:    nDB.networkClock.Increment(),
-		NodeName: nDB.config.NodeName,
+		NodeName: nDB.config.NodeID,
 	}
 
 	raw, err := encodeMessage(MessageTypeNodeEvent, &nEvent)
@@ -129,7 +129,7 @@ func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname st
 	tEvent := TableEvent{
 		Type:      event,
 		LTime:     entry.ltime,
-		NodeName:  nDB.config.NodeName,
+		NodeName:  nDB.config.NodeID,
 		NetworkID: nid,
 		TableName: tname,
 		Key:       key,
@@ -145,7 +145,7 @@ func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname st
 
 	var broadcastQ *memberlist.TransmitLimitedQueue
 	nDB.RLock()
-	thisNodeNetworks, ok := nDB.networks[nDB.config.NodeName]
+	thisNodeNetworks, ok := nDB.networks[nDB.config.NodeID]
 	if ok {
 		// The network may have been removed
 		network, networkOk := thisNodeNetworks[nid]
@@ -168,7 +168,7 @@ func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname st
 		id:    nid,
 		tname: tname,
 		key:   key,
-		node:  nDB.config.NodeName,
+		node:  nDB.config.NodeID,
 	})
 	return nil
 }

components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go

@@ -106,7 +106,7 @@ func (nDB *NetworkDB) clusterInit() error {
 	nDB.lastHealthTimestamp = nDB.lastStatsTimestamp
 
 	config := memberlist.DefaultLANConfig()
-	config.Name = nDB.config.NodeName
+	config.Name = nDB.config.NodeID
 	config.BindAddr = nDB.config.BindAddr
 	config.AdvertiseAddr = nDB.config.AdvertiseAddr
 	config.UDPBufferSize = nDB.config.PacketBufferSize
@@ -329,7 +329,7 @@ func (nDB *NetworkDB) reapTableEntries() {
 	var nodeNetworks []string
 	// This is best effort, if the list of network changes will be picked up in the next cycle
 	nDB.RLock()
-	for nid := range nDB.networks[nDB.config.NodeName] {
+	for nid := range nDB.networks[nDB.config.NodeID] {
 		nodeNetworks = append(nodeNetworks, nid)
 	}
 	nDB.RUnlock()
@@ -376,7 +376,7 @@ func (nDB *NetworkDB) reapTableEntries() {
 func (nDB *NetworkDB) gossip() {
 	networkNodes := make(map[string][]string)
 	nDB.RLock()
-	thisNodeNetworks := nDB.networks[nDB.config.NodeName]
+	thisNodeNetworks := nDB.networks[nDB.config.NodeID]
 	for nid := range thisNodeNetworks {
 		networkNodes[nid] = nDB.networkNodes[nid]
 
@@ -388,7 +388,7 @@ func (nDB *NetworkDB) gossip() {
 	if printHealth {
 		healthScore := nDB.memberlist.GetHealthScore()
 		if healthScore != 0 {
-			logrus.Warnf("NetworkDB stats - healthscore:%d (connectivity issues)", healthScore)
+			logrus.Warnf("NetworkDB stats %v(%v) - healthscore:%d (connectivity issues)", nDB.config.Hostname, nDB.config.NodeID, healthScore)
 		}
 		nDB.lastHealthTimestamp = time.Now()
 	}
@@ -419,7 +419,8 @@ func (nDB *NetworkDB) gossip() {
 		// Collect stats and print the queue info, note this code is here also to have a view of the queues empty
 		network.qMessagesSent += len(msgs)
 		if printStats {
-			logrus.Infof("NetworkDB stats - netID:%s leaving:%t netPeers:%d entries:%d Queue qLen:%d netMsg/s:%d",
+			logrus.Infof("NetworkDB stats %v(%v) - netID:%s leaving:%t netPeers:%d entries:%d Queue qLen:%d netMsg/s:%d",
+				nDB.config.Hostname, nDB.config.NodeID,
 				nid, network.leaving, broadcastQ.NumNodes(), network.entriesNumber, broadcastQ.NumQueued(),
 				network.qMessagesSent/int((nDB.config.StatsPrintPeriod/time.Second)))
 			network.qMessagesSent = 0
@@ -456,7 +457,7 @@ func (nDB *NetworkDB) gossip() {
 func (nDB *NetworkDB) bulkSyncTables() {
 	var networks []string
 	nDB.RLock()
-	for nid, network := range nDB.networks[nDB.config.NodeName] {
+	for nid, network := range nDB.networks[nDB.config.NodeID] {
 		if network.leaving {
 			continue
 		}
@@ -522,10 +523,10 @@ func (nDB *NetworkDB) bulkSync(nodes []string, all bool) ([]string, error) {
 	var err error
 	var networks []string
 	for _, node := range nodes {
-		if node == nDB.config.NodeName {
+		if node == nDB.config.NodeID {
 			continue
 		}
-		logrus.Debugf("%s: Initiating bulk sync with node %v", nDB.config.NodeName, node)
+		logrus.Debugf("%v(%v): Initiating bulk sync with node %v", nDB.config.Hostname, nDB.config.NodeID, node)
 		networks = nDB.findCommonNetworks(node)
 		err = nDB.bulkSyncNode(networks, node, true)
 		// if its periodic bulksync stop after the first successful sync
@@ -556,7 +557,8 @@ func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited b
 		unsolMsg = "unsolicited"
 	}
 
-	logrus.Debugf("%s: Initiating %s bulk sync for networks %v with node %s", nDB.config.NodeName, unsolMsg, networks, node)
+	logrus.Debugf("%v(%v): Initiating %s bulk sync for networks %v with node %s",
+		nDB.config.Hostname, nDB.config.NodeID, unsolMsg, networks, node)
 
 	nDB.RLock()
 	mnode := nDB.nodes[node]
@@ -608,7 +610,7 @@ func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited b
 	bsm := BulkSyncMessage{
 		LTime:       nDB.tableClock.Time(),
 		Unsolicited: unsolicited,
-		NodeName:    nDB.config.NodeName,
+		NodeName:    nDB.config.NodeID,
 		Networks:    networks,
 		Payload:     compound,
 	}
@@ -640,7 +642,7 @@ func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited b
 		case <-t.C:
 			logrus.Errorf("Bulk sync to node %s timed out", node)
 		case <-ch:
-			logrus.Debugf("%s: Bulk sync to node %s took %s", nDB.config.NodeName, node, time.Since(startTime))
+			logrus.Debugf("%v(%v): Bulk sync to node %s took %s", nDB.config.Hostname, nDB.config.NodeID, node, time.Since(startTime))
 		}
 		t.Stop()
 	}
@@ -677,7 +679,7 @@ OUTER:
 		idx := randomOffset(n)
 		node := nodes[idx]
 
-		if node == nDB.config.NodeName {
+		if node == nDB.config.NodeID {
 			continue
 		}
 

components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go

@@ -2,7 +2,6 @@ package networkdb
 
 import (
 	"net"
-	"strings"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
@@ -58,29 +57,6 @@ func (nDB *NetworkDB) checkAndGetNode(nEvent *NodeEvent) *node {
 	return nil
 }
 
-func (nDB *NetworkDB) purgeSameNode(n *node) {
-	nDB.Lock()
-	defer nDB.Unlock()
-
-	prefix := strings.Split(n.Name, "-")[0]
-	for _, nodes := range []map[string]*node{
-		nDB.failedNodes,
-		nDB.leftNodes,
-		nDB.nodes,
-	} {
-		var nodeNames []string
-		for name, node := range nodes {
-			if strings.HasPrefix(name, prefix) && n.Addr.Equal(node.Addr) {
-				nodeNames = append(nodeNames, name)
-			}
-		}
-
-		for _, name := range nodeNames {
-			delete(nodes, name)
-		}
-	}
-}
-
 func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool {
 	// Update our local clock if the received messages has newer
 	// time.
@@ -108,7 +84,6 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool {
 		return false
 	}
 
-	nDB.purgeSameNode(n)
 	n.ltime = nEvent.LTime
 
 	switch nEvent.Type {
@@ -140,7 +115,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
 	nDB.Lock()
 	defer nDB.Unlock()
 
-	if nEvent.NodeName == nDB.config.NodeName {
+	if nEvent.NodeName == nDB.config.NodeID {
 		return false
 	}
 
@@ -203,7 +178,7 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
 
 	// Ignore the table events for networks that are in the process of going away
 	nDB.RLock()
-	networks := nDB.networks[nDB.config.NodeName]
+	networks := nDB.networks[nDB.config.NodeID]
 	network, ok := networks[tEvent.NetworkID]
 	// Check if the owner of the event is still part of the network
 	nodes := nDB.networkNodes[tEvent.NetworkID]
@@ -253,7 +228,8 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
 		// If it is a delete event and we did not have a state for it, don't propagate to the application
 		// If the residual reapTime is lower or equal to 1/6 of the total reapTime don't bother broadcasting it around
 		// most likely the cluster is already aware of it, if not who will sync with this node will catch the state too.
-		return e.reapTime > reapPeriod/6
+		// This also avoids that deletion of entries close to their garbage collection ends up circuling around forever
+		return e.reapTime > reapEntryInterval/6
 	}
 
 	var op opType
@@ -292,7 +268,7 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
 	}
 
 	// Ignore messages that this node generated.
-	if tEvent.NodeName == nDB.config.NodeName {
+	if tEvent.NodeName == nDB.config.NodeID {
 		return
 	}
 
@@ -305,7 +281,7 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
 		}
 
 		nDB.RLock()
-		n, ok := nDB.networks[nDB.config.NodeName][tEvent.NetworkID]
+		n, ok := nDB.networks[nDB.config.NodeID][tEvent.NetworkID]
 		nDB.RUnlock()
 
 		// if the network is not there anymore, OR we are leaving the network OR the broadcast queue is not present
@@ -424,7 +400,7 @@ func (nDB *NetworkDB) handleMessage(buf []byte, isBulkSync bool) {
 	case MessageTypeCompound:
 		nDB.handleCompound(data, isBulkSync)
 	default:
-		logrus.Errorf("%s: unknown message type %d", nDB.config.NodeName, mType)
+		logrus.Errorf("%v(%v): unknown message type %d", nDB.config.Hostname, nDB.config.NodeID, mType)
 	}
 }
 
@@ -457,7 +433,7 @@ func (d *delegate) LocalState(join bool) []byte {
 
 	pp := NetworkPushPull{
 		LTime:    d.nDB.networkClock.Time(),
-		NodeName: d.nDB.config.NodeName,
+		NodeName: d.nDB.config.NodeID,
 	}
 
 	for name, nn := range d.nDB.networks {

components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/armon/go-radix"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-events"
 	"github.com/docker/libnetwork/types"
 	"github.com/hashicorp/memberlist"
@@ -151,8 +152,11 @@ type network struct {
 // Config represents the configuration of the networdb instance and
 // can be passed by the caller.
 type Config struct {
-	// NodeName is the cluster wide unique name for this node.
-	NodeName string
+	// NodeID is the node unique identifier of the node when is part of the cluster
+	NodeID string
+
+	// Hostname is the node hostname.
+	Hostname string
 
 	// BindAddr is the IP on which networkdb listens. It can be
 	// 0.0.0.0 to listen on all addresses on the host.
@@ -210,7 +214,8 @@ type entry struct {
 func DefaultConfig() *Config {
 	hostname, _ := os.Hostname()
 	return &Config{
-		NodeName:          hostname,
+		NodeID:            stringid.TruncateID(stringid.GenerateRandomID()),
+		Hostname:          hostname,
 		BindAddr:          "0.0.0.0",
 		PacketBufferSize:  1400,
 		StatsPrintPeriod:  5 * time.Minute,
@@ -236,6 +241,7 @@ func New(c *Config) (*NetworkDB, error) {
 	nDB.indexes[byTable] = radix.New()
 	nDB.indexes[byNetwork] = radix.New()
 
+	logrus.Debugf("New memberlist node - Node:%v will use memberlist nodeID:%v", c.Hostname, c.NodeID)
 	if err := nDB.clusterInit(); err != nil {
 		return nil, err
 	}
@@ -259,8 +265,11 @@ func (nDB *NetworkDB) Join(members []string) error {
 // stopping timers, canceling goroutines etc.
 func (nDB *NetworkDB) Close() {
 	if err := nDB.clusterLeave(); err != nil {
-		logrus.Errorf("Could not close DB %s: %v", nDB.config.NodeName, err)
+		logrus.Errorf("%v(%v) Could not close DB: %v", nDB.config.Hostname, nDB.config.NodeID, err)
 	}
+
+	//Avoid (*Broadcaster).run goroutine leak
+	nDB.broadcaster.Close()
 }
 
 // ClusterPeers returns all the gossip cluster peers.
@@ -334,7 +343,7 @@ func (nDB *NetworkDB) CreateEntry(tname, nid, key string, value []byte) error {
 
 	entry := &entry{
 		ltime: nDB.tableClock.Increment(),
-		node:  nDB.config.NodeName,
+		node:  nDB.config.NodeID,
 		value: value,
 	}
 
@@ -360,7 +369,7 @@ func (nDB *NetworkDB) UpdateEntry(tname, nid, key string, value []byte) error {
 
 	entry := &entry{
 		ltime: nDB.tableClock.Increment(),
-		node:  nDB.config.NodeName,
+		node:  nDB.config.NodeID,
 		value: value,
 	}
 
@@ -402,7 +411,7 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error {
 
 	entry := &entry{
 		ltime:    nDB.tableClock.Increment(),
-		node:     nDB.config.NodeName,
+		node:     nDB.config.NodeID,
 		value:    value,
 		deleting: true,
 		reapTime: reapEntryInterval,
@@ -451,7 +460,7 @@ func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) {
 //		  entries owned by remote nodes, we will accept them and we notify the application
 func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) {
 	// Indicates if the delete is triggered for the local node
-	isNodeLocal := node == nDB.config.NodeName
+	isNodeLocal := node == nDB.config.NodeID
 
 	nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid),
 		func(path string, v interface{}) bool {
@@ -496,7 +505,10 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) {
 				nDB.deleteEntry(nid, tname, key)
 			}
 
-			nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value))
+			// Notify to the upper layer only entries not already marked for deletion
+			if !oldEntry.deleting {
+				nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value))
+			}
 			return false
 		})
 }
@@ -552,10 +564,10 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 	ltime := nDB.networkClock.Increment()
 
 	nDB.Lock()
-	nodeNetworks, ok := nDB.networks[nDB.config.NodeName]
+	nodeNetworks, ok := nDB.networks[nDB.config.NodeID]
 	if !ok {
 		nodeNetworks = make(map[string]*network)
-		nDB.networks[nDB.config.NodeName] = nodeNetworks
+		nDB.networks[nDB.config.NodeID] = nodeNetworks
 	}
 	n, ok := nodeNetworks[nid]
 	var entries int
@@ -571,8 +583,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 		},
 		RetransmitMult: 4,
 	}
-
-	nDB.addNetworkNode(nid, nDB.config.NodeName)
+	nDB.addNetworkNode(nid, nDB.config.NodeID)
 	networkNodes := nDB.networkNodes[nid]
 	nDB.Unlock()
 
@@ -580,7 +591,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error {
 		return fmt.Errorf("failed to send leave network event for %s: %v", nid, err)
 	}
 
-	logrus.Debugf("%s: joined network %s", nDB.config.NodeName, nid)
+	logrus.Debugf("%v(%v): joined network %s", nDB.config.Hostname, nDB.config.NodeID, nid)
 	if _, err := nDB.bulkSync(networkNodes, true); err != nil {
 		logrus.Errorf("Error bulk syncing while joining network %s: %v", nid, err)
 	}
@@ -604,12 +615,12 @@ func (nDB *NetworkDB) LeaveNetwork(nid string) error {
 	defer nDB.Unlock()
 
 	// Remove myself from the list of the nodes participating to the network
-	nDB.deleteNetworkNode(nid, nDB.config.NodeName)
+	nDB.deleteNetworkNode(nid, nDB.config.NodeID)
 
 	// Update all the local entries marking them for deletion and delete all the remote entries
-	nDB.deleteNodeNetworkEntries(nid, nDB.config.NodeName)
+	nDB.deleteNodeNetworkEntries(nid, nDB.config.NodeID)
 
-	nodeNetworks, ok := nDB.networks[nDB.config.NodeName]
+	nodeNetworks, ok := nDB.networks[nDB.config.NodeID]
 	if !ok {
 		return fmt.Errorf("could not find self node for network %s while trying to leave", nid)
 	}
@@ -619,7 +630,7 @@ func (nDB *NetworkDB) LeaveNetwork(nid string) error {
 		return fmt.Errorf("could not find network %s while trying to leave", nid)
 	}
 
-	logrus.Debugf("%s: leaving network %s", nDB.config.NodeName, nid)
+	logrus.Debugf("%v(%v): leaving network %s", nDB.config.Hostname, nDB.config.NodeID, nid)
 	n.ltime = ltime
 	n.reapTime = reapNetworkInterval
 	n.leaving = true
@@ -665,7 +676,7 @@ func (nDB *NetworkDB) findCommonNetworks(nodeName string) []string {
 	defer nDB.RUnlock()
 
 	var networks []string
-	for nid := range nDB.networks[nDB.config.NodeName] {
+	for nid := range nDB.networks[nDB.config.NodeID] {
 		if n, ok := nDB.networks[nodeName][nid]; ok {
 			if !n.leaving {
 				networks = append(networks, nid)
@@ -681,7 +692,7 @@ func (nDB *NetworkDB) updateLocalNetworkTime() {
 	defer nDB.Unlock()
 
 	ltime := nDB.networkClock.Increment()
-	for _, n := range nDB.networks[nDB.config.NodeName] {
+	for _, n := range nDB.networks[nDB.config.NodeID] {
 		n.ltime = ltime
 	}
 }
@@ -693,7 +704,7 @@ func (nDB *NetworkDB) createOrUpdateEntry(nid, tname, key string, entry interfac
 	_, okNetwork := nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry)
 	if !okNetwork {
 		// Add only if it is an insert not an update
-		n, ok := nDB.networks[nDB.config.NodeName][nid]
+		n, ok := nDB.networks[nDB.config.NodeID][nid]
 		if ok {
 			n.entriesNumber++
 		}
@@ -708,7 +719,7 @@ func (nDB *NetworkDB) deleteEntry(nid, tname, key string) (bool, bool) {
 	_, okNetwork := nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key))
 	if okNetwork {
 		// Remove only if the delete is successful
-		n, ok := nDB.networks[nDB.config.NodeName][nid]
+		n, ok := nDB.networks[nDB.config.NodeID][nid]
 		if ok {
 			n.entriesNumber--
 		}

components/engine/vendor/github.com/docker/libnetwork/osl/neigh_linux.go

@@ -9,6 +9,17 @@ import (
 	"github.com/vishvananda/netlink"
 )
 
+// NeighborSearchError indicates that the neighbor is already present
+type NeighborSearchError struct {
+	ip      net.IP
+	mac     net.HardwareAddr
+	present bool
+}
+
+func (n NeighborSearchError) Error() string {
+	return fmt.Sprintf("Search neighbor failed for IP %v, mac %v, present in db:%t", n.ip, n.mac, n.present)
+}
+
 // NeighOption is a function option type to set interface options
 type NeighOption func(nh *neigh)
 
@@ -41,7 +52,7 @@ func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr,
 
 	nh := n.findNeighbor(dstIP, dstMac)
 	if nh == nil {
-		return fmt.Errorf("could not find the neighbor entry to delete")
+		return NeighborSearchError{dstIP, dstMac, false}
 	}
 
 	if osDelete {
@@ -103,26 +114,27 @@ func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr,
 		}
 	}
 	n.Unlock()
-	logrus.Debugf("Neighbor entry deleted for IP %v, mac %v", dstIP, dstMac)
+	logrus.Debugf("Neighbor entry deleted for IP %v, mac %v osDelete:%t", dstIP, dstMac, osDelete)
 
 	return nil
 }
 
 func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, force bool, options ...NeighOption) error {
 	var (
-		iface netlink.Link
-		err   error
+		iface                  netlink.Link
+		err                    error
+		neighborAlreadyPresent bool
 	)
 
 	// If the namespace already has the neighbor entry but the AddNeighbor is called
 	// because of a miss notification (force flag) program the kernel anyway.
 	nh := n.findNeighbor(dstIP, dstMac)
 	if nh != nil {
+		neighborAlreadyPresent = true
+		logrus.Warnf("Neighbor entry already present for IP %v, mac %v neighbor:%+v forceUpdate:%t", dstIP, dstMac, nh, force)
 		if !force {
-			logrus.Warnf("Neighbor entry already present for IP %v, mac %v", dstIP, dstMac)
-			return nil
+			return NeighborSearchError{dstIP, dstMac, true}
 		}
-		logrus.Warnf("Force kernel update, Neighbor entry already present for IP %v, mac %v", dstIP, dstMac)
 	}
 
 	nh = &neigh{
@@ -146,8 +158,7 @@ func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, fo
 	if nh.linkDst != "" {
 		iface, err = nlh.LinkByName(nh.linkDst)
 		if err != nil {
-			return fmt.Errorf("could not find interface with destination name %s: %v",
-				nh.linkDst, err)
+			return fmt.Errorf("could not find interface with destination name %s: %v", nh.linkDst, err)
 		}
 	}
 
@@ -167,13 +178,17 @@ func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, fo
 	}
 
 	if err := nlh.NeighSet(nlnh); err != nil {
-		return fmt.Errorf("could not add neighbor entry: %v", err)
+		return fmt.Errorf("could not add neighbor entry:%+v error:%v", nlnh, err)
+	}
+
+	if neighborAlreadyPresent {
+		return nil
 	}
 
 	n.Lock()
 	n.neighbors = append(n.neighbors, nh)
 	n.Unlock()
-	logrus.Debugf("Neighbor entry added for IP %v, mac %v", dstIP, dstMac)
+	logrus.Debugf("Neighbor entry added for IP:%v, mac:%v on ifc:%s", dstIP, dstMac, nh.linkName)
 
 	return nil
 }

components/engine/vendor/github.com/docker/swarmkit/manager/allocator/cnmallocator/networkallocator.go

@@ -574,6 +574,7 @@ func (na *cnmNetworkAllocator) releaseEndpoints(networks []*api.NetworkAttachmen
 
 // allocate virtual IP for a single endpoint attachment of the service.
 func (na *cnmNetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error {
+	var opts map[string]string
 	localNet := na.getNetwork(vip.NetworkID)
 	if localNet == nil {
 		return errors.New("networkallocator: could not find local network state")
@@ -603,9 +604,13 @@ func (na *cnmNetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error {
 			return err
 		}
 	}
+	if localNet.nw.IPAM != nil && localNet.nw.IPAM.Driver != nil {
+		// set ipam allocation method to serial
+		opts = setIPAMSerialAlloc(localNet.nw.IPAM.Driver.Options)
+	}
 
 	for _, poolID := range localNet.pools {
-		ip, _, err := ipam.RequestAddress(poolID, addr, nil)
+		ip, _, err := ipam.RequestAddress(poolID, addr, opts)
 		if err != nil && err != ipamapi.ErrNoAvailableIPs && err != ipamapi.ErrIPOutOfRange {
 			return errors.Wrap(err, "could not allocate VIP from IPAM")
 		}
@@ -657,6 +662,7 @@ func (na *cnmNetworkAllocator) deallocateVIP(vip *api.Endpoint_VirtualIP) error
 // allocate the IP addresses for a single network attachment of the task.
 func (na *cnmNetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) error {
 	var ip *net.IPNet
+	var opts map[string]string
 
 	ipam, _, _, err := na.resolveIPAM(nAttach.Network)
 	if err != nil {
@@ -686,11 +692,16 @@ func (na *cnmNetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment
 				}
 			}
 		}
+		// Set the ipam options if the network has an ipam driver.
+		if localNet.nw.IPAM != nil && localNet.nw.IPAM.Driver != nil {
+			// set ipam allocation method to serial
+			opts = setIPAMSerialAlloc(localNet.nw.IPAM.Driver.Options)
+		}
 
 		for _, poolID := range localNet.pools {
 			var err error
 
-			ip, _, err = ipam.RequestAddress(poolID, addr, nil)
+			ip, _, err = ipam.RequestAddress(poolID, addr, opts)
 			if err != nil && err != ipamapi.ErrNoAvailableIPs && err != ipamapi.ErrIPOutOfRange {
 				return errors.Wrap(err, "could not allocate IP from IPAM")
 			}
@@ -918,8 +929,16 @@ func (na *cnmNetworkAllocator) allocatePools(n *api.Network) (map[string]string,
 			}
 			gwIP.IP = ip
 		}
+		if dOptions == nil {
+			dOptions = make(map[string]string)
+		}
+		dOptions[ipamapi.RequestAddressType] = netlabel.Gateway
+		// set ipam allocation method to serial
+		dOptions = setIPAMSerialAlloc(dOptions)
+		defer delete(dOptions, ipamapi.RequestAddressType)
+
 		if ic.Gateway != "" || gwIP == nil {
-			gwIP, _, err = ipam.RequestAddress(poolID, net.ParseIP(ic.Gateway), map[string]string{ipamapi.RequestAddressType: netlabel.Gateway})
+			gwIP, _, err = ipam.RequestAddress(poolID, net.ParseIP(ic.Gateway), dOptions)
 			if err != nil {
 				// Rollback by releasing all the resources allocated so far.
 				releasePools(ipam, ipamConfigs[:i], pools)
@@ -980,3 +999,14 @@ func IsBuiltInDriver(name string) bool {
 	}
 	return false
 }
+
+// setIPAMSerialAlloc sets the ipam allocation method to serial
+func setIPAMSerialAlloc(opts map[string]string) map[string]string {
+	if opts == nil {
+		opts = make(map[string]string)
+	}
+	if _, ok := opts[ipamapi.AllocSerialPrefix]; !ok {
+		opts[ipamapi.AllocSerialPrefix] = "true"
+	}
+	return opts
+}

components/engine/vendor/github.com/docker/swarmkit/manager/allocator/cnmallocator/portallocator.go

@@ -382,7 +382,7 @@ func (ps *portSpace) allocate(p *api.PortConfig) (err error) {
 	}
 
 	// Check out an arbitrary port from dynamic port space.
-	swarmPort, err := ps.dynamicPortSpace.GetID()
+	swarmPort, err := ps.dynamicPortSpace.GetID(true)
 	if err != nil {
 		return
 	}

components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go

@@ -169,7 +169,7 @@ func (f *PluginFilter) Check(n *NodeInfo) bool {
 		}
 	}
 
-	if f.t.Spec.LogDriver != nil {
+	if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" {
 		// If there are no log driver types in the list at all, most likely this is
 		// an older daemon that did not report this information. In this case don't filter
 		if typeFound, exists := f.pluginExistsOnNode("Log", f.t.Spec.LogDriver.Name, nodePlugins); !exists && typeFound {

components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go

@@ -180,9 +180,12 @@ type NodeOptions struct {
 	ClockSource clock.Clock
 	// SendTimeout is the timeout on the sending messages to other raft
 	// nodes. Leave this as 0 to get the default value.
-	SendTimeout    time.Duration
-	TLSCredentials credentials.TransportCredentials
-	KeyRotator     EncryptionKeyRotator
+	SendTimeout time.Duration
+	// LargeSendTimeout is the timeout on the sending snapshots to other raft
+	// nodes. Leave this as 0 to get the default value.
+	LargeSendTimeout time.Duration
+	TLSCredentials   credentials.TransportCredentials
+	KeyRotator       EncryptionKeyRotator
 	// DisableStackDump prevents Run from dumping goroutine stacks when the
 	// store becomes stuck.
 	DisableStackDump bool
@@ -204,6 +207,11 @@ func NewNode(opts NodeOptions) *Node {
 	if opts.SendTimeout == 0 {
 		opts.SendTimeout = 2 * time.Second
 	}
+	if opts.LargeSendTimeout == 0 {
+		// a "slow" 100Mbps connection can send over 240MB data in 20 seconds
+		// which is well over the gRPC message limit of 128MB allowed by SwarmKit
+		opts.LargeSendTimeout = 20 * time.Second
+	}
 
 	raftStore := raft.NewMemoryStorage()
 
@@ -349,6 +357,7 @@ func (n *Node) initTransport() {
 	transportConfig := &transport.Config{
 		HeartbeatInterval: time.Duration(n.Config.ElectionTick) * n.opts.TickInterval,
 		SendTimeout:       n.opts.SendTimeout,
+		LargeSendTimeout:  n.opts.LargeSendTimeout,
 		Credentials:       n.opts.TLSCredentials,
 		Raft:              n,
 	}
@@ -542,6 +551,7 @@ func (n *Node) Run(ctx context.Context) error {
 		n.done()
 	}()
 
+	// Flag that indicates if this manager node is *currently* the raft leader.
 	wasLeader := false
 	transferLeadershipLimit := rate.NewLimiter(rate.Every(time.Minute), 1)
 
@@ -563,10 +573,13 @@ func (n *Node) Run(ctx context.Context) error {
 				return errors.Wrap(err, "failed to save entries to storage")
 			}
 
+			// If the memory store lock has been held for too long,
+			// transferring leadership is an easy way to break out of it.
 			if wasLeader &&
 				(rd.SoftState == nil || rd.SoftState.RaftState == raft.StateLeader) &&
 				n.memoryStore.Wedged() &&
 				transferLeadershipLimit.Allow() {
+				log.G(ctx).Error("Attempting to transfer leadership")
 				if !n.opts.DisableStackDump {
 					signal.DumpStacks("")
 				}
@@ -612,6 +625,8 @@ func (n *Node) Run(ctx context.Context) error {
 			if rd.SoftState != nil {
 				if wasLeader && rd.SoftState.RaftState != raft.StateLeader {
 					wasLeader = false
+					log.G(ctx).Error("soft state changed, node no longer a leader, resetting and cancelling all waits")
+
 					if atomic.LoadUint32(&n.signalledLeadership) == 1 {
 						atomic.StoreUint32(&n.signalledLeadership, 0)
 						n.leadershipBroadcast.Publish(IsFollower)
@@ -630,6 +645,7 @@ func (n *Node) Run(ctx context.Context) error {
 					// cancelAll, or by its own check of signalledLeadership.
 					n.wait.cancelAll()
 				} else if !wasLeader && rd.SoftState.RaftState == raft.StateLeader {
+					// Node just became a leader.
 					wasLeader = true
 				}
 			}
@@ -1478,7 +1494,7 @@ func (n *Node) registerNode(node *api.RaftMember) error {
 	return nil
 }
 
-// ProposeValue calls Propose on the raft and waits
+// ProposeValue calls Propose on the underlying raft library(etcd/raft) and waits
 // on the commit log action before returning a result
 func (n *Node) ProposeValue(ctx context.Context, storeAction []api.StoreAction, cb func()) error {
 	ctx, cancel := n.WithContext(ctx)
@@ -1654,11 +1670,14 @@ func (n *Node) saveToStorage(
 	return nil
 }
 
-// processInternalRaftRequest sends a message to nodes participating
-// in the raft to apply a log entry and then waits for it to be applied
-// on the server. It will block until the update is performed, there is
-// an error or until the raft node finalizes all the proposals on node
-// shutdown.
+// processInternalRaftRequest proposes a value to be appended to the raft log.
+// It calls Propose() on etcd/raft, which calls back into the raft FSM,
+// which then sends a message to each of the participating nodes
+// in the raft group to apply a log entry and then waits for it to be applied
+// on this node. It will block until the this node:
+// 1. Gets the necessary replies back from the participating nodes and also performs the commit itself, or
+// 2. There is an error, or
+// 3. Until the raft node finalizes all the proposals on node shutdown.
 func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRaftRequest, cb func()) (proto.Message, error) {
 	n.stopMu.RLock()
 	if !n.IsMember() {
@@ -1679,6 +1698,7 @@ func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRa
 
 	// Do this check after calling register to avoid a race.
 	if atomic.LoadUint32(&n.signalledLeadership) != 1 {
+		log.G(ctx).Error("node is no longer leader, aborting propose")
 		n.wait.cancel(r.ID)
 		return nil, ErrLostLeadership
 	}
@@ -1703,14 +1723,23 @@ func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRa
 	select {
 	case x, ok := <-ch:
 		if !ok {
+			// Wait notification channel was closed. This should only happen if the wait was cancelled.
+			log.G(ctx).Error("wait cancelled")
+			if atomic.LoadUint32(&n.signalledLeadership) == 1 {
+				log.G(ctx).Error("wait cancelled but node is still a leader")
+			}
 			return nil, ErrLostLeadership
 		}
 		return x.(proto.Message), nil
 	case <-waitCtx.Done():
 		n.wait.cancel(r.ID)
-		// if channel is closed, wait item was canceled, otherwise it was triggered
+		// If we can read from the channel, wait item was triggered. Otherwise it was cancelled.
 		x, ok := <-ch
 		if !ok {
+			log.G(ctx).WithError(waitCtx.Err()).Error("wait context cancelled")
+			if atomic.LoadUint32(&n.signalledLeadership) == 1 {
+				log.G(ctx).Error("wait context cancelled but node is still a leader")
+			}
 			return nil, ErrLostLeadership
 		}
 		return x.(proto.Message), nil
@@ -1779,21 +1808,26 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error {
 	}
 
 	if !n.wait.trigger(r.ID, r) {
+		log.G(ctx).Errorf("wait not found for raft request id %x", r.ID)
+
 		// There was no wait on this ID, meaning we don't have a
 		// transaction in progress that would be committed to the
 		// memory store by the "trigger" call. Either a different node
 		// wrote this to raft, or we wrote it before losing the leader
-		// position and cancelling the transaction. Create a new
-		// transaction to commit the data.
+		// position and cancelling the transaction. This entry still needs
+		// to be committed since other nodes have already committed it.
+		// Create a new transaction to commit this entry.
 
 		// It should not be possible for processInternalRaftRequest
 		// to be running in this situation, but out of caution we
 		// cancel any current invocations to avoid a deadlock.
+		// TODO(anshul) This call is likely redundant, remove after consideration.
 		n.wait.cancelAll()
 
 		err := n.memoryStore.ApplyStoreActions(r.Action)
 		if err != nil {
 			log.G(ctx).WithError(err).Error("failed to apply actions from raft")
+			// TODO(anshul) return err here ?
 		}
 	}
 	return nil

components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go

@@ -133,7 +133,14 @@ func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) {
 }
 
 func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error {
-	ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout)
+	timeout := p.tr.config.SendTimeout
+	// if a snapshot is being sent, set timeout to LargeSendTimeout because
+	// sending snapshots can take more time than other messages sent between peers.
+	// The same applies to AppendEntries as well, where messages can get large.
+	if m.Type == raftpb.MsgSnap || m.Type == raftpb.MsgApp {
+		timeout = p.tr.config.LargeSendTimeout
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 	_, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
 	if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() {

components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go

@@ -35,6 +35,7 @@ type Raft interface {
 type Config struct {
 	HeartbeatInterval time.Duration
 	SendTimeout       time.Duration
+	LargeSendTimeout  time.Duration
 	Credentials       credentials.TransportCredentials
 	RaftID            string
 

components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go

@@ -83,8 +83,7 @@ func register(os ObjectStoreConfig) {
 	schema.Tables[os.Table.Name] = os.Table
 }
 
-// timedMutex wraps a sync.Mutex, and keeps track of how long it has been
-// locked.
+// timedMutex wraps a sync.Mutex, and keeps track of when it was locked.
 type timedMutex struct {
 	sync.Mutex
 	lockedAt atomic.Value

components/engine/vendor/github.com/docker/swarmkit/vendor.conf

@@ -24,7 +24,7 @@ github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/docker/go-units 954fed01cc617c55d838fa2230073f2cb17386c8
 github.com/docker/libkv 9fd56606e928ff1f309808f5d5a0b7a2ef73f9a8
-github.com/docker/libnetwork 19ac3ea7f52bb46e0eb10669756cdae0c441a5b1 
+github.com/docker/libnetwork 21544598c53fa36a3c771a8725c643dd2340f845 
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 github.com/opencontainers/runc d40db12e72a40109dfcf28539f5ee0930d2f0277
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 

components/packaging/deb/ubuntu-trusty/Dockerfile.armv7l

@@ -1,5 +1,7 @@
 FROM arm32v7/ubuntu:trusty
 
+# Temorary fix until ubuntu trusty package repositories are back up
+RUN sed -i 's|security.ubuntu.com/ubuntu|ports.ubuntu.com/ubuntu-ports|' /etc/apt/sources.list
 RUN apt-get update && apt-get install -y apparmor bash-completion btrfs-tools build-essential cmake curl ca-certificates debhelper dh-apparmor dh-systemd git libapparmor-dev libdevmapper-dev libltdl-dev  pkg-config vim-common libsystemd-journal-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 
 ENV GO_VERSION 1.8.3