Merge pull request #2048 from thaJeztah/19.03_backport_ci_improvements

[19.03 backport] CI and testing improvements

.dockerignore

@@ -1,2 +1,6 @@
+.dockerignore
 .git
-build
+.gitignore
+appveyor.yml
+build
+circle.yml

Jenkinsfile

@@ -5,8 +5,9 @@ wrappedNode(label: 'linux && x86_64', cleanWorkspace: true) {
 
     stage "Run end-to-end test suite"
     sh "docker version"
+    sh "docker info"
     sh "E2E_UNIQUE_ID=clie2e${BUILD_NUMBER} \
         IMAGE_TAG=clie2e${BUILD_NUMBER} \
-        make -f docker.Makefile test-e2e"
+        DOCKER_BUILDKIT=1 make -f docker.Makefile test-e2e"
   }
 }

appveyor.yml

@@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\docker\cli
 
 environment:
   GOPATH: c:\gopath
-  GOVERSION: 1.12.5
+  GOVERSION: 1.12.8
   DEPVERSION: v0.4.1
 
 install:

circle.yml

@@ -4,35 +4,39 @@ jobs:
 
   lint:
     working_directory: /work
-    docker: [{image: 'docker:18.03-git'}]
+    docker: [{image: 'docker:18.09-git'}]
+    environment:
+      DOCKER_BUILDKIT: 1
     steps:
       - checkout
       - setup_remote_docker:
-            version: 18.03.1-ce
-            reusable: true
-            exclusive: false
+          version: 18.09.3
+          reusable: true
+          exclusive: false
       - run:
           command: docker version
       - run:
           name: "Lint"
           command: |
-            docker build -f dockerfiles/Dockerfile.lint --tag cli-linter:$CIRCLE_BUILD_NUM .
+            docker build --progress=plain -f dockerfiles/Dockerfile.lint --tag cli-linter:$CIRCLE_BUILD_NUM .
             docker run --rm cli-linter:$CIRCLE_BUILD_NUM
 
   cross:
     working_directory: /work
-    docker: [{image: 'docker:18.03-git'}]
+    docker: [{image: 'docker:18.09-git'}]
+    environment:
+      DOCKER_BUILDKIT: 1
     parallelism: 3
     steps:
       - checkout
       - setup_remote_docker:
-            version: 18.03.1-ce
-            reusable: true
-            exclusive: false
+          version: 18.09.3
+          reusable: true
+          exclusive: false
       - run:
           name: "Cross"
           command: |
-            docker build -f dockerfiles/Dockerfile.cross --tag cli-builder:$CIRCLE_BUILD_NUM .
+            docker build --progress=plain -f dockerfiles/Dockerfile.cross --tag cli-builder:$CIRCLE_BUILD_NUM .
             name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
             docker run \
                 -e CROSS_GROUP=$CIRCLE_NODE_INDEX \
@@ -46,18 +50,20 @@ jobs:
 
   test:
     working_directory: /work
-    docker: [{image: 'docker:18.03-git'}]
+    docker: [{image: 'docker:18.09-git'}]
+    environment:
+      DOCKER_BUILDKIT: 1
     steps:
       - checkout
       - setup_remote_docker:
-            version: 18.03.1-ce
-            reusable: true
-            exclusive: false
+          version: 18.09.3
+          reusable: true
+          exclusive: false
       - run:
           name: "Unit Test with Coverage"
           command: |
             mkdir -p test-results/unit-tests
-            docker build -f dockerfiles/Dockerfile.dev --tag cli-builder:$CIRCLE_BUILD_NUM .
+            docker build --progress=plain -f dockerfiles/Dockerfile.dev --tag cli-builder:$CIRCLE_BUILD_NUM .
             docker run \
                 -e GOTESTSUM_JUNITFILE=/tmp/junit.xml \
                 --name \
@@ -82,34 +88,38 @@ jobs:
 
   validate:
     working_directory: /work
-    docker: [{image: 'docker:18.03-git'}]
+    docker: [{image: 'docker:18.09-git'}]
+    environment:
+      DOCKER_BUILDKIT: 1
     steps:
       - checkout
       - setup_remote_docker:
-            version: 18.03.1-ce
-            reusable: true
-            exclusive: false
+          version: 18.09.3
+          reusable: true
+          exclusive: false
       - run:
           name: "Validate Vendor, Docs, and Code Generation"
           command: |
             rm -f .dockerignore # include .git
-            docker build -f dockerfiles/Dockerfile.dev --tag cli-builder-with-git:$CIRCLE_BUILD_NUM .
+            docker build --progress=plain -f dockerfiles/Dockerfile.dev --tag cli-builder-with-git:$CIRCLE_BUILD_NUM .
             docker run --rm cli-builder-with-git:$CIRCLE_BUILD_NUM \
                 make ci-validate
           no_output_timeout: 15m
   shellcheck:
     working_directory: /work
-    docker: [{image: 'docker:18.03-git'}]
+    docker: [{image: 'docker:18.09-git'}]
+    environment:
+      DOCKER_BUILDKIT: 1
     steps:
       - checkout
       - setup_remote_docker:
-            version: 18.03.1-ce
-            reusable: true
-            exclusive: false
+          version: 18.09.3
+          reusable: true
+          exclusive: false
       - run:
           name: "Run shellcheck"
           command: |
-            docker build -f dockerfiles/Dockerfile.shellcheck --tag cli-validator:$CIRCLE_BUILD_NUM .
+            docker build --progress=plain -f dockerfiles/Dockerfile.shellcheck --tag cli-validator:$CIRCLE_BUILD_NUM .
             docker run --rm cli-validator:$CIRCLE_BUILD_NUM \
                 make shellcheck
 workflows:

cli/command/cli_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"runtime"
 	"testing"
@@ -79,6 +80,24 @@ func TestNewAPIClientFromFlagsWithAPIVersionFromEnv(t *testing.T) {
 	assert.Check(t, is.Equal(customVersion, apiclient.ClientVersion()))
 }
 
+func TestNewAPIClientFromFlagsWithHttpProxyEnv(t *testing.T) {
+	defer env.Patch(t, "HTTP_PROXY", "http://proxy.acme.com:1234")()
+	defer env.Patch(t, "DOCKER_HOST", "tcp://docker.acme.com:2376")()
+
+	opts := &flags.CommonOptions{}
+	configFile := &configfile.ConfigFile{}
+	apiclient, err := NewAPIClientFromFlags(opts, configFile)
+	assert.NilError(t, err)
+	transport, ok := apiclient.HTTPClient().Transport.(*http.Transport)
+	assert.Assert(t, ok)
+	assert.Assert(t, transport.Proxy != nil)
+	request, err := http.NewRequest(http.MethodGet, "tcp://docker.acme.com:2376", nil)
+	assert.NilError(t, err)
+	url, err := transport.Proxy(request)
+	assert.NilError(t, err)
+	assert.Check(t, is.Equal("http://proxy.acme.com:1234", url.String()))
+}
+
 type fakeClient struct {
 	client.Client
 	pingFunc   func() (types.Ping, error)

cli/command/context/create.go

@@ -78,13 +78,19 @@ func RunCreate(cli command.Cli, o *CreateOptions) error {
 	if err != nil {
 		return errors.Wrap(err, "unable to parse default-stack-orchestrator")
 	}
-	if o.From == "" && o.Docker == nil && o.Kubernetes == nil {
-		return createFromExistingContext(s, cli.CurrentContext(), stackOrchestrator, o)
+	switch {
+	case o.From == "" && o.Docker == nil && o.Kubernetes == nil:
+		err = createFromExistingContext(s, cli.CurrentContext(), stackOrchestrator, o)
+	case o.From != "":
+		err = createFromExistingContext(s, o.From, stackOrchestrator, o)
+	default:
+		err = createNewContext(o, stackOrchestrator, cli, s)
 	}
-	if o.From != "" {
-		return createFromExistingContext(s, o.From, stackOrchestrator, o)
+	if err == nil {
+		fmt.Fprintln(cli.Out(), o.Name)
+		fmt.Fprintf(cli.Err(), "Successfully created context %q\n", o.Name)
 	}
-	return createNewContext(o, stackOrchestrator, cli, s)
+	return err
 }
 
 func createNewContext(o *CreateOptions, stackOrchestrator command.Orchestrator, cli command.Cli, s store.Writer) error {
@@ -127,8 +133,6 @@ func createNewContext(o *CreateOptions, stackOrchestrator command.Orchestrator,
 	if err := s.ResetTLSMaterial(o.Name, &contextTLSData); err != nil {
 		return err
 	}
-	fmt.Fprintln(cli.Out(), o.Name)
-	fmt.Fprintf(cli.Err(), "Successfully created context %q\n", o.Name)
 	return nil
 }
 

cli/command/context/create_test.go

@@ -1,6 +1,7 @@
 package context
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -131,6 +132,11 @@ func TestCreateInvalids(t *testing.T) {
 	}
 }
 
+func assertContextCreateLogging(t *testing.T, cli *test.FakeCli, n string) {
+	assert.Equal(t, n+"\n", cli.OutBuffer().String())
+	assert.Equal(t, fmt.Sprintf("Successfully created context %q\n", n), cli.ErrBuffer().String())
+}
+
 func TestCreateOrchestratorSwarm(t *testing.T) {
 	cli, cleanup := makeFakeCli(t)
 	defer cleanup()
@@ -141,8 +147,7 @@ func TestCreateOrchestratorSwarm(t *testing.T) {
 		Docker:                   map[string]string{},
 	})
 	assert.NilError(t, err)
-	assert.Equal(t, "test\n", cli.OutBuffer().String())
-	assert.Equal(t, "Successfully created context \"test\"\n", cli.ErrBuffer().String())
+	assertContextCreateLogging(t, cli, "test")
 }
 
 func TestCreateOrchestratorEmpty(t *testing.T) {
@@ -154,6 +159,7 @@ func TestCreateOrchestratorEmpty(t *testing.T) {
 		Docker: map[string]string{},
 	})
 	assert.NilError(t, err)
+	assertContextCreateLogging(t, cli, "test")
 }
 
 func validateTestKubeEndpoint(t *testing.T, s store.Reader, name string) {
@@ -189,6 +195,7 @@ func TestCreateOrchestratorAllKubernetesEndpointFromCurrent(t *testing.T) {
 	cli, cleanup := makeFakeCli(t)
 	defer cleanup()
 	createTestContextWithKube(t, cli)
+	assertContextCreateLogging(t, cli, "test")
 	validateTestKubeEndpoint(t, cli.ContextStore(), "test")
 }
 
@@ -225,6 +232,7 @@ func TestCreateFromContext(t *testing.T) {
 	defer cleanup()
 	revert := env.Patch(t, "KUBECONFIG", "./testdata/test-kubeconfig")
 	defer revert()
+	cli.ResetOutputBuffers()
 	assert.NilError(t, RunCreate(cli, &CreateOptions{
 		Name:        "original",
 		Description: "original description",
@@ -236,6 +244,9 @@ func TestCreateFromContext(t *testing.T) {
 		},
 		DefaultStackOrchestrator: "swarm",
 	}))
+	assertContextCreateLogging(t, cli, "original")
+
+	cli.ResetOutputBuffers()
 	assert.NilError(t, RunCreate(cli, &CreateOptions{
 		Name:        "dummy",
 		Description: "dummy description",
@@ -247,11 +258,13 @@ func TestCreateFromContext(t *testing.T) {
 		},
 		DefaultStackOrchestrator: "swarm",
 	}))
+	assertContextCreateLogging(t, cli, "dummy")
 
 	cli.SetCurrentContext("dummy")
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
+			cli.ResetOutputBuffers()
 			err := RunCreate(cli, &CreateOptions{
 				From:                     "original",
 				Name:                     c.name,
@@ -261,6 +274,7 @@ func TestCreateFromContext(t *testing.T) {
 				Kubernetes:               c.kubernetes,
 			})
 			assert.NilError(t, err)
+			assertContextCreateLogging(t, cli, c.name)
 			newContext, err := cli.ContextStore().GetMetadata(c.name)
 			assert.NilError(t, err)
 			newContextTyped, err := command.GetDockerContext(newContext)
@@ -308,6 +322,7 @@ func TestCreateFromCurrent(t *testing.T) {
 	defer cleanup()
 	revert := env.Patch(t, "KUBECONFIG", "./testdata/test-kubeconfig")
 	defer revert()
+	cli.ResetOutputBuffers()
 	assert.NilError(t, RunCreate(cli, &CreateOptions{
 		Name:        "original",
 		Description: "original description",
@@ -319,17 +334,20 @@ func TestCreateFromCurrent(t *testing.T) {
 		},
 		DefaultStackOrchestrator: "swarm",
 	}))
+	assertContextCreateLogging(t, cli, "original")
 
 	cli.SetCurrentContext("original")
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
+			cli.ResetOutputBuffers()
 			err := RunCreate(cli, &CreateOptions{
 				Name:                     c.name,
 				Description:              c.description,
 				DefaultStackOrchestrator: c.orchestrator,
 			})
 			assert.NilError(t, err)
+			assertContextCreateLogging(t, cli, c.name)
 			newContext, err := cli.ContextStore().GetMetadata(c.name)
 			assert.NilError(t, err)
 			newContextTyped, err := command.GetDockerContext(newContext)

cli/command/trust/key_load.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"runtime"
 
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
@@ -69,12 +70,14 @@ func loadPrivKey(streams command.Streams, keyPath string, options keyLoadOptions
 }
 
 func getPrivKeyBytesFromPath(keyPath string) ([]byte, error) {
-	fileInfo, err := os.Stat(keyPath)
-	if err != nil {
-		return nil, err
-	}
-	if fileInfo.Mode()&nonOwnerReadWriteMask != 0 {
-		return nil, fmt.Errorf("private key file %s must not be readable or writable by others", keyPath)
+	if runtime.GOOS != "windows" {
+		fileInfo, err := os.Stat(keyPath)
+		if err != nil {
+			return nil, err
+		}
+		if fileInfo.Mode()&nonOwnerReadWriteMask != 0 {
+			return nil, fmt.Errorf("private key file %s must not be readable or writable by others", keyPath)
+		}
 	}
 
 	from, err := os.OpenFile(keyPath, os.O_RDONLY, notary.PrivExecPerms)

cli/compose/loader/loader.go

@@ -479,12 +479,13 @@ func resolveVolumePaths(volumes []types.ServiceVolumeConfig, workingDir string,
 		}
 
 		filePath := expandUser(volume.Source, lookupEnv)
-		// Check for a Unix absolute path first, to handle a Windows client
-		// with a Unix daemon. This handles a Windows client connecting to a
-		// Unix daemon. Note that this is not required for Docker for Windows
-		// when specifying a local Windows path, because Docker for Windows
-		// translates the Windows path into a valid path within the VM.
-		if !path.IsAbs(filePath) {
+		// Check if source is an absolute path (either Unix or Windows), to
+		// handle a Windows client with a Unix daemon or vice-versa.
+		//
+		// Note that this is not required for Docker for Windows when specifying
+		// a local Windows path, because Docker for Windows translates the Windows
+		// path into a valid path within the VM.
+		if !path.IsAbs(filePath) && !isAbs(filePath) {
 			filePath = absPath(workingDir, filePath)
 		}
 		volume.Source = filePath

cli/compose/loader/loader_test.go

@@ -985,6 +985,84 @@ services:
 	assert.Error(t, err, `invalid mount config for type "bind": field Source must not be empty`)
 }
 
+func TestLoadBindMountSourceIsWindowsAbsolute(t *testing.T) {
+	tests := []struct {
+		doc      string
+		yaml     string
+		expected types.ServiceVolumeConfig
+	}{
+		{
+			doc: "Z-drive lowercase",
+			yaml: `
+version: '3.3'
+
+services:
+  windows:
+    image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019
+    volumes:
+      - type: bind
+        source: z:\
+        target: c:\data
+`,
+			expected: types.ServiceVolumeConfig{Type: "bind", Source: `z:\`, Target: `c:\data`},
+		},
+		{
+			doc: "Z-drive uppercase",
+			yaml: `
+version: '3.3'
+
+services:
+  windows:
+    image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019
+    volumes:
+      - type: bind
+        source: Z:\
+        target: C:\data
+`,
+			expected: types.ServiceVolumeConfig{Type: "bind", Source: `Z:\`, Target: `C:\data`},
+		},
+		{
+			doc: "Z-drive subdirectory",
+			yaml: `
+version: '3.3'
+
+services:
+  windows:
+    image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019
+    volumes:
+      - type: bind
+        source: Z:\some-dir
+        target: C:\data
+`,
+			expected: types.ServiceVolumeConfig{Type: "bind", Source: `Z:\some-dir`, Target: `C:\data`},
+		},
+		{
+			doc: "forward-slashes",
+			yaml: `
+version: '3.3'
+
+services:
+  app:
+    image: app:latest
+    volumes:
+      - type: bind
+        source: /z/some-dir
+        target: /c/data
+`,
+			expected: types.ServiceVolumeConfig{Type: "bind", Source: `/z/some-dir`, Target: `/c/data`},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.doc, func(t *testing.T) {
+			config, err := loadYAML(tc.yaml)
+			assert.NilError(t, err)
+			assert.Check(t, is.Len(config.Services[0].Volumes, 1))
+			assert.Check(t, is.DeepEqual(tc.expected, config.Services[0].Volumes[0]))
+		})
+	}
+}
+
 func TestLoadBindMountWithSource(t *testing.T) {
 	config, err := loadYAML(`
 version: "3.5"

cli/compose/loader/windows_path.go

@@ -0,0 +1,66 @@
+package loader
+
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// https://github.com/golang/go/blob/master/LICENSE
+
+// This file contains utilities to check for Windows absolute paths on Linux.
+// The code in this file was largely copied from the Golang filepath package
+// https://github.com/golang/go/blob/1d0e94b1e13d5e8a323a63cd1cc1ef95290c9c36/src/path/filepath/path_windows.go#L12-L65
+
+func isSlash(c uint8) bool {
+	return c == '\\' || c == '/'
+}
+
+// isAbs reports whether the path is a Windows absolute path.
+func isAbs(path string) (b bool) {
+	l := volumeNameLen(path)
+	if l == 0 {
+		return false
+	}
+	path = path[l:]
+	if path == "" {
+		return false
+	}
+	return isSlash(path[0])
+}
+
+// volumeNameLen returns length of the leading volume name on Windows.
+// It returns 0 elsewhere.
+// nolint: gocyclo
+func volumeNameLen(path string) int {
+	if len(path) < 2 {
+		return 0
+	}
+	// with drive letter
+	c := path[0]
+	if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
+		return 2
+	}
+	// is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
+	if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) &&
+		!isSlash(path[2]) && path[2] != '.' {
+		// first, leading `\\` and next shouldn't be `\`. its server name.
+		for n := 3; n < l-1; n++ {
+			// second, next '\' shouldn't be repeated.
+			if isSlash(path[n]) {
+				n++
+				// third, following something characters. its share name.
+				if !isSlash(path[n]) {
+					if path[n] == '.' {
+						break
+					}
+					for ; n < l; n++ {
+						if isSlash(path[n]) {
+							break
+						}
+					}
+					return n
+				}
+				break
+			}
+		}
+	}
+	return 0
+}

cli/compose/loader/windows_path_test.go

@@ -0,0 +1,61 @@
+package loader
+
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// https://github.com/golang/go/blob/master/LICENSE
+
+// The code in this file was copied from the Golang filepath package with some
+// small modifications to run it on non-Windows platforms.
+// https://github.com/golang/go/blob/1d0e94b1e13d5e8a323a63cd1cc1ef95290c9c36/src/path/filepath/path_test.go#L711-L763
+
+import "testing"
+
+type IsAbsTest struct {
+	path  string
+	isAbs bool
+}
+
+var isabstests = []IsAbsTest{
+	{"", false},
+	{"/", true},
+	{"/usr/bin/gcc", true},
+	{"..", false},
+	{"/a/../bb", true},
+	{".", false},
+	{"./", false},
+	{"lala", false},
+}
+
+var winisabstests = []IsAbsTest{
+	{`C:\`, true},
+	{`c\`, false},
+	{`c::`, false},
+	{`c:`, false},
+	{`/`, false},
+	{`\`, false},
+	{`\Windows`, false},
+	{`c:a\b`, false},
+	{`c:\a\b`, true},
+	{`c:/a/b`, true},
+	{`\\host\share\foo`, true},
+	{`//host/share/foo/bar`, true},
+}
+
+func TestIsAbs(t *testing.T) {
+	tests := append(isabstests, winisabstests...)
+	// All non-windows tests should fail, because they have no volume letter.
+	for _, test := range isabstests {
+		tests = append(tests, IsAbsTest{test.path, false})
+	}
+	// All non-windows test should work as intended if prefixed with volume letter.
+	for _, test := range isabstests {
+		tests = append(tests, IsAbsTest{"c:" + test.path, test.isAbs})
+	}
+
+	for _, test := range winisabstests {
+		if r := isAbs(test.path); r != test.isAbs {
+			t.Errorf("IsAbs(%q) = %v, want %v", test.path, r, test.isAbs)
+		}
+	}
+}

cli/context/docker/load.go

@@ -104,8 +104,8 @@ func (c *Endpoint) ClientOpts() ([]client.Opt, error) {
 				return nil, err
 			}
 			result = append(result,
-				client.WithHost(c.Host),
 				withHTTPClient(tlsConfig),
+				client.WithHost(c.Host),
 			)
 
 		} else {

contrib/completion/zsh/_docker

@@ -9,6 +9,7 @@
 #   - Felix Riedel
 #   - Steve Durrheimer
 #   - Vincent Bernat
+#   - Rohan Verma
 #
 # license:
 #
@@ -2784,7 +2785,7 @@ __docker_subcommand() {
                 $opts_help \
                 "($help -p --password)"{-p=,--password=}"[Password]:password: " \
                 "($help)--password-stdin[Read password from stdin]" \
-                "($help -u --user)"{-u=,--user=}"[Username]:username: " \
+                "($help -u --username)"{-u=,--username=}"[Username]:username: " \
                 "($help -)1:server: " && ret=0
             ;;
         (logout)

docker.Makefile

@@ -27,32 +27,32 @@ ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM -e TESTFLAGS -e TESTDIR
 .PHONY: build_docker_image
 build_docker_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) -
+	cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(DEV_DOCKER_IMAGE_NAME) -
 
 # build docker image having the linting tools (dockerfiles/Dockerfile.lint)
 .PHONY: build_linter_image
 build_linter_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.lint | docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) -
+	cat ./dockerfiles/Dockerfile.lint | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(LINTER_IMAGE_NAME) -
 
 .PHONY: build_cross_image
 build_cross_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.cross | docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) -
+	cat ./dockerfiles/Dockerfile.cross | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(CROSS_IMAGE_NAME) -
 
 .PHONY: build_shell_validate_image
 build_shell_validate_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.shellcheck | docker build -t $(VALIDATE_IMAGE_NAME) -
+	cat ./dockerfiles/Dockerfile.shellcheck | docker build --build-arg=GO_VERSION -t $(VALIDATE_IMAGE_NAME) -
 
 .PHONY: build_binary_native_image
 build_binary_native_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.binary-native | docker build -t $(BINARY_NATIVE_IMAGE_NAME) -
+	cat ./dockerfiles/Dockerfile.binary-native | docker build --build-arg=GO_VERSION -t $(BINARY_NATIVE_IMAGE_NAME) -
 
 .PHONY: build_e2e_image
 build_e2e_image:
-	docker build -t $(E2E_IMAGE_NAME) --build-arg VERSION=$(VERSION) --build-arg GITCOMMIT=$(GITCOMMIT) -f ./dockerfiles/Dockerfile.e2e .
+	docker build -t $(E2E_IMAGE_NAME) --build-arg=GO_VERSION --build-arg VERSION=$(VERSION) --build-arg GITCOMMIT=$(GITCOMMIT) -f ./dockerfiles/Dockerfile.e2e .
 
 DOCKER_RUN_NAME_OPTION := $(if $(DOCKER_CLI_CONTAINER_NAME),--name $(DOCKER_CLI_CONTAINER_NAME),)
 DOCKER_RUN := docker run --rm $(ENVVARS) $(DOCKER_CLI_MOUNTS) $(DOCKER_RUN_NAME_OPTION)

dockerfiles/Dockerfile.binary-native

@@ -1,4 +1,6 @@
-FROM    golang:1.12.5-alpine
+ARG GO_VERSION=1.12.8
+
+FROM    golang:${GO_VERSION}-alpine
 
 RUN     apk add -U git bash coreutils gcc musl-dev
 

dockerfiles/Dockerfile.cross

@@ -1,4 +1,6 @@
-FROM	dockercore/golang-cross:1.12.5
+ARG GO_VERSION=1.12.8
+
+FROM	dockercore/golang-cross:${GO_VERSION}
 ENV     DISABLE_WARN_OUTSIDE_CONTAINER=1
 WORKDIR /go/src/github.com/docker/cli
 COPY    . .

dockerfiles/Dockerfile.dev

@@ -1,4 +1,6 @@
-FROM    golang:1.12.5-alpine
+ARG GO_VERSION=1.12.8
+
+FROM    golang:${GO_VERSION}-alpine
 
 RUN     apk add -U git make bash coreutils ca-certificates curl
 

dockerfiles/Dockerfile.e2e

@@ -1,6 +1,4 @@
-ARG GO_VERSION=1.12.5
-
-FROM docker/containerd-shim-process:a4d1531 AS containerd-shim-process
+ARG GO_VERSION=1.12.8
 
 # Use Debian based image as docker-compose requires glibc.
 FROM golang:${GO_VERSION}
@@ -9,10 +7,6 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
     openssl \
-    btrfs-tools \
-    libapparmor-dev \
-    libseccomp-dev \
-    iptables \
     openssh-client \
     && rm -rf /var/lib/apt/lists/*
 

dockerfiles/Dockerfile.lint

@@ -1,4 +1,6 @@
-FROM    golang:1.12.5-alpine
+ARG GO_VERSION=1.12.8
+
+FROM    golang:${GO_VERSION}-alpine
 
 RUN     apk add -U git
 

docs/reference/commandline/events.md

@@ -31,7 +31,12 @@ Options:
 ## Description
 
 Use `docker events` to get real-time events from the server. These events differ
-per Docker object type.
+per Docker object type. Different event types have different scopes. Local 
+scoped events are only seen on the node they take place on, and swarm scoped 
+events are seen on all managers.
+
+Only the last 1000 log events are returned. You can use filters to further limit 
+the number of events returned.
 
 ### Object types
 
@@ -160,6 +165,9 @@ that have elapsed since January 1, 1970 (midnight UTC/GMT), not counting leap
 seconds (aka Unix epoch or Unix time), and the optional .nanoseconds field is a
 fraction of a second no more than nine digits long.
 
+Only the last 1000 log events are returned. You can use filters to further limit 
+the number of events returned.
+
 #### Filtering
 
 The filtering flag (`-f` or `--filter`) format is of "key=value". If you would

docs/reference/commandline/run.md

@@ -605,6 +605,33 @@ PS C:\> docker run --device=class/86E0D1E0-8089-11D0-9CE4-08003E301F73 mcr.micro
 > Windows containers. This option fails if the container isolation is `hyperv`
 > or when running Linux Containers on Windows (LCOW).
 
+### Access an NVIDIA GPU
+
+The `--gpus­` flag allows you to access NVIDIA GPU resources. First you need to
+install [nvidia-container-runtime](https://nvidia.github.io/nvidia-container-runtime/).
+Visit [Specify a container's resources](https://docs.docker.com/config/containers/resource_constraints/)
+for more information.
+
+To use `--gpus`, specify which GPUs (or all) to use. If no value is provied, all
+available GPUs are used. The example below exposes all available GPUs.
+
+```bash
+$ docker run -it --rm --gpus all ubuntu nvidia-smi
+```
+
+Use the `device` option to specify GPUs. The example below exposes a specific
+GPU.
+
+```bash
+$ docker run -it --rm --gpus device=GPU-3a23c669-1f69-c64e-cf85-44e9b07e7a2a ubuntu nvidia-smi
+```
+
+The example below exposes the first and third GPUs.
+
+```bash
+$ docker run -it --rm --gpus device=0,2 nvidia-smi
+```
+
 ### Restart policies (--restart)
 
 Use Docker's `--restart` to specify a container's *restart policy*. A restart

e2e/compose-env.yaml

@@ -5,9 +5,11 @@ services:
     image: 'registry:2'
 
   engine:
-      image: 'docker:${TEST_ENGINE_VERSION:-edge-dind}'
+      image: 'docker:${TEST_ENGINE_VERSION:-stable-dind}'
       privileged: true
       command: ['--insecure-registry=registry:5000']
+      environment:
+        - DOCKER_TLS_CERTDIR=
 
   notary-server:
       build:

internal/test/cli.go

@@ -169,6 +169,12 @@ func (c *FakeCli) ErrBuffer() *bytes.Buffer {
 	return c.err
 }
 
+// ResetOutputBuffers resets the .OutBuffer() and.ErrBuffer() back to empty
+func (c *FakeCli) ResetOutputBuffers() {
+	c.outBuffer.Reset()
+	c.err.Reset()
+}
+
 // SetNotaryClient sets the internal getter for retrieving a NotaryClient
 func (c *FakeCli) SetNotaryClient(notaryClientFunc NotaryClientFuncType) {
 	c.notaryClientFunc = notaryClientFunc

opts/hosts_test.go

@@ -53,8 +53,8 @@ func TestParseHost(t *testing.T) {
 func TestParseDockerDaemonHost(t *testing.T) {
 	invalids := map[string]string{
 
-		"tcp:a.b.c.d":                   "Invalid bind address format: tcp:a.b.c.d",
-		"tcp:a.b.c.d/path":              "Invalid bind address format: tcp:a.b.c.d/path",
+		"tcp:a.b.c.d":                   "",
+		"tcp:a.b.c.d/path":              "",
 		"udp://127.0.0.1":               "Invalid bind address format: udp://127.0.0.1",
 		"udp://127.0.0.1:2375":          "Invalid bind address format: udp://127.0.0.1:2375",
 		"tcp://unix:///run/docker.sock": "Invalid proto, expected tcp: unix:///run/docker.sock",
@@ -83,7 +83,7 @@ func TestParseDockerDaemonHost(t *testing.T) {
 		"localhost:5555/path":         "tcp://localhost:5555/path",
 	}
 	for invalidAddr, expectedError := range invalids {
-		if addr, err := parseDockerDaemonHost(invalidAddr); err == nil || err.Error() != expectedError {
+		if addr, err := parseDockerDaemonHost(invalidAddr); err == nil || expectedError != "" && err.Error() != expectedError {
 			t.Errorf("tcp %v address expected error %q return, got %q and addr %v", invalidAddr, expectedError, err, addr)
 		}
 	}
@@ -99,8 +99,8 @@ func TestParseTCP(t *testing.T) {
 		defaultHTTPHost = "tcp://127.0.0.1:2376"
 	)
 	invalids := map[string]string{
-		"tcp:a.b.c.d":          "Invalid bind address format: tcp:a.b.c.d",
-		"tcp:a.b.c.d/path":     "Invalid bind address format: tcp:a.b.c.d/path",
+		"tcp:a.b.c.d":          "",
+		"tcp:a.b.c.d/path":     "",
 		"udp://127.0.0.1":      "Invalid proto, expected tcp: udp://127.0.0.1",
 		"udp://127.0.0.1:2375": "Invalid proto, expected tcp: udp://127.0.0.1:2375",
 	}
@@ -125,7 +125,7 @@ func TestParseTCP(t *testing.T) {
 		"localhost:5555/path":         "tcp://localhost:5555/path",
 	}
 	for invalidAddr, expectedError := range invalids {
-		if addr, err := ParseTCPAddr(invalidAddr, defaultHTTPHost); err == nil || err.Error() != expectedError {
+		if addr, err := ParseTCPAddr(invalidAddr, defaultHTTPHost); err == nil || expectedError != "" && err.Error() != expectedError {
 			t.Errorf("tcp %v address expected error %v return, got %s and addr %v", invalidAddr, expectedError, err, addr)
 		}
 	}

vendor.conf

@@ -16,7 +16,7 @@ github.com/dgrijalva/jwt-go                         a2c85815a77d0f951e33ba4db5ae
 github.com/docker/compose-on-kubernetes             cc4914dfd1b6684a9750a59f3613fc0a95291824 # v0.4.23
 github.com/docker/distribution                      0d3efadf0154c2b8a4e7b6621fff9809655cc580
 github.com/docker/docker                            a004854097417a591c3f6a3aeaab75efae3c5814 https://github.com/docker/engine.git # 19.03 branch
-github.com/docker/docker-credential-helpers         5241b46610f2491efdf9d1c85f1ddf5b02f6d962
+github.com/docker/docker-credential-helpers         54f0238b6bf101fc3ad3b34114cb5520beb562f5 # v0.6.3
 github.com/docker/go                                d30aec9fd63c35133f8f79c3412ad91a3b08be06 # Contains a customized version of canonical/json and is used by Notary. The package is periodically rebased on current Go versions.
 github.com/docker/go-connections                    7395e3f8aa162843a74ed6d48e79627d9792ac55 # v0.4.0
 github.com/docker/go-events                         9461782956ad83b30282bf90e31fa6a70c255ba9

vendor/github.com/docker/docker-credential-helpers/README.md

@@ -16,7 +16,7 @@ The programs in this repository are written with the Go programming language. Th
 $ go get github.com/docker/docker-credential-helpers
 ```
 
-2 - Use `make` to build the program you want. That will leave any executable in the `bin` directory inside the repository.
+2 - Use `make` to build the program you want. That will leave an executable in the `bin` directory inside the repository.
 
 ```
 $ cd $GOPATH/docker/docker-credentials-helpers

vendor/github.com/docker/docker-credential-helpers/credentials/version.go

@@ -1,4 +1,4 @@
 package credentials
 
 // Version holds a string describing the current version
-const Version = "0.6.0"
+const Version = "0.6.3"

vendor/github.com/docker/docker-credential-helpers/osxkeychain/osxkeychain_darwin.c

@@ -224,5 +224,4 @@ void freeListData(char *** data, unsigned int length) {
      for(int i=0; i<length; i++) {
         free((*data)[i]);
      }
-     free(*data);
 }

vendor/github.com/docker/docker-credential-helpers/osxkeychain/osxkeychain_darwin.go

@@ -1,8 +1,8 @@
 package osxkeychain
 
 /*
-#cgo CFLAGS: -x objective-c -mmacosx-version-min=10.10
-#cgo LDFLAGS: -framework Security -framework Foundation -mmacosx-version-min=10.10
+#cgo CFLAGS: -x objective-c -mmacosx-version-min=10.11
+#cgo LDFLAGS: -framework Security -framework Foundation -mmacosx-version-min=10.11
 
 #include "osxkeychain_darwin.h"
 #include <stdlib.h>
@@ -10,12 +10,11 @@ package osxkeychain
 import "C"
 import (
 	"errors"
-	"net/url"
 	"strconv"
-	"strings"
 	"unsafe"
 
 	"github.com/docker/docker-credential-helpers/credentials"
+	"github.com/docker/docker-credential-helpers/registryurl"
 )
 
 // errCredentialsNotFound is the specific error message returned by OS X
@@ -110,15 +109,18 @@ func (h Osxkeychain) List() (map[string]string, error) {
 	defer C.free(unsafe.Pointer(acctsC))
 	var listLenC C.uint
 	errMsg := C.keychain_list(credsLabelC, &pathsC, &acctsC, &listLenC)
+	defer C.freeListData(&pathsC, listLenC)
+	defer C.freeListData(&acctsC, listLenC)
 	if errMsg != nil {
 		defer C.free(unsafe.Pointer(errMsg))
 		goMsg := C.GoString(errMsg)
+		if goMsg == errCredentialsNotFound {
+			return make(map[string]string), nil
+		}
+
 		return nil, errors.New(goMsg)
 	}
 
-	defer C.freeListData(&pathsC, listLenC)
-	defer C.freeListData(&acctsC, listLenC)
-
 	var listLen int
 	listLen = int(listLenC)
 	pathTmp := (*[1 << 30]*C.char)(unsafe.Pointer(pathsC))[:listLen:listLen]
@@ -135,7 +137,7 @@ func (h Osxkeychain) List() (map[string]string, error) {
 }
 
 func splitServer(serverURL string) (*C.struct_Server, error) {
-	u, err := parseURL(serverURL)
+	u, err := registryurl.Parse(serverURL)
 	if err != nil {
 		return nil, err
 	}
@@ -145,7 +147,7 @@ func splitServer(serverURL string) (*C.struct_Server, error) {
 		proto = C.kSecProtocolTypeHTTP
 	}
 	var port int
-	p := getPort(u)
+	p := registryurl.GetPort(u)
 	if p != "" {
 		port, err = strconv.Atoi(p)
 		if err != nil {
@@ -155,7 +157,7 @@ func splitServer(serverURL string) (*C.struct_Server, error) {
 
 	return &C.struct_Server{
 		proto: C.SecProtocolType(proto),
-		host:  C.CString(getHostname(u)),
+		host:  C.CString(registryurl.GetHostname(u)),
 		port:  C.uint(port),
 		path:  C.CString(u.Path),
 	}, nil
@@ -165,32 +167,3 @@ func freeServer(s *C.struct_Server) {
 	C.free(unsafe.Pointer(s.host))
 	C.free(unsafe.Pointer(s.path))
 }
-
-// parseURL parses and validates a given serverURL to an url.URL, and
-// returns an error if validation failed. Querystring parameters are
-// omitted in the resulting URL, because they are not used in the helper.
-//
-// If serverURL does not have a valid scheme, `//` is used as scheme
-// before parsing. This prevents the hostname being used as path,
-// and the credentials being stored without host.
-func parseURL(serverURL string) (*url.URL, error) {
-	// Check if serverURL has a scheme, otherwise add `//` as scheme.
-	if !strings.Contains(serverURL, "://") && !strings.HasPrefix(serverURL, "//") {
-		serverURL = "//" + serverURL
-	}
-
-	u, err := url.Parse(serverURL)
-	if err != nil {
-		return nil, err
-	}
-
-	if u.Scheme != "" && u.Scheme != "https" && u.Scheme != "http" {
-		return nil, errors.New("unsupported scheme: " + u.Scheme)
-	}
-	if getHostname(u) == "" {
-		return nil, errors.New("no hostname in URL")
-	}
-
-	u.RawQuery = ""
-	return u, nil
-}

vendor/github.com/docker/docker-credential-helpers/osxkeychain/url_go18.go

@@ -1,13 +0,0 @@
-//+build go1.8
-
-package osxkeychain
-
-import "net/url"
-
-func getHostname(u *url.URL) string {
-	return u.Hostname()
-}
-
-func getPort(u *url.URL) string {
-	return u.Port()
-}

vendor/github.com/docker/docker-credential-helpers/osxkeychain/url_non_go18.go

@@ -1,41 +0,0 @@
-//+build !go1.8
-
-package osxkeychain
-
-import (
-	"net/url"
-	"strings"
-)
-
-func getHostname(u *url.URL) string {
-	return stripPort(u.Host)
-}
-
-func getPort(u *url.URL) string {
-	return portOnly(u.Host)
-}
-
-func stripPort(hostport string) string {
-	colon := strings.IndexByte(hostport, ':')
-	if colon == -1 {
-		return hostport
-	}
-	if i := strings.IndexByte(hostport, ']'); i != -1 {
-		return strings.TrimPrefix(hostport[:i], "[")
-	}
-	return hostport[:colon]
-}
-
-func portOnly(hostport string) string {
-	colon := strings.IndexByte(hostport, ':')
-	if colon == -1 {
-		return ""
-	}
-	if i := strings.Index(hostport, "]:"); i != -1 {
-		return hostport[i+len("]:"):]
-	}
-	if strings.Contains(hostport, "]") {
-		return ""
-	}
-	return hostport[colon+len(":"):]
-}

vendor/github.com/docker/docker-credential-helpers/secretservice/secretservice_linux.c

@@ -158,5 +158,4 @@ void freeListData(char *** data, unsigned int length) {
 	for(i=0; i<length; i++) {
 		free((*data)[i]);
 	}
-	free(*data);
 }

vendor/github.com/docker/docker-credential-helpers/secretservice/secretservice_linux.go

@@ -92,12 +92,12 @@ func (h Secretservice) List() (map[string]string, error) {
 	defer C.free(unsafe.Pointer(acctsC))
 	var listLenC C.uint
 	err := C.list(credsLabelC, &pathsC, &acctsC, &listLenC)
-	if err != nil {
-		defer C.free(unsafe.Pointer(err))
-		return nil, errors.New("Error from list function in secretservice_linux.c likely due to error in secretservice library")
-	}
 	defer C.freeListData(&pathsC, listLenC)
 	defer C.freeListData(&acctsC, listLenC)
+	if err != nil {
+		defer C.g_error_free(err)
+		return nil, errors.New("Error from list function in secretservice_linux.c likely due to error in secretservice library")
+	}
 
 	resp := make(map[string]string)