Merge pull request #1580 from thaJeztah/18.09_backport_e2e-invocation-nit

[18.09 backport] e2e updates

appveyor.yml

@@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\docker\cli
 
 environment:
   GOPATH: c:\gopath
-  GOVERSION: 1.10.4
+  GOVERSION: 1.10.6
   DEPVERSION: v0.4.1
 
 install:

cli/command/cli.go

@@ -274,21 +274,17 @@ func NewDockerCli(in io.ReadCloser, out, err io.Writer, isTrusted bool, containe
 
 // NewAPIClientFromFlags creates a new APIClient from command line flags
 func NewAPIClientFromFlags(opts *cliflags.CommonOptions, configFile *configfile.ConfigFile) (client.APIClient, error) {
-	unparsedHost, err := getUnparsedServerHost(opts.Hosts)
+	host, err := getServerHost(opts.Hosts, opts.TLSOptions)
 	if err != nil {
 		return &client.Client{}, err
 	}
 	var clientOpts []func(*client.Client) error
-	helper, err := connhelper.GetConnectionHelper(unparsedHost)
+	helper, err := connhelper.GetConnectionHelper(host)
 	if err != nil {
 		return &client.Client{}, err
 	}
 	if helper == nil {
 		clientOpts = append(clientOpts, withHTTPClient(opts.TLSOptions))
-		host, err := dopts.ParseHost(opts.TLSOptions != nil, unparsedHost)
-		if err != nil {
-			return &client.Client{}, err
-		}
 		clientOpts = append(clientOpts, client.WithHost(host))
 	} else {
 		clientOpts = append(clientOpts, func(c *client.Client) error {
@@ -321,7 +317,7 @@ func NewAPIClientFromFlags(opts *cliflags.CommonOptions, configFile *configfile.
 	return client.NewClientWithOpts(clientOpts...)
 }
 
-func getUnparsedServerHost(hosts []string) (string, error) {
+func getServerHost(hosts []string, tlsOptions *tlsconfig.Options) (string, error) {
 	var host string
 	switch len(hosts) {
 	case 0:
@@ -331,7 +327,8 @@ func getUnparsedServerHost(hosts []string) (string, error) {
 	default:
 		return "", errors.New("Please specify only one -H")
 	}
-	return host, nil
+
+	return dopts.ParseHost(tlsOptions != nil, host)
 }
 
 func withHTTPClient(tlsOpts *tlsconfig.Options) func(*client.Client) error {

cli/command/cli_test.go

@@ -43,6 +43,26 @@ func TestNewAPIClientFromFlags(t *testing.T) {
 	assert.Check(t, is.Equal(api.DefaultVersion, apiclient.ClientVersion()))
 }
 
+func TestNewAPIClientFromFlagsForDefaultSchema(t *testing.T) {
+	host := ":2375"
+	opts := &flags.CommonOptions{Hosts: []string{host}}
+	configFile := &configfile.ConfigFile{
+		HTTPHeaders: map[string]string{
+			"My-Header": "Custom-Value",
+		},
+	}
+	apiclient, err := NewAPIClientFromFlags(opts, configFile)
+	assert.NilError(t, err)
+	assert.Check(t, is.Equal("tcp://localhost"+host, apiclient.DaemonHost()))
+
+	expectedHeaders := map[string]string{
+		"My-Header":  "Custom-Value",
+		"User-Agent": UserAgent(),
+	}
+	assert.Check(t, is.DeepEqual(expectedHeaders, apiclient.(*client.Client).CustomHTTPHeaders()))
+	assert.Check(t, is.Equal(api.DefaultVersion, apiclient.ClientVersion()))
+}
+
 func TestNewAPIClientFromFlagsWithAPIVersionFromEnv(t *testing.T) {
 	customVersion := "v3.3.3"
 	defer env.Patch(t, "DOCKER_API_VERSION", customVersion)()

cli/command/image/prune.go

@@ -8,7 +8,9 @@ import (
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/opts"
+	"github.com/docker/docker/api/types/filters"
 	units "github.com/docker/go-units"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -55,8 +57,24 @@ Are you sure you want to continue?`
 Are you sure you want to continue?`
 )
 
+// cloneFilter is a temporary workaround that uses existing public APIs from the filters package to clone a filter.
+// TODO(tiborvass): remove this once filters.Args.Clone() is added.
+func cloneFilter(args filters.Args) (newArgs filters.Args, err error) {
+	if args.Len() == 0 {
+		return filters.NewArgs(), nil
+	}
+	b, err := args.MarshalJSON()
+	if err != nil {
+		return newArgs, err
+	}
+	return filters.FromJSON(string(b))
+}
+
 func runPrune(dockerCli command.Cli, options pruneOptions) (spaceReclaimed uint64, output string, err error) {
-	pruneFilters := options.filter.Value()
+	pruneFilters, err := cloneFilter(options.filter.Value())
+	if err != nil {
+		return 0, "", errors.Wrap(err, "could not copy filter in image prune")
+	}
 	pruneFilters.Add("dangling", fmt.Sprintf("%v", !options.all))
 	pruneFilters = command.PruneFilters(dockerCli, pruneFilters)
 

cli/command/image/prune_test.go

@@ -70,6 +70,14 @@ func TestNewPruneCommandSuccess(t *testing.T) {
 				}, nil
 			},
 		},
+		{
+			name: "label-filter",
+			args: []string{"--force", "--filter", "label=foobar"},
+			imagesPruneFunc: func(pruneFilter filters.Args) (types.ImagesPruneReport, error) {
+				assert.Check(t, is.Equal("foobar", pruneFilter.Get("label")[0]))
+				return types.ImagesPruneReport{}, nil
+			},
+		},
 		{
 			name: "force-untagged",
 			args: []string{"--force"},

cli/command/image/testdata/prune-command-success.label-filter.golden

@@ -0,0 +1 @@
+Total reclaimed space: 0B

cli/command/manifest/util.go

@@ -18,6 +18,7 @@ type osArch struct {
 // list of valid os/arch values (see "Optional Environment Variables" section
 // of https://golang.org/doc/install/source
 // Added linux/s390x as we know System z support already exists
+// Keep in sync with _docker_manifest_annotate in contrib/completion/bash/docker
 var validOSArches = map[osArch]bool{
 	{os: "darwin", arch: "386"}:      true,
 	{os: "darwin", arch: "amd64"}:    true,

cli/command/service/update.go

@@ -302,6 +302,12 @@ func updateService(ctx context.Context, apiClient client.NetworkAPIClient, flags
 		if task.Resources == nil {
 			task.Resources = &swarm.ResourceRequirements{}
 		}
+		if task.Resources.Limits == nil {
+			task.Resources.Limits = &swarm.Resources{}
+		}
+		if task.Resources.Reservations == nil {
+			task.Resources.Reservations = &swarm.Resources{}
+		}
 		return task.Resources
 	}
 

cli/command/service/update_test.go

@@ -617,6 +617,38 @@ func TestUpdateIsolationValid(t *testing.T) {
 // and that values are not updated are not reset to their default value
 func TestUpdateLimitsReservations(t *testing.T) {
 	spec := swarm.ServiceSpec{
+		TaskTemplate: swarm.TaskSpec{
+			ContainerSpec: &swarm.ContainerSpec{},
+		},
+	}
+
+	// test that updating works if the service did not previously
+	// have limits set (https://github.com/moby/moby/issues/38363)
+	flags := newUpdateCommand(nil).Flags()
+	err := flags.Set(flagLimitCPU, "2")
+	assert.NilError(t, err)
+	err = flags.Set(flagLimitMemory, "200M")
+	assert.NilError(t, err)
+	err = updateService(context.Background(), nil, flags, &spec)
+	assert.NilError(t, err)
+
+	spec = swarm.ServiceSpec{
+		TaskTemplate: swarm.TaskSpec{
+			ContainerSpec: &swarm.ContainerSpec{},
+		},
+	}
+
+	// test that updating works if the service did not previously
+	// have reservations set (https://github.com/moby/moby/issues/38363)
+	flags = newUpdateCommand(nil).Flags()
+	err = flags.Set(flagReserveCPU, "2")
+	assert.NilError(t, err)
+	err = flags.Set(flagReserveMemory, "200M")
+	assert.NilError(t, err)
+	err = updateService(context.Background(), nil, flags, &spec)
+	assert.NilError(t, err)
+
+	spec = swarm.ServiceSpec{
 		TaskTemplate: swarm.TaskSpec{
 			ContainerSpec: &swarm.ContainerSpec{},
 			Resources: &swarm.ResourceRequirements{
@@ -632,8 +664,8 @@ func TestUpdateLimitsReservations(t *testing.T) {
 		},
 	}
 
-	flags := newUpdateCommand(nil).Flags()
-	err := flags.Set(flagLimitCPU, "2")
+	flags = newUpdateCommand(nil).Flags()
+	err = flags.Set(flagLimitCPU, "2")
 	assert.NilError(t, err)
 	err = flags.Set(flagReserveCPU, "2")
 	assert.NilError(t, err)

cli/command/system/prune.go

@@ -73,11 +73,10 @@ func runPrune(dockerCli command.Cli, options pruneOptions) error {
 	if options.pruneVolumes {
 		pruneFuncs = append(pruneFuncs, volume.RunPrune)
 	}
+	pruneFuncs = append(pruneFuncs, image.RunPrune)
 	if options.pruneBuildCache {
 		pruneFuncs = append(pruneFuncs, builder.CachePrune)
 	}
-	// FIXME: modify image.RunPrune to not modify options.filter, otherwise this has to be last in the list.
-	pruneFuncs = append(pruneFuncs, image.RunPrune)
 
 	var spaceReclaimed uint64
 	for _, pruneFn := range pruneFuncs {

contrib/completion/bash/docker

@@ -563,23 +563,39 @@ __docker_append_to_completions() {
 	COMPREPLY=( ${COMPREPLY[@]/%/"$1"} )
 }
 
-# __docker_daemon_is_experimental tests whether the currently configured Docker
-# daemon runs in experimental mode. If so, the function exits with 0 (true).
-# Otherwise, or if the result cannot be determined, the exit value is 1 (false).
-__docker_daemon_is_experimental() {
-	[ "$(__docker_q version -f '{{.Server.Experimental}}')" = "true" ]
+# __docker_fetch_info fetches information about the configured Docker server and updates
+# several variables with the results.
+# The result is cached for the duration of one invocation of bash completion.
+__docker_fetch_info() {
+	if [ -z "$info_fetched" ] ; then
+		read -r client_experimental server_experimental server_os < <(__docker_q version -f '{{.Client.Experimental}} {{.Server.Experimental}} {{.Server.Os}}')
+		info_fetched=true
+	fi
 }
 
-# __docker_daemon_os_is tests whether the currently configured Docker daemon runs
+# __docker_client_is_experimental tests whether the Docker cli is configured to support
+# experimental features. If so, the function exits with 0 (true).
+# Otherwise, or if the result cannot be determined, the exit value is 1 (false).
+__docker_client_is_experimental() {
+	__docker_fetch_info
+	[ "$client_experimental" = "true" ]
+}
+
+# __docker_server_is_experimental tests whether the currently configured Docker
+# server runs in experimental mode. If so, the function exits with 0 (true).
+# Otherwise, or if the result cannot be determined, the exit value is 1 (false).
+__docker_server_is_experimental() {
+	__docker_fetch_info
+	[ "$server_experimental" = "true" ]
+}
+
+# __docker_server_os_is tests whether the currently configured Docker server runs
 # on the operating system passed in as the first argument.
-# It does so by querying the daemon for its OS. The result is cached for the duration
-# of one invocation of bash completion so that this function can be used to test for
-# several different operating systems without additional costs.
 # Known operating systems: linux, windows.
-__docker_daemon_os_is() {
+__docker_server_os_is() {
 	local expected_os="$1"
-	local actual_os=${daemon_os=$(__docker_q version -f '{{.Server.Os}}')}
-	[ "$actual_os" = "$expected_os" ]
+	__docker_fetch_info
+	[ "$server_os" = "$expected_os" ]
 }
 
 # __docker_stack_orchestrator_is tests whether the client is configured to use
@@ -865,6 +881,7 @@ __docker_complete_log_drivers() {
 		gelf
 		journald
 		json-file
+		local
 		logentries
 		none
 		splunk
@@ -888,7 +905,8 @@ __docker_complete_log_options() {
 	local gcplogs_options="$common_options1 $common_options2 gcp-log-cmd gcp-meta-id gcp-meta-name gcp-meta-zone gcp-project"
 	local gelf_options="$common_options1 $common_options2 gelf-address gelf-compression-level gelf-compression-type gelf-tcp-max-reconnect gelf-tcp-reconnect-delay tag"
 	local journald_options="$common_options1 $common_options2 tag"
-	local json_file_options="$common_options1 $common_options2 max-file max-size"
+	local json_file_options="$common_options1 $common_options2 compress max-file max-size"
+	local local_options="$common_options1 compress max-file max-size"
 	local logentries_options="$common_options1 $common_options2 line-only logentries-token tag"
 	local splunk_options="$common_options1 $common_options2 splunk-caname splunk-capath splunk-format splunk-gzip splunk-gzip-level splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url splunk-verify-connection tag"
 	local syslog_options="$common_options1 $common_options2 syslog-address syslog-facility syslog-format syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify tag"
@@ -917,6 +935,9 @@ __docker_complete_log_options() {
 		json-file)
 			COMPREPLY=( $( compgen -W "$json_file_options" -S = -- "$cur" ) )
 			;;
+		local)
+			COMPREPLY=( $( compgen -W "$local_options" -S = -- "$cur" ) )
+			;;
 		logentries)
 			COMPREPLY=( $( compgen -W "$logentries_options" -S = -- "$cur" ) )
 			;;
@@ -946,7 +967,7 @@ __docker_complete_log_driver_options() {
 			__docker_nospace
 			return
 			;;
-		fluentd-async-connect)
+		compress|fluentd-async-connect)
 			COMPREPLY=( $( compgen -W "false true" -- "${cur##*=}" ) )
 			return
 			;;
@@ -1128,7 +1149,8 @@ _docker_docker() {
 		*)
 			local counter=$( __docker_pos_first_nonflag "$(__docker_to_extglob "$global_options_with_args")" )
 			if [ "$cword" -eq "$counter" ]; then
-				__docker_daemon_is_experimental && commands+=(${experimental_commands[*]})
+				__docker_client_is_experimental && commands+=(${experimental_client_commands[*]})
+				__docker_server_is_experimental && commands+=(${experimental_server_commands[*]})
 				COMPREPLY=( $( compgen -W "${commands[*]} help" -- "$cur" ) )
 			fi
 			;;
@@ -1837,14 +1859,14 @@ _docker_container_run_and_create() {
 		--volume -v
 		--workdir -w
 	"
-	__docker_daemon_os_is windows && options_with_args+="
+	__docker_server_os_is windows && options_with_args+="
 		--cpu-count
 		--cpu-percent
 		--io-maxbandwidth
 		--io-maxiops
 		--isolation
 	"
-	__docker_daemon_is_experimental && options_with_args+="
+	__docker_server_is_experimental && options_with_args+="
 		--platform
 	"
 
@@ -1960,7 +1982,7 @@ _docker_container_run_and_create() {
 			return
 			;;
 		--isolation)
-			if __docker_daemon_os_is windows ; then
+			if __docker_server_os_is windows ; then
 				__docker_complete_isolation
 				return
 			fi
@@ -2071,12 +2093,12 @@ _docker_container_start() {
 	__docker_complete_detach_keys && return
 	case "$prev" in
 		--checkpoint)
-			if __docker_daemon_is_experimental ; then
+			if __docker_server_is_experimental ; then
 				return
 			fi
 			;;
 		--checkpoint-dir)
-			if __docker_daemon_is_experimental ; then
+			if __docker_server_is_experimental ; then
 				_filedir -d
 				return
 			fi
@@ -2086,7 +2108,7 @@ _docker_container_start() {
 	case "$cur" in
 		-*)
 			local options="--attach -a --detach-keys --help --interactive -i"
-			__docker_daemon_is_experimental && options+=" --checkpoint --checkpoint-dir"
+			__docker_server_is_experimental && options+=" --checkpoint --checkpoint-dir"
 			COMPREPLY=( $( compgen -W "$options" -- "$cur" ) )
 			;;
 		*)
@@ -2449,7 +2471,7 @@ _docker_daemon() {
 }
 
 _docker_deploy() {
-	__docker_daemon_is_experimental && _docker_stack_deploy
+	__docker_server_is_experimental && _docker_stack_deploy
 }
 
 _docker_diff() {
@@ -2535,7 +2557,7 @@ _docker_image_build() {
 		--target
 		--ulimit
 	"
-	__docker_daemon_os_is windows && options_with_args+="
+	__docker_server_os_is windows && options_with_args+="
 		--isolation
 	"
 
@@ -2549,7 +2571,7 @@ _docker_image_build() {
 		--quiet -q
 		--rm
 	"
-	if __docker_daemon_is_experimental ; then
+	if __docker_server_is_experimental ; then
 		options_with_args+="
 			--platform
 		"
@@ -2584,7 +2606,7 @@ _docker_image_build() {
 			return
 			;;
 		--isolation)
-			if __docker_daemon_os_is windows ; then
+			if __docker_server_os_is windows ; then
 				__docker_complete_isolation
 				return
 			fi
@@ -2664,14 +2686,16 @@ _docker_image_images() {
 
 _docker_image_import() {
 	case "$prev" in
-		--change|-c|--message|-m)
+		--change|-c|--message|-m|--platform)
 			return
 			;;
 	esac
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--change -c --help --message -m" -- "$cur" ) )
+			local options="--change -c --help --message -m"
+			__docker_server_is_experimental && options+=" --platform"
+			COMPREPLY=( $( compgen -W "$options" -- "$cur" ) )
 			;;
 		*)
 			local counter=$(__docker_pos_first_nonflag '--change|-c|--message|-m')
@@ -2779,7 +2803,7 @@ _docker_image_pull() {
 	case "$cur" in
 		-*)
 			local options="--all-tags -a --disable-content-trust=false --help"
-			__docker_daemon_is_experimental && options+=" --platform"
+			__docker_server_is_experimental && options+=" --platform"
 
 			COMPREPLY=( $( compgen -W "$options" -- "$cur" ) )
 			;;
@@ -3395,7 +3419,6 @@ _docker_service_update_and_create() {
 	local options_with_args="
 		--endpoint-mode
 		--entrypoint
-		--force
 		--health-cmd
 		--health-interval
 		--health-retries
@@ -3431,7 +3454,7 @@ _docker_service_update_and_create() {
 		--user -u
 		--workdir -w
 	"
-	__docker_daemon_os_is windows && options_with_args+="
+	__docker_server_os_is windows && options_with_args+="
 		--credential-spec
 	"
 
@@ -3520,6 +3543,10 @@ _docker_service_update_and_create() {
 			--secret-rm
 		"
 
+		boolean_options="$boolean_options
+			--force
+		"
+
 		case "$prev" in
 			--env-rm)
 				COMPREPLY=( $( compgen -e -- "$cur" ) )
@@ -3817,6 +3844,109 @@ _docker_swarm_update() {
 	esac
 }
 
+_docker_manifest() {
+	local subcommands="
+		annotate
+		create
+		inspect
+		push
+	"
+	__docker_subcommands "$subcommands" && return
+
+	case "$cur" in
+		-*)
+			COMPREPLY=( $( compgen -W "--help" -- "$cur" ) )
+			;;
+		*)
+			COMPREPLY=( $( compgen -W "$subcommands" -- "$cur" ) )
+			;;
+	esac
+}
+
+_docker_manifest_annotate() {
+	case "$prev" in
+		--arch)
+			COMPREPLY=( $( compgen -W "
+				386
+				amd64
+				arm
+				arm64
+				mips64
+				mips64le
+				ppc64le
+				s390x" -- "$cur" ) )
+			return
+			;;
+		--os)
+			COMPREPLY=( $( compgen -W "
+				darwin
+				dragonfly
+				freebsd
+				linux
+				netbsd
+				openbsd
+				plan9
+				solaris
+				windows" -- "$cur" ) )
+			return
+			;;
+		--os-features|--variant)
+			return
+			;;
+	esac
+
+	case "$cur" in
+		-*)
+			COMPREPLY=( $( compgen -W "--arch --help --os --os-features --variant" -- "$cur" ) )
+			;;
+		*)
+			local counter=$( __docker_pos_first_nonflag "--arch|--os|--os-features|--variant" )
+			if [ "$cword" -eq "$counter" ] || [ "$cword" -eq "$((counter + 1))" ]; then
+				__docker_complete_images --force-tag --id
+			fi
+			;;
+	esac
+}
+
+_docker_manifest_create() {
+	case "$cur" in
+		-*)
+			COMPREPLY=( $( compgen -W "--amend -a --help --insecure" -- "$cur" ) )
+			;;
+		*)
+			__docker_complete_images --force-tag --id
+			;;
+	esac
+}
+
+_docker_manifest_inspect() {
+	case "$cur" in
+		-*)
+			COMPREPLY=( $( compgen -W "--help --insecure --verbose -v" -- "$cur" ) )
+			;;
+		*)
+			local counter=$( __docker_pos_first_nonflag )
+			if [ "$cword" -eq "$counter" ] || [ "$cword" -eq "$((counter + 1))" ]; then
+				__docker_complete_images --force-tag --id
+			fi
+			;;
+	esac
+}
+
+_docker_manifest_push() {
+	case "$cur" in
+		-*)
+			COMPREPLY=( $( compgen -W "--help --insecure --purge -p" -- "$cur" ) )
+			;;
+		*)
+			local counter=$( __docker_pos_first_nonflag )
+			if [ "$cword" -eq "$counter" ]; then
+				__docker_complete_images --force-tag --id
+			fi
+			;;
+	esac
+}
+
 _docker_node() {
 	local subcommands="
 		demote
@@ -4451,7 +4581,7 @@ _docker_stack_deploy() {
 	case "$cur" in
 		-*)
 			local options="--compose-file -c --help --orchestrator"
-			__docker_daemon_is_experimental && __docker_stack_orchestrator_is swarm && options+=" --bundle-file"
+			__docker_server_is_experimental && __docker_stack_orchestrator_is swarm && options+=" --bundle-file"
 			__docker_stack_orchestrator_is kubernetes && options+=" --kubeconfig --namespace"
 			__docker_stack_orchestrator_is swarm && options+=" --prune --resolve-image --with-registry-auth"
 			COMPREPLY=( $( compgen -W "$options" -- "$cur" ) )
@@ -5074,7 +5204,11 @@ _docker() {
 		wait
 	)
 
-	local experimental_commands=(
+	local experimental_client_commands=(
+		manifest
+	)
+
+	local experimental_server_commands=(
 		checkpoint
 		deploy
 	)
@@ -5098,10 +5232,12 @@ _docker() {
 		--tlskey
 	"
 
-	local host config daemon_os
-
+	# variables to cache server info, populated on demand for performance reasons
+	local info_fetched server_experimental server_os
 	# variables to cache client info, populated on demand for performance reasons
-	local stack_orchestrator_is_kubernetes stack_orchestrator_is_swarm
+	local client_experimental stack_orchestrator_is_kubernetes stack_orchestrator_is_swarm
+
+	local host config
 
 	COMPREPLY=()
 	local cur prev words cword

docker.Makefile

@@ -105,7 +105,7 @@ shellcheck: build_shell_validate_image ## run shellcheck validation
 	docker run -ti --rm $(ENVVARS) $(MOUNTS) $(VALIDATE_IMAGE_NAME) make shellcheck
 
 .PHONY: test-e2e ## run e2e tests
-test-e2e: test-e2e-non-experimental test-e2e-experimental
+test-e2e: test-e2e-non-experimental test-e2e-experimental test-e2e-connhelper-ssh
 
 .PHONY: test-e2e-experimental
 test-e2e-experimental: build_e2e_image
@@ -115,6 +115,10 @@ test-e2e-experimental: build_e2e_image
 test-e2e-non-experimental: build_e2e_image
 	docker run --rm -v /var/run/docker.sock:/var/run/docker.sock $(E2E_IMAGE_NAME)
 
+.PHONY: test-e2e-connhelper-ssh
+test-e2e-connhelper-ssh: build_e2e_image
+	docker run -e TEST_CONNHELPER=ssh -e DOCKERD_EXPERIMENTAL=1 --rm -v /var/run/docker.sock:/var/run/docker.sock $(E2E_IMAGE_NAME)
+
 .PHONY: help
 help: ## print this help
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

dockerfiles/Dockerfile.binary-native

@@ -1,4 +1,4 @@
-FROM    golang:1.10.4-alpine
+FROM    golang:1.10.6-alpine
 
 RUN     apk add -U git bash coreutils gcc musl-dev
 

dockerfiles/Dockerfile.cross

@@ -1,3 +1,3 @@
-FROM    dockercore/golang-cross:1.10.4@sha256:55c7b933ac944f4922b673b4d4340d1a0404f3c324bd0b3f13a4326c427b1f2a
+FROM    dockercore/golang-cross:1.10.6@sha256:2054e793010774e9dbafc808dd550a99f8468013731728bea7fce56681510078
 ENV     DISABLE_WARN_OUTSIDE_CONTAINER=1
 WORKDIR /go/src/github.com/docker/cli

dockerfiles/Dockerfile.dev

@@ -1,5 +1,5 @@
 
-FROM    golang:1.10.4-alpine
+FROM    golang:1.10.6-alpine
 
 RUN     apk add -U git make bash coreutils ca-certificates curl
 

dockerfiles/Dockerfile.e2e

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.10.4
+ARG GO_VERSION=1.10.6
 
 FROM docker/containerd-shim-process:a4d1531 AS containerd-shim-process
 
@@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y \
     libapparmor-dev \
     libseccomp-dev \
     iptables \
+    openssh-client \
     && rm -rf /var/lib/apt/lists/*
 
 ARG COMPOSE_VERSION=1.21.2

dockerfiles/Dockerfile.lint

@@ -1,4 +1,4 @@
-FROM    golang:1.10.4-alpine
+FROM    golang:1.10.6-alpine
 
 RUN     apk add -U git
 

docs/reference/builder.md

@@ -121,6 +121,28 @@ registries.
 When you're done with your build, you're ready to look into [*Pushing a
 repository to its registry*](https://docs.docker.com/engine/tutorials/dockerrepos/#/contributing-to-docker-hub).
 
+
+## BuildKit
+
+Starting with version 18.09, Docker supports a new backend for executing your
+builds that is provided by the [moby/buildkit](https://github.com/moby/buildkit)
+project. The BuildKit backend provides many benefits compared to the old
+implementation. For example, BuildKit can:
+
+* Detect and skip executing unused build stages
+* Parallelize building independent build stages
+* Incrementally transfer only the changed files in your build context between builds
+* Detect and skip transferring unused files in your build context
+* Use external Dockerfile implementations with many new features
+* Avoid side-effects with rest of the API (intermediate images and containers)
+* Prioritize your build cache for automatic pruning
+
+To use the BuildKit backend, you need to set an environment variable 
+`DOCKER_BUILDKIT=1` on the CLI before invoking `docker build`.
+
+To learn about the experimental Dockerfile syntax available to BuildKit-based
+builds [refer to the documentation in the BuildKit repository](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md).
+
 ## Format
 
 Here is the format of the `Dockerfile`:
@@ -224,10 +246,64 @@ following lines are all treated identically:
 #	  dIrEcTiVe=value
 ```
 
-The following parser directive is supported:
+The following parser directives are supported:
 
+* `syntax`
 * `escape`
 
+## syntax
+
+    # syntax=[remote image reference]
+
+For example:
+
+    # syntax=docker/dockerfile
+    # syntax=docker/dockerfile:1.0
+    # syntax=docker.io/docker/dockerfile:1
+    # syntax=docker/dockerfile:1.0.0-experimental
+    # syntax=example.com/user/repo:tag@sha256:abcdef...
+
+This feature is only enabled if the [BuildKit](#buildkit) backend is used.
+
+The syntax directive defines the location of the Dockerfile builder that is used for
+building the current Dockerfile. The BuildKit backend allows to seamlessly use
+external implementations of builders that are distributed as Docker images and 
+execute inside a container sandbox environment.
+
+Custom Dockerfile implementation allows you to:
+  - Automatically get bugfixes without updating the daemon
+  - Make sure all users are using the same implementation to build your Dockerfile
+  - Use the latest features without updating the daemon
+  - Try out new experimental or third-party features
+
+### Official releases
+
+Docker distributes official versions of the images that can be used for building 
+Dockerfiles under `docker/dockerfile` repository on Docker Hub. There are two 
+channels where new images are released: stable and experimental. 
+
+Stable channel follows semantic versioning. For example:
+
+  - docker/dockerfile:1.0.0 - only allow immutable version 1.0.0
+  - docker/dockerfile:1.0 - allow versions 1.0.*
+  - docker/dockerfile:1 - allow versions 1.*.*
+  - docker/dockerfile:latest - latest release on stable channel
+
+The experimental channel uses incremental versioning with the major and minor
+component from the stable channel on the time of the release. For example:
+
+  - docker/dockerfile:1.0.1-experimental - only allow immutable version 1.0.1-experimental
+  - docker/dockerfile:1.0-experimental - latest experimental releases after 1.0
+  - docker/dockerfile:experimental - latest release on experimental channel
+
+You should choose a channel that best fits your needs. If you only want 
+bugfixes, you should use `docker/dockerfile:1.0`. If you want to benefit from
+experimental features, you should use the experimental channel. If you are using 
+the experimental channel, newer releases may not be backwards compatible, so it
+is recommended to use an immutable full version variant.
+
+For master builds and nightly feature releases refer to the description in [the source repository](https://github.com/moby/buildkit/blob/master/README.md).
+
 ## escape
 
     # escape=\ (backslash)
@@ -1339,6 +1415,10 @@ The table below shows what command is executed for different `ENTRYPOINT` / `CMD
 | **CMD ["p1_cmd", "p2_cmd"]**   | p1_cmd p2_cmd              | /bin/sh -c exec_entry p1_entry | exec_entry p1_entry p1_cmd p2_cmd              |
 | **CMD exec_cmd p1_cmd**        | /bin/sh -c exec_cmd p1_cmd | /bin/sh -c exec_entry p1_entry | exec_entry p1_entry /bin/sh -c exec_cmd p1_cmd |
 
+> **Note**: If `CMD` is defined from the base image, setting `ENTRYPOINT` will
+> reset `CMD` to an empty value. In this scenario, `CMD` must be defined in the
+> current image to have a value.
+
 ## VOLUME
 
     VOLUME ["/data"]
@@ -1623,6 +1703,38 @@ RUN echo "Hello World"
 When building this Dockerfile, the `HTTP_PROXY` is preserved in the
 `docker history`, and changing its value invalidates the build cache.
 
+### Automatic platform ARGs in the global scope
+
+This feature is only available when using the [BuildKit](#buildkit) backend.
+
+Docker predefines a set of `ARG` variables with information on the platform of
+the node performing the build (build platform) and on the platform of the
+resulting image (target platform). The target platform can be specified with
+the `--platform` flag on `docker build`.
+
+The following `ARG` variables are set automatically:
+
+* `TARGETPLATFORM` - platform of the build result. Eg `linux/amd64`, `linux/arm/v7`, `windows/amd64`.
+* `TARGETOS` - OS component of TARGETPLATFORM
+* `TARGETARCH` - architecture component of TARGETPLATFORM
+* `TARGETVARIANT` - variant component of TARGETPLATFORM
+* `BUILDPLATFORM` - platform of the node performing the build.
+* `BUILDOS` - OS component of BUILDPLATFORM
+* `BUILDARCH` - OS component of BUILDPLATFORM
+* `BUILDVARIANT` - OS component of BUILDPLATFORM
+
+These arguments are defined in the global scope so are not automatically
+available inside build stages or for your `RUN` commands. To expose one of
+these arguments inside the build stage redefine it without value.
+
+For example:
+
+```Dockerfile
+FROM alpine
+ARG TARGETPLATFORM
+RUN echo "I'm building for $TARGETPLATFORM"
+```
+
 ### Impact on build caching
 
 `ARG` variables are not persisted into the built image as `ENV` variables are.
@@ -1931,6 +2043,14 @@ required such as `zsh`, `csh`, `tcsh` and others.
 
 The `SHELL` feature was added in Docker 1.12.
 
+## External implementation features
+
+This feature is only available when using the  [BuildKit](#buildkit) backend.
+
+Docker build supports experimental features like cache mounts, build secrets and
+ssh forwarding that are enabled by using an external implementation of the 
+builder with a syntax directive. To learn about these features, [refer to the documentation in BuildKit repository](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md).
+
 ## Dockerfile examples
 
 Below you can see some examples of Dockerfile syntax. If you're interested in

docs/reference/commandline/dockerd.md

@@ -1305,8 +1305,13 @@ This is a full example of the allowed configuration options on Linux:
 	"storage-opts": [],
 	"labels": [],
 	"live-restore": true,
-	"log-driver": "",
-	"log-opts": {},
+	"log-driver": "json-file",
+	"log-opts": {
+		"max-size": "10m",
+		"max-files":"5",
+		"labels": "somelabel",
+		"env": "os,customer"
+	},
 	"mtu": 0,
 	"pidfile": "",
 	"cluster-store": "",

docs/yaml/generate.go

@@ -22,6 +22,7 @@ func generateCliYaml(opts *options) error {
 	dockerCli := command.NewDockerCli(stdin, stdout, stderr, false, nil)
 	cmd := &cobra.Command{Use: "docker"}
 	commands.AddCommands(cmd, dockerCli)
+	disableFlagsInUseLine(cmd)
 	source := filepath.Join(opts.source, descriptionSourcePath)
 	if err := loadLongDescription(cmd, source); err != nil {
 		return err
@@ -31,6 +32,23 @@ func generateCliYaml(opts *options) error {
 	return GenYamlTree(cmd, opts.target)
 }
 
+func disableFlagsInUseLine(cmd *cobra.Command) {
+	visitAll(cmd, func(ccmd *cobra.Command) {
+		// do not add a `[flags]` to the end of the usage line.
+		ccmd.DisableFlagsInUseLine = true
+	})
+}
+
+// visitAll will traverse all commands from the root.
+// This is different from the VisitAll of cobra.Command where only parents
+// are checked.
+func visitAll(root *cobra.Command, fn func(*cobra.Command)) {
+	for _, cmd := range root.Commands() {
+		visitAll(cmd, fn)
+	}
+	fn(root)
+}
+
 func loadLongDescription(cmd *cobra.Command, path ...string) error {
 	for _, cmd := range cmd.Commands() {
 		if cmd.Name() == "" {

e2e/compose-env.connhelper-ssh.yaml

@@ -0,0 +1,9 @@
+version: '2.1'
+
+services:
+  engine:
+      build:
+        context: ./testdata
+        dockerfile: Dockerfile.connhelper-ssh
+      environment:
+        - TEST_CONNHELPER_SSH_ID_RSA_PUB

e2e/testdata/Dockerfile.connhelper-ssh

@@ -0,0 +1,14 @@
+FROM docker:test-dind
+RUN apk --no-cache add shadow openssh-server && \
+  groupadd -f docker && \
+  useradd -m penguin && \
+  usermod -aG docker penguin && \
+  usermod -p $(head -c32 /dev/urandom | base64) penguin && \
+  chsh -s /bin/sh penguin && \
+  ssh-keygen -A
+# workaround: ssh session excludes /usr/local/bin from $PATH
+RUN  ln -s /usr/local/bin/docker /usr/bin/docker
+COPY ./connhelper-ssh/entrypoint.sh /
+EXPOSE 22
+ENTRYPOINT ["/entrypoint.sh"]
+# usage: docker run --privileged -e TEST_CONNHELPER_SSH_ID_RSA_PUB=$(cat ~/.ssh/id_rsa.pub) -p 22 $THIS_IMAGE

e2e/testdata/connhelper-ssh/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -ex
+mkdir -m 0700 -p /home/penguin/.ssh
+echo ${TEST_CONNHELPER_SSH_ID_RSA_PUB} > /home/penguin/.ssh/authorized_keys
+chmod 0600 /home/penguin/.ssh/authorized_keys
+chown -R penguin:penguin /home/penguin
+/usr/sbin/sshd -E /var/log/sshd.log
+exec dockerd-entrypoint.sh $@

internal/test/environment/testenv.go

@@ -1,15 +1,13 @@
 package environment
 
 import (
-	"context"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
-	"gotest.tools/assert"
+	"gotest.tools/icmd"
 	"gotest.tools/poll"
 	"gotest.tools/skip"
 )
@@ -79,21 +77,14 @@ func boolFromString(val string) bool {
 	}
 }
 
-func dockerClient(t *testing.T) client.APIClient {
-	t.Helper()
-	c, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion("1.37"))
-	assert.NilError(t, err)
-	return c
-}
-
 // DefaultPollSettings used with gotestyourself/poll
 var DefaultPollSettings = poll.WithDelay(100 * time.Millisecond)
 
 // SkipIfNotExperimentalDaemon returns whether the test docker daemon is in experimental mode
 func SkipIfNotExperimentalDaemon(t *testing.T) {
 	t.Helper()
-	c := dockerClient(t)
-	info, err := c.Info(context.Background())
-	assert.NilError(t, err)
-	skip.If(t, !info.ExperimentalBuild, "running against a non-experimental daemon")
+	result := icmd.RunCmd(icmd.Command("docker", "info", "--format", "{{.ExperimentalBuild}}"))
+	result.Assert(t, icmd.Expected{Err: icmd.None})
+	experimentalBuild := strings.TrimSpace(result.Stdout()) == "true"
+	skip.If(t, !experimentalBuild, "running against a non-experimental daemon")
 }

opts/hosts.go

@@ -77,6 +77,8 @@ func parseDockerDaemonHost(addr string) (string, error) {
 		return parseSimpleProtoAddr("npipe", addrParts[1], DefaultNamedPipe)
 	case "fd":
 		return addr, nil
+	case "ssh":
+		return addr, nil
 	default:
 		return "", fmt.Errorf("Invalid bind address format: %s", addr)
 	}

scripts/test/e2e/run

@@ -17,7 +17,15 @@ function setup {
     local project=$1
     local file=$2
 
-    test "${DOCKERD_EXPERIMENTAL:-}" -eq "1" && file="${file}:./e2e/compose-env.experimental.yaml"
+    test "${DOCKERD_EXPERIMENTAL:-0}" -eq "1" && file="${file}:./e2e/compose-env.experimental.yaml"
+
+    if [[ "${TEST_CONNHELPER:-}" = "ssh" ]];then
+        test ! -f "${HOME}/.ssh/id_rsa" && ssh-keygen -t rsa -C docker-e2e-dummy -N "" -f "${HOME}/.ssh/id_rsa" -q
+        grep "^StrictHostKeyChecking no" "${HOME}/.ssh/config" > /dev/null 2>&1 || echo "StrictHostKeyChecking no" > "${HOME}/.ssh/config"
+        TEST_CONNHELPER_SSH_ID_RSA_PUB=$(cat "${HOME}/.ssh/id_rsa.pub")
+        export TEST_CONNHELPER_SSH_ID_RSA_PUB
+        file="${file}:./e2e/compose-env.connhelper-ssh.yaml"
+    fi
     COMPOSE_PROJECT_NAME=$project COMPOSE_FILE=$file docker-compose up --build -d >&2
 
     local network="${project}_default"
@@ -26,6 +34,9 @@ function setup {
 
     engine_ip="$(container_ip "${project}_engine_1" "$network")"
     engine_host="tcp://$engine_ip:2375"
+    if [[ "${TEST_CONNHELPER:-}" = "ssh" ]];then
+        engine_host="ssh://penguin@${engine_ip}"
+    fi
     (
         export DOCKER_HOST="$engine_host"
         timeout 200 ./scripts/test/e2e/wait-on-daemon
@@ -57,7 +68,7 @@ function runtests {
         TEST_REMOTE_DAEMON="${REMOTE_DAEMON-}" \
         TEST_SKIP_PLUGIN_TESTS="${SKIP_PLUGIN_TESTS-}" \
         GOPATH="$GOPATH" \
-        PATH="$PWD/build/" \
+        PATH="$PWD/build/:/usr/bin" \
         "$(which go)" test -v ./e2e/... ${TESTFLAGS-}
 }