Merge component 'engine' from git@github.com:moby/moby master

components/engine/builder/dockerfile/builder.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/builder/remotecontext"
 	"github.com/docker/docker/errdefs"
@@ -256,8 +257,8 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error
 	return aux.Emit(types.BuildResult{ID: state.imageID})
 }
 
-func processMetaArg(meta instructions.ArgCommand, shlex *ShellLex, args *buildArgs) error {
-	// ShellLex currently only support the concatenated string format
+func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *buildArgs) error {
+	// shell.Lex currently only support the concatenated string format
 	envs := convertMapToEnvList(args.GetAllAllowed())
 	if err := meta.Expand(func(word string) (string, error) {
 		return shlex.ProcessWord(word, envs)
@@ -283,7 +284,7 @@ func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.
 	for _, stage := range parseResult {
 		totalCommands += len(stage.Commands)
 	}
-	shlex := NewShellLex(escapeToken)
+	shlex := shell.NewLex(escapeToken)
 	for _, meta := range metaArgs {
 		currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta)
 

components/engine/builder/dockerfile/dispatchers.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/jsonmessage"
@@ -47,7 +48,7 @@ func dispatchEnv(d dispatchRequest, c *instructions.EnvCommand) error {
 		for i, envVar := range runConfig.Env {
 			envParts := strings.SplitN(envVar, "=", 2)
 			compareFrom := envParts[0]
-			if equalEnvKeys(compareFrom, name) {
+			if shell.EqualEnvKeys(compareFrom, name) {
 				runConfig.Env[i] = newVar
 				gotOne = true
 				break
@@ -197,7 +198,7 @@ func dispatchTriggeredOnBuild(d dispatchRequest, triggers []string) error {
 	return nil
 }
 
-func (d *dispatchRequest) getExpandedImageName(shlex *ShellLex, name string) (string, error) {
+func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (string, error) {
 	substitutionArgs := []string{}
 	for key, value := range d.state.buildArgs.GetAllMeta() {
 		substitutionArgs = append(substitutionArgs, key+"="+value)
@@ -242,7 +243,7 @@ func (d *dispatchRequest) getImageOrStage(name string) (builder.Image, error) {
 	}
 	return imageMount.Image(), nil
 }
-func (d *dispatchRequest) getFromImage(shlex *ShellLex, name string) (builder.Image, error) {
+func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string) (builder.Image, error) {
 	name, err := d.getExpandedImageName(shlex, name)
 	if err != nil {
 		return nil, err

components/engine/builder/dockerfile/dispatchers_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/api/types/strslice"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-connections/nat"
 	"github.com/stretchr/testify/assert"
@@ -141,7 +142,7 @@ func TestFromWithArg(t *testing.T) {
 	cmd := &instructions.Stage{
 		BaseName: "alpine:${THETAG}",
 	}
-	err := processMetaArg(metaArg, NewShellLex('\\'), args)
+	err := processMetaArg(metaArg, shell.NewLex('\\'), args)
 
 	sb := newDispatchRequest(b, '\\', nil, args, newStagesBuildResults())
 	require.NoError(t, err)

components/engine/builder/dockerfile/dispatchers_unix.go

@@ -21,9 +21,3 @@ func normalizeWorkdir(_ string, current string, requested string) (string, error
 	}
 	return requested, nil
 }
-
-// equalEnvKeys compare two strings and returns true if they are equal. On
-// Windows this comparison is case insensitive.
-func equalEnvKeys(from, to string) bool {
-	return from == to
-}

components/engine/builder/dockerfile/dispatchers_windows.go

@@ -93,9 +93,3 @@ func normalizeWorkdirWindows(current string, requested string) (string, error) {
 	// Upper-case drive letter
 	return (strings.ToUpper(string(requested[0])) + requested[1:]), nil
 }
-
-// equalEnvKeys compare two strings and returns true if they are equal. On
-// Windows this comparison is case insensitive.
-func equalEnvKeys(from, to string) bool {
-	return strings.ToUpper(from) == strings.ToUpper(to)
-}

components/engine/builder/dockerfile/evaluator.go

@@ -28,6 +28,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/runconfig/opts"
@@ -187,7 +188,7 @@ func commitStage(state *dispatchState, stages *stagesBuildResults) error {
 
 type dispatchRequest struct {
 	state   *dispatchState
-	shlex   *ShellLex
+	shlex   *shell.Lex
 	builder *Builder
 	source  builder.Source
 	stages  *stagesBuildResults
@@ -196,7 +197,7 @@ type dispatchRequest struct {
 func newDispatchRequest(builder *Builder, escapeToken rune, source builder.Source, buildArgs *buildArgs, stages *stagesBuildResults) dispatchRequest {
 	return dispatchRequest{
 		state:   newDispatchState(buildArgs),
-		shlex:   NewShellLex(escapeToken),
+		shlex:   shell.NewLex(escapeToken),
 		builder: builder,
 		source:  source,
 		stages:  stages,

components/engine/builder/dockerfile/shell/equal_env_unix.go

@@ -0,0 +1,9 @@
+// +build !windows
+
+package shell
+
+// EqualEnvKeys compare two strings and returns true if they are equal. On
+// Windows this comparison is case insensitive.
+func EqualEnvKeys(from, to string) bool {
+	return from == to
+}

components/engine/builder/dockerfile/shell/equal_env_windows.go

@@ -0,0 +1,9 @@
+package shell
+
+import "strings"
+
+// EqualEnvKeys compare two strings and returns true if they are equal. On
+// Windows this comparison is case insensitive.
+func EqualEnvKeys(from, to string) bool {
+	return strings.ToUpper(from) == strings.ToUpper(to)
+}

components/engine/builder/dockerfile/shell/lex.go

@@ -1,4 +1,4 @@
-package dockerfile
+package shell
 
 import (
 	"bytes"
@@ -9,25 +9,25 @@ import (
 	"github.com/pkg/errors"
 )
 
-// ShellLex performs shell word splitting and variable expansion.
+// Lex performs shell word splitting and variable expansion.
 //
-// ShellLex takes a string and an array of env variables and
+// Lex takes a string and an array of env variables and
 // process all quotes (" and ') as well as $xxx and ${xxx} env variable
 // tokens.  Tries to mimic bash shell process.
 // It doesn't support all flavors of ${xx:...} formats but new ones can
 // be added by adding code to the "special ${} format processing" section
-type ShellLex struct {
+type Lex struct {
 	escapeToken rune
 }
 
-// NewShellLex creates a new ShellLex which uses escapeToken to escape quotes.
-func NewShellLex(escapeToken rune) *ShellLex {
-	return &ShellLex{escapeToken: escapeToken}
+// NewLex creates a new Lex which uses escapeToken to escape quotes.
+func NewLex(escapeToken rune) *Lex {
+	return &Lex{escapeToken: escapeToken}
 }
 
 // ProcessWord will use the 'env' list of environment variables,
 // and replace any env var references in 'word'.
-func (s *ShellLex) ProcessWord(word string, env []string) (string, error) {
+func (s *Lex) ProcessWord(word string, env []string) (string, error) {
 	word, _, err := s.process(word, env)
 	return word, err
 }
@@ -39,12 +39,12 @@ func (s *ShellLex) ProcessWord(word string, env []string) (string, error) {
 // this splitting is done **after** the env var substitutions are done.
 // Note, each one is trimmed to remove leading and trailing spaces (unless
 // they are quoted", but ProcessWord retains spaces between words.
-func (s *ShellLex) ProcessWords(word string, env []string) ([]string, error) {
+func (s *Lex) ProcessWords(word string, env []string) ([]string, error) {
 	_, words, err := s.process(word, env)
 	return words, err
 }
 
-func (s *ShellLex) process(word string, env []string) (string, []string, error) {
+func (s *Lex) process(word string, env []string) (string, []string, error) {
 	sw := &shellWord{
 		envs:        env,
 		escapeToken: s.escapeToken,
@@ -327,7 +327,7 @@ func (sw *shellWord) getEnv(name string) string {
 	for _, env := range sw.envs {
 		i := strings.Index(env, "=")
 		if i < 0 {
-			if equalEnvKeys(name, env) {
+			if EqualEnvKeys(name, env) {
 				// Should probably never get here, but just in case treat
 				// it like "var" and "var=" are the same
 				return ""
@@ -335,7 +335,7 @@ func (sw *shellWord) getEnv(name string) string {
 			continue
 		}
 		compareName := env[:i]
-		if !equalEnvKeys(name, compareName) {
+		if !EqualEnvKeys(name, compareName) {
 			continue
 		}
 		return env[i+1:]

components/engine/builder/dockerfile/shell/lex_test.go

@@ -1,4 +1,4 @@
-package dockerfile
+package shell
 
 import (
 	"bufio"
@@ -18,7 +18,7 @@ func TestShellParser4EnvVars(t *testing.T) {
 	assert.NoError(t, err)
 	defer file.Close()
 
-	shlex := NewShellLex('\\')
+	shlex := NewLex('\\')
 	scanner := bufio.NewScanner(file)
 	envs := []string{"PWD=/home", "SHELL=bash", "KOREAN=한국어"}
 	for scanner.Scan() {
@@ -70,7 +70,7 @@ func TestShellParser4Words(t *testing.T) {
 	}
 	defer file.Close()
 
-	shlex := NewShellLex('\\')
+	shlex := NewLex('\\')
 	envs := []string{}
 	scanner := bufio.NewScanner(file)
 	lineNum := 0

components/engine/cmd/dockerd/config.go

@@ -67,7 +67,7 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) {
 
 	flags.StringVar(&conf.MetricsAddress, "metrics-addr", "", "Set default address and port to serve the metrics api on")
 
-	flags.Var(opts.NewListOptsRef(&conf.NodeGenericResources, opts.ValidateSingleGenericResource), "node-generic-resources", "Advertise user-defined resource")
+	flags.Var(opts.NewNamedListOptsRef("node-generic-resources", &conf.NodeGenericResources, opts.ValidateSingleGenericResource), "node-generic-resource", "Advertise user-defined resource")
 
 	flags.IntVar(&conf.NetworkControlPlaneMTU, "network-control-plane-mtu", config.DefaultNetworkMtu, "Network Control plane MTU")
 

components/engine/cmd/dockerd/daemon_test.go

@@ -61,6 +61,22 @@ func TestLoadDaemonCliConfigWithConflicts(t *testing.T) {
 	testutil.ErrorContains(t, err, "as a flag and in the configuration file: labels")
 }
 
+func TestLoadDaemonCliWithConflictingNodeGenericResources(t *testing.T) {
+	tempFile := fs.NewFile(t, "config", fs.WithContent(`{"node-generic-resources": ["foo=bar", "bar=baz"]}`))
+	defer tempFile.Remove()
+	configFile := tempFile.Path()
+
+	opts := defaultOptions(configFile)
+	flags := opts.flags
+
+	assert.NoError(t, flags.Set("config-file", configFile))
+	assert.NoError(t, flags.Set("node-generic-resource", "r1=bar"))
+	assert.NoError(t, flags.Set("node-generic-resource", "r2=baz"))
+
+	_, err := loadDaemonCliConfig(opts)
+	testutil.ErrorContains(t, err, "as a flag and in the configuration file: node-generic-resources")
+}
+
 func TestLoadDaemonCliWithConflictingLabels(t *testing.T) {
 	opts := defaultOptions("")
 	flags := opts.flags

components/engine/integration-cli/docker_api_auth_test.go

@@ -1,25 +0,0 @@
-package main
-
-import (
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
-	"github.com/docker/docker/integration-cli/checker"
-	"github.com/go-check/check"
-	"golang.org/x/net/context"
-)
-
-// Test case for #22244
-func (s *DockerSuite) TestAuthAPI(c *check.C) {
-	testRequires(c, Network)
-	config := types.AuthConfig{
-		Username: "no-user",
-		Password: "no-password",
-	}
-	cli, err := client.NewEnvClient()
-	c.Assert(err, checker.IsNil)
-	defer cli.Close()
-
-	_, err = cli.RegistryLogin(context.Background(), config)
-	expected := "Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password"
-	c.Assert(err.Error(), checker.Contains, expected)
-}

components/engine/integration-cli/docker_cli_links_unix_test.go

@@ -1,26 +0,0 @@
-// +build !windows
-
-package main
-
-import (
-	"io/ioutil"
-	"os"
-
-	"github.com/docker/docker/integration-cli/checker"
-	"github.com/go-check/check"
-)
-
-func (s *DockerSuite) TestLinksEtcHostsContentMatch(c *check.C) {
-	// In a _unix file as using Unix specific files, and must be on the
-	// same host as the daemon.
-	testRequires(c, SameHostDaemon, NotUserNamespace)
-
-	out, _ := dockerCmd(c, "run", "--net=host", "busybox", "cat", "/etc/hosts")
-	hosts, err := ioutil.ReadFile("/etc/hosts")
-	if os.IsNotExist(err) {
-		c.Skip("/etc/hosts does not exist, skip this test")
-	}
-
-	c.Assert(out, checker.Equals, string(hosts), check.Commentf("container: %s\n\nhost:%s", out, hosts))
-
-}

components/engine/integration/container/links_linux_test.go

@@ -0,0 +1,59 @@
+package container
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/integration/util/request"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/gotestyourself/gotestyourself/poll"
+	"github.com/gotestyourself/gotestyourself/skip"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLinksEtcHostsContentMatch(t *testing.T) {
+	skip.If(t, !testEnv.IsLocalDaemon())
+
+	hosts, err := ioutil.ReadFile("/etc/hosts")
+	skip.If(t, os.IsNotExist(err))
+
+	defer setupTest(t)()
+	client := request.NewAPIClient(t)
+	ctx := context.Background()
+
+	c, err := client.ContainerCreate(ctx,
+		&container.Config{
+			Image: "busybox",
+			Cmd:   []string{"cat", "/etc/hosts"},
+		},
+		&container.HostConfig{
+			NetworkMode: "host",
+		},
+		nil,
+		"")
+	require.NoError(t, err)
+
+	err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{})
+	require.NoError(t, err)
+
+	poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond))
+
+	body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{
+		ShowStdout: true,
+	})
+	require.NoError(t, err)
+	defer body.Close()
+
+	var b bytes.Buffer
+	_, err = stdcopy.StdCopy(&b, ioutil.Discard, body)
+	require.NoError(t, err)
+
+	assert.Equal(t, string(hosts), b.String())
+}

components/engine/integration/system/login_test.go

@@ -0,0 +1,27 @@
+package system
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/integration/util/request"
+	"github.com/docker/docker/integration/util/requirement"
+	"github.com/gotestyourself/gotestyourself/skip"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+)
+
+// Test case for GitHub 22244
+func TestLoginFailsWithBadCredentials(t *testing.T) {
+	skip.IfCondition(t, !requirement.HasHubConnectivity(t))
+
+	client := request.NewAPIClient(t)
+
+	config := types.AuthConfig{
+		Username: "no-user",
+		Password: "no-password",
+	}
+	_, err := client.RegistryLogin(context.Background(), config)
+	expected := "Error response from daemon: Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password"
+	assert.EqualError(t, err, expected)
+}