diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index 374f301ac5..a32210ef96 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/builder/dockerfile/shell" "github.com/docker/docker/builder/fscache" "github.com/docker/docker/builder/remotecontext" "github.com/docker/docker/errdefs" @@ -256,8 +257,8 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error return aux.Emit(types.BuildResult{ID: state.imageID}) } -func processMetaArg(meta instructions.ArgCommand, shlex *ShellLex, args *buildArgs) error { - // ShellLex currently only support the concatenated string format +func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *buildArgs) error { + // shell.Lex currently only support the concatenated string format envs := convertMapToEnvList(args.GetAllAllowed()) if err := meta.Expand(func(word string) (string, error) { return shlex.ProcessWord(word, envs) @@ -283,7 +284,7 @@ func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions. for _, stage := range parseResult { totalCommands += len(stage.Commands) } - shlex := NewShellLex(escapeToken) + shlex := shell.NewLex(escapeToken) for _, meta := range metaArgs { currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta) diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index 4239e3ccc7..8e37097dd4 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/builder/dockerfile/shell" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/pkg/jsonmessage" @@ -47,7 +48,7 @@ func dispatchEnv(d dispatchRequest, c *instructions.EnvCommand) error { for i, envVar := range runConfig.Env { envParts := strings.SplitN(envVar, "=", 2) compareFrom := envParts[0] - if equalEnvKeys(compareFrom, name) { + if shell.EqualEnvKeys(compareFrom, name) { runConfig.Env[i] = newVar gotOne = true break @@ -197,7 +198,7 @@ func dispatchTriggeredOnBuild(d dispatchRequest, triggers []string) error { return nil } -func (d *dispatchRequest) getExpandedImageName(shlex *ShellLex, name string) (string, error) { +func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (string, error) { substitutionArgs := []string{} for key, value := range d.state.buildArgs.GetAllMeta() { substitutionArgs = append(substitutionArgs, key+"="+value) @@ -242,7 +243,7 @@ func (d *dispatchRequest) getImageOrStage(name string) (builder.Image, error) { } return imageMount.Image(), nil } -func (d *dispatchRequest) getFromImage(shlex *ShellLex, name string) (builder.Image, error) { +func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string) (builder.Image, error) { name, err := d.getExpandedImageName(shlex, name) if err != nil { return nil, err diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index 29cecd8fcd..0cdf2ad437 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -12,6 +12,7 @@ import ( "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/docker/docker/builder/dockerfile/shell" "github.com/docker/docker/pkg/system" "github.com/docker/go-connections/nat" "github.com/stretchr/testify/assert" @@ -141,7 +142,7 @@ func TestFromWithArg(t *testing.T) { cmd := &instructions.Stage{ BaseName: "alpine:${THETAG}", } - err := processMetaArg(metaArg, NewShellLex('\\'), args) + err := processMetaArg(metaArg, shell.NewLex('\\'), args) sb := newDispatchRequest(b, '\\', nil, args, newStagesBuildResults()) require.NoError(t, err) diff --git a/components/engine/builder/dockerfile/dispatchers_unix.go b/components/engine/builder/dockerfile/dispatchers_unix.go index 6f0d581b94..6a25b598b8 100644 --- a/components/engine/builder/dockerfile/dispatchers_unix.go +++ b/components/engine/builder/dockerfile/dispatchers_unix.go @@ -21,9 +21,3 @@ func normalizeWorkdir(_ string, current string, requested string) (string, error } return requested, nil } - -// equalEnvKeys compare two strings and returns true if they are equal. On -// Windows this comparison is case insensitive. -func equalEnvKeys(from, to string) bool { - return from == to -} diff --git a/components/engine/builder/dockerfile/dispatchers_windows.go b/components/engine/builder/dockerfile/dispatchers_windows.go index 8f6eaac180..256ebda9af 100644 --- a/components/engine/builder/dockerfile/dispatchers_windows.go +++ b/components/engine/builder/dockerfile/dispatchers_windows.go @@ -93,9 +93,3 @@ func normalizeWorkdirWindows(current string, requested string) (string, error) { // Upper-case drive letter return (strings.ToUpper(string(requested[0])) + requested[1:]), nil } - -// equalEnvKeys compare two strings and returns true if they are equal. On -// Windows this comparison is case insensitive. -func equalEnvKeys(from, to string) bool { - return strings.ToUpper(from) == strings.ToUpper(to) -} diff --git a/components/engine/builder/dockerfile/evaluator.go b/components/engine/builder/dockerfile/evaluator.go index f60869d7c4..8a370d2d5d 100644 --- a/components/engine/builder/dockerfile/evaluator.go +++ b/components/engine/builder/dockerfile/evaluator.go @@ -28,6 +28,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/docker/docker/builder/dockerfile/shell" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig/opts" @@ -187,7 +188,7 @@ func commitStage(state *dispatchState, stages *stagesBuildResults) error { type dispatchRequest struct { state *dispatchState - shlex *ShellLex + shlex *shell.Lex builder *Builder source builder.Source stages *stagesBuildResults @@ -196,7 +197,7 @@ type dispatchRequest struct { func newDispatchRequest(builder *Builder, escapeToken rune, source builder.Source, buildArgs *buildArgs, stages *stagesBuildResults) dispatchRequest { return dispatchRequest{ state: newDispatchState(buildArgs), - shlex: NewShellLex(escapeToken), + shlex: shell.NewLex(escapeToken), builder: builder, source: source, stages: stages, diff --git a/components/engine/builder/dockerfile/envVarTest b/components/engine/builder/dockerfile/shell/envVarTest similarity index 100% rename from components/engine/builder/dockerfile/envVarTest rename to components/engine/builder/dockerfile/shell/envVarTest diff --git a/components/engine/builder/dockerfile/shell/equal_env_unix.go b/components/engine/builder/dockerfile/shell/equal_env_unix.go new file mode 100644 index 0000000000..6e3f6b890e --- /dev/null +++ b/components/engine/builder/dockerfile/shell/equal_env_unix.go @@ -0,0 +1,9 @@ +// +build !windows + +package shell + +// EqualEnvKeys compare two strings and returns true if they are equal. On +// Windows this comparison is case insensitive. +func EqualEnvKeys(from, to string) bool { + return from == to +} diff --git a/components/engine/builder/dockerfile/shell/equal_env_windows.go b/components/engine/builder/dockerfile/shell/equal_env_windows.go new file mode 100644 index 0000000000..7780fb67e8 --- /dev/null +++ b/components/engine/builder/dockerfile/shell/equal_env_windows.go @@ -0,0 +1,9 @@ +package shell + +import "strings" + +// EqualEnvKeys compare two strings and returns true if they are equal. On +// Windows this comparison is case insensitive. +func EqualEnvKeys(from, to string) bool { + return strings.ToUpper(from) == strings.ToUpper(to) +} diff --git a/components/engine/builder/dockerfile/shell_parser.go b/components/engine/builder/dockerfile/shell/lex.go similarity index 92% rename from components/engine/builder/dockerfile/shell_parser.go rename to components/engine/builder/dockerfile/shell/lex.go index b72ac291d9..e4a5a36942 100644 --- a/components/engine/builder/dockerfile/shell_parser.go +++ b/components/engine/builder/dockerfile/shell/lex.go @@ -1,4 +1,4 @@ -package dockerfile +package shell import ( "bytes" @@ -9,25 +9,25 @@ import ( "github.com/pkg/errors" ) -// ShellLex performs shell word splitting and variable expansion. +// Lex performs shell word splitting and variable expansion. // -// ShellLex takes a string and an array of env variables and +// Lex takes a string and an array of env variables and // process all quotes (" and ') as well as $xxx and ${xxx} env variable // tokens. Tries to mimic bash shell process. // It doesn't support all flavors of ${xx:...} formats but new ones can // be added by adding code to the "special ${} format processing" section -type ShellLex struct { +type Lex struct { escapeToken rune } -// NewShellLex creates a new ShellLex which uses escapeToken to escape quotes. -func NewShellLex(escapeToken rune) *ShellLex { - return &ShellLex{escapeToken: escapeToken} +// NewLex creates a new Lex which uses escapeToken to escape quotes. +func NewLex(escapeToken rune) *Lex { + return &Lex{escapeToken: escapeToken} } // ProcessWord will use the 'env' list of environment variables, // and replace any env var references in 'word'. -func (s *ShellLex) ProcessWord(word string, env []string) (string, error) { +func (s *Lex) ProcessWord(word string, env []string) (string, error) { word, _, err := s.process(word, env) return word, err } @@ -39,12 +39,12 @@ func (s *ShellLex) ProcessWord(word string, env []string) (string, error) { // this splitting is done **after** the env var substitutions are done. // Note, each one is trimmed to remove leading and trailing spaces (unless // they are quoted", but ProcessWord retains spaces between words. -func (s *ShellLex) ProcessWords(word string, env []string) ([]string, error) { +func (s *Lex) ProcessWords(word string, env []string) ([]string, error) { _, words, err := s.process(word, env) return words, err } -func (s *ShellLex) process(word string, env []string) (string, []string, error) { +func (s *Lex) process(word string, env []string) (string, []string, error) { sw := &shellWord{ envs: env, escapeToken: s.escapeToken, @@ -327,7 +327,7 @@ func (sw *shellWord) getEnv(name string) string { for _, env := range sw.envs { i := strings.Index(env, "=") if i < 0 { - if equalEnvKeys(name, env) { + if EqualEnvKeys(name, env) { // Should probably never get here, but just in case treat // it like "var" and "var=" are the same return "" @@ -335,7 +335,7 @@ func (sw *shellWord) getEnv(name string) string { continue } compareName := env[:i] - if !equalEnvKeys(name, compareName) { + if !EqualEnvKeys(name, compareName) { continue } return env[i+1:] diff --git a/components/engine/builder/dockerfile/shell_parser_test.go b/components/engine/builder/dockerfile/shell/lex_test.go similarity index 97% rename from components/engine/builder/dockerfile/shell_parser_test.go rename to components/engine/builder/dockerfile/shell/lex_test.go index c4f7e0efd4..14d1a7d12a 100644 --- a/components/engine/builder/dockerfile/shell_parser_test.go +++ b/components/engine/builder/dockerfile/shell/lex_test.go @@ -1,4 +1,4 @@ -package dockerfile +package shell import ( "bufio" @@ -18,7 +18,7 @@ func TestShellParser4EnvVars(t *testing.T) { assert.NoError(t, err) defer file.Close() - shlex := NewShellLex('\\') + shlex := NewLex('\\') scanner := bufio.NewScanner(file) envs := []string{"PWD=/home", "SHELL=bash", "KOREAN=한국어"} for scanner.Scan() { @@ -70,7 +70,7 @@ func TestShellParser4Words(t *testing.T) { } defer file.Close() - shlex := NewShellLex('\\') + shlex := NewLex('\\') envs := []string{} scanner := bufio.NewScanner(file) lineNum := 0 diff --git a/components/engine/builder/dockerfile/wordsTest b/components/engine/builder/dockerfile/shell/wordsTest similarity index 100% rename from components/engine/builder/dockerfile/wordsTest rename to components/engine/builder/dockerfile/shell/wordsTest diff --git a/components/engine/cmd/dockerd/config.go b/components/engine/cmd/dockerd/config.go index c55332e183..abdac9a7fb 100644 --- a/components/engine/cmd/dockerd/config.go +++ b/components/engine/cmd/dockerd/config.go @@ -67,7 +67,7 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.StringVar(&conf.MetricsAddress, "metrics-addr", "", "Set default address and port to serve the metrics api on") - flags.Var(opts.NewListOptsRef(&conf.NodeGenericResources, opts.ValidateSingleGenericResource), "node-generic-resources", "Advertise user-defined resource") + flags.Var(opts.NewNamedListOptsRef("node-generic-resources", &conf.NodeGenericResources, opts.ValidateSingleGenericResource), "node-generic-resource", "Advertise user-defined resource") flags.IntVar(&conf.NetworkControlPlaneMTU, "network-control-plane-mtu", config.DefaultNetworkMtu, "Network Control plane MTU") diff --git a/components/engine/cmd/dockerd/daemon_test.go b/components/engine/cmd/dockerd/daemon_test.go index e5e4aa34e2..b065831871 100644 --- a/components/engine/cmd/dockerd/daemon_test.go +++ b/components/engine/cmd/dockerd/daemon_test.go @@ -61,6 +61,22 @@ func TestLoadDaemonCliConfigWithConflicts(t *testing.T) { testutil.ErrorContains(t, err, "as a flag and in the configuration file: labels") } +func TestLoadDaemonCliWithConflictingNodeGenericResources(t *testing.T) { + tempFile := fs.NewFile(t, "config", fs.WithContent(`{"node-generic-resources": ["foo=bar", "bar=baz"]}`)) + defer tempFile.Remove() + configFile := tempFile.Path() + + opts := defaultOptions(configFile) + flags := opts.flags + + assert.NoError(t, flags.Set("config-file", configFile)) + assert.NoError(t, flags.Set("node-generic-resource", "r1=bar")) + assert.NoError(t, flags.Set("node-generic-resource", "r2=baz")) + + _, err := loadDaemonCliConfig(opts) + testutil.ErrorContains(t, err, "as a flag and in the configuration file: node-generic-resources") +} + func TestLoadDaemonCliWithConflictingLabels(t *testing.T) { opts := defaultOptions("") flags := opts.flags diff --git a/components/engine/integration-cli/docker_api_auth_test.go b/components/engine/integration-cli/docker_api_auth_test.go deleted file mode 100644 index a1f7a098cd..0000000000 --- a/components/engine/integration-cli/docker_api_auth_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/checker" - "github.com/go-check/check" - "golang.org/x/net/context" -) - -// Test case for #22244 -func (s *DockerSuite) TestAuthAPI(c *check.C) { - testRequires(c, Network) - config := types.AuthConfig{ - Username: "no-user", - Password: "no-password", - } - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - _, err = cli.RegistryLogin(context.Background(), config) - expected := "Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password" - c.Assert(err.Error(), checker.Contains, expected) -} diff --git a/components/engine/integration-cli/docker_cli_links_unix_test.go b/components/engine/integration-cli/docker_cli_links_unix_test.go deleted file mode 100644 index dbff2911a5..0000000000 --- a/components/engine/integration-cli/docker_cli_links_unix_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build !windows - -package main - -import ( - "io/ioutil" - "os" - - "github.com/docker/docker/integration-cli/checker" - "github.com/go-check/check" -) - -func (s *DockerSuite) TestLinksEtcHostsContentMatch(c *check.C) { - // In a _unix file as using Unix specific files, and must be on the - // same host as the daemon. - testRequires(c, SameHostDaemon, NotUserNamespace) - - out, _ := dockerCmd(c, "run", "--net=host", "busybox", "cat", "/etc/hosts") - hosts, err := ioutil.ReadFile("/etc/hosts") - if os.IsNotExist(err) { - c.Skip("/etc/hosts does not exist, skip this test") - } - - c.Assert(out, checker.Equals, string(hosts), check.Commentf("container: %s\n\nhost:%s", out, hosts)) - -} diff --git a/components/engine/integration/container/links_linux_test.go b/components/engine/integration/container/links_linux_test.go new file mode 100644 index 0000000000..4a56ee853b --- /dev/null +++ b/components/engine/integration/container/links_linux_test.go @@ -0,0 +1,59 @@ +package container + +import ( + "bytes" + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/integration/util/request" + "github.com/docker/docker/pkg/stdcopy" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLinksEtcHostsContentMatch(t *testing.T) { + skip.If(t, !testEnv.IsLocalDaemon()) + + hosts, err := ioutil.ReadFile("/etc/hosts") + skip.If(t, os.IsNotExist(err)) + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + c, err := client.ContainerCreate(ctx, + &container.Config{ + Image: "busybox", + Cmd: []string{"cat", "/etc/hosts"}, + }, + &container.HostConfig{ + NetworkMode: "host", + }, + nil, + "") + require.NoError(t, err) + + err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) + require.NoError(t, err) + + poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond)) + + body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{ + ShowStdout: true, + }) + require.NoError(t, err) + defer body.Close() + + var b bytes.Buffer + _, err = stdcopy.StdCopy(&b, ioutil.Discard, body) + require.NoError(t, err) + + assert.Equal(t, string(hosts), b.String()) +} diff --git a/components/engine/integration/system/login_test.go b/components/engine/integration/system/login_test.go new file mode 100644 index 0000000000..a3e296c3ac --- /dev/null +++ b/components/engine/integration/system/login_test.go @@ -0,0 +1,27 @@ +package system + +import ( + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/util/request" + "github.com/docker/docker/integration/util/requirement" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +// Test case for GitHub 22244 +func TestLoginFailsWithBadCredentials(t *testing.T) { + skip.IfCondition(t, !requirement.HasHubConnectivity(t)) + + client := request.NewAPIClient(t) + + config := types.AuthConfig{ + Username: "no-user", + Password: "no-password", + } + _, err := client.RegistryLogin(context.Background(), config) + expected := "Error response from daemon: Get https://registry-1.docker.io/v2/: unauthorized: incorrect username or password" + assert.EqualError(t, err, expected) +}