Merge component 'engine' from git@github.com:moby/moby master

components/engine/api/common.go

@@ -3,7 +3,7 @@ package api
 // Common constants for daemon and client.
 const (
 	// DefaultVersion of Current REST API
-	DefaultVersion string = "1.32"
+	DefaultVersion string = "1.33"
 
 	// NoBaseImageSpecifier is the symbol used by the FROM
 	// command to specify that no base image is to be used.

components/engine/api/server/httputils/write_log_stream.go

@@ -11,7 +11,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/jsonlog"
+	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/stdcopy"
 )
 
@@ -49,11 +49,7 @@ func WriteLogStream(_ context.Context, w io.Writer, msgs <-chan *backend.LogMess
 			logLine = append(logLine, msg.Line...)
 		}
 		if config.Timestamps {
-			// TODO(dperny) the format is defined in
-			// daemon/logger/logger.go as logger.TimeFormat. importing
-			// logger is verboten (not part of backend) so idk if just
-			// importing the same thing from jsonlog is good enough
-			logLine = append([]byte(msg.Timestamp.Format(jsonlog.RFC3339NanoFixed)+" "), logLine...)
+			logLine = append([]byte(msg.Timestamp.Format(jsonmessage.RFC3339NanoFixed)+" "), logLine...)
 		}
 		if msg.Source == "stdout" && config.ShowStdout {
 			outStream.Write(logLine)

components/engine/api/swagger.yaml

@@ -19,10 +19,10 @@ produces:
 consumes:
   - "application/json"
   - "text/plain"
-basePath: "/v1.32"
+basePath: "/v1.33"
 info:
   title: "Docker Engine API"
-  version: "1.32"
+  version: "1.33"
   x-logo:
     url: "https://docs.docker.com/images/logo-docker-main.png"
   description: |
@@ -44,7 +44,7 @@ info:
 
     The API is usually changed in each release of Docker, so API calls are versioned to ensure that clients don't break.
 
-    For Docker Engine 17.07, the API version is 1.31. To lock to this version, you prefix the URL with `/v1.31`. For example, calling `/info` is the same as calling `/v1.31/info`.
+    For Docker Engine 17.09, the API version is 1.32. To lock to this version, you prefix the URL with `/v1.32`. For example, calling `/info` is the same as calling `/v1.32/info`.
 
     Engine releases in the near future should support this version of the API, so your client will continue to work even if it is talking to a newer Engine.
 
@@ -52,10 +52,11 @@ info:
 
     The API uses an open schema model, which means server may add extra properties to responses. Likewise, the server will ignore any extra query parameters and request body properties. When you write clients, you need to ignore additional properties in responses to ensure they do not break when talking to newer Docker daemons.
 
-    This documentation is for version 1.32 of the API. Use this table to find documentation for previous versions of the API:
+    This documentation is for version 1.33 of the API. Use this table to find documentation for previous versions of the API:
 
     Docker version  | API version | Changes
     ----------------|-------------|---------
+    17.09.x | [1.31](https://docs.docker.com/engine/api/v1.32/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-32-api-changes)
     17.07.x | [1.31](https://docs.docker.com/engine/api/v1.31/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-31-api-changes)
     17.06.x | [1.30](https://docs.docker.com/engine/api/v1.30/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-30-api-changes)
     17.05.x | [1.29](https://docs.docker.com/engine/api/v1.29/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-29-api-changes)

components/engine/builder/dockerfile/buildargs.go

@@ -42,6 +42,26 @@ func newBuildArgs(argsFromOptions map[string]*string) *buildArgs {
 	}
 }
 
+func (b *buildArgs) Clone() *buildArgs {
+	result := newBuildArgs(b.argsFromOptions)
+	for k, v := range b.allowedBuildArgs {
+		result.allowedBuildArgs[k] = v
+	}
+	for k, v := range b.allowedMetaArgs {
+		result.allowedMetaArgs[k] = v
+	}
+	for k := range b.referencedArgs {
+		result.referencedArgs[k] = struct{}{}
+	}
+	return result
+}
+
+func (b *buildArgs) MergeReferencedArgs(other *buildArgs) {
+	for k := range other.referencedArgs {
+		b.referencedArgs[k] = struct{}{}
+	}
+}
+
 // WarnOnUnusedBuildArgs checks if there are any leftover build-args that were
 // passed but not consumed during build. Print a warning, if there are any.
 func (b *buildArgs) WarnOnUnusedBuildArgs(out io.Writer) {

components/engine/builder/dockerfile/builder.go

@@ -13,7 +13,7 @@ import (
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder"
-	"github.com/docker/docker/builder/dockerfile/command"
+	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/builder/remotecontext"
@@ -41,6 +41,10 @@ var validCommitCommands = map[string]bool{
 	"workdir":     true,
 }
 
+const (
+	stepFormat = "Step %d/%d : %v"
+)
+
 // SessionGetter is object used to get access to a session by uuid
 type SessionGetter interface {
 	Get(ctx context.Context, uuid string) (session.Caller, error)
@@ -176,9 +180,7 @@ type Builder struct {
 	clientCtx context.Context
 
 	idMappings       *idtools.IDMappings
-	buildStages      *buildStages
 	disableCommit    bool
-	buildArgs        *buildArgs
 	imageSources     *imageSources
 	pathCache        pathCache
 	containerManager *containerManager
@@ -218,8 +220,6 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		Output:           options.ProgressWriter.Output,
 		docker:           options.Backend,
 		idMappings:       options.IDMappings,
-		buildArgs:        newBuildArgs(config.BuildArgs),
-		buildStages:      newBuildStages(),
 		imageSources:     newImageSources(clientCtx, options),
 		pathCache:        options.PathCache,
 		imageProber:      newImageProber(options.Backend, config.CacheFrom, options.Platform, config.NoCache),
@@ -237,24 +237,27 @@ func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*buil
 
 	addNodesForLabelOption(dockerfile.AST, b.options.Labels)
 
-	if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
-		buildsFailed.WithValues(metricsDockerfileSyntaxError).Inc()
+	stages, metaArgs, err := instructions.Parse(dockerfile.AST)
+	if err != nil {
+		if instructions.IsUnknownInstruction(err) {
+			buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
+		}
 		return nil, validationError{err}
 	}
-
-	dispatchState, err := b.dispatchDockerfileWithCancellation(dockerfile, source)
-	if err != nil {
-		return nil, err
-	}
-
-	if b.options.Target != "" && !dispatchState.isCurrentStage(b.options.Target) {
-		buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc()
-		return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)
+	if b.options.Target != "" {
+		targetIx, found := instructions.HasStage(stages, b.options.Target)
+		if !found {
+			buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc()
+			return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)
+		}
+		stages = stages[:targetIx+1]
 	}
 
 	dockerfile.PrintWarnings(b.Stderr)
-	b.buildArgs.WarnOnUnusedBuildArgs(b.Stderr)
-
+	dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source)
+	if err != nil {
+		return nil, err
+	}
 	if dispatchState.imageID == "" {
 		buildsFailed.WithValues(metricsDockerfileEmptyError).Inc()
 		return nil, errors.New("No image was generated. Is your Dockerfile empty?")
@@ -269,61 +272,91 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error
 	return aux.Emit(types.BuildResult{ID: state.imageID})
 }
 
-func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result, source builder.Source) (*dispatchState, error) {
-	shlex := NewShellLex(dockerfile.EscapeToken)
-	state := newDispatchState()
-	total := len(dockerfile.AST.Children)
-	var err error
-	for i, n := range dockerfile.AST.Children {
-		select {
-		case <-b.clientCtx.Done():
-			logrus.Debug("Builder: build cancelled!")
-			fmt.Fprint(b.Stdout, "Build cancelled")
-			buildsFailed.WithValues(metricsBuildCanceled).Inc()
-			return nil, errors.New("Build cancelled")
-		default:
-			// Not cancelled yet, keep going...
-		}
+func processMetaArg(meta instructions.ArgCommand, shlex *ShellLex, args *buildArgs) error {
+	// ShellLex currently only support the concatenated string format
+	envs := convertMapToEnvList(args.GetAllAllowed())
+	if err := meta.Expand(func(word string) (string, error) {
+		return shlex.ProcessWord(word, envs)
+	}); err != nil {
+		return err
+	}
+	args.AddArg(meta.Key, meta.Value)
+	args.AddMetaArg(meta.Key, meta.Value)
+	return nil
+}
 
-		// If this is a FROM and we have a previous image then
-		// emit an aux message for that image since it is the
-		// end of the previous stage
-		if n.Value == command.From {
-			if err := emitImageID(b.Aux, state); err != nil {
-				return nil, err
-			}
-		}
+func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int {
+	fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd)
+	fmt.Fprintln(out)
+	return currentCommandIndex + 1
+}
 
-		if n.Value == command.From && state.isCurrentStage(b.options.Target) {
-			break
-		}
+func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) {
+	dispatchRequest := dispatchRequest{}
+	buildArgs := newBuildArgs(b.options.BuildArgs)
+	totalCommands := len(metaArgs) + len(parseResult)
+	currentCommandIndex := 1
+	for _, stage := range parseResult {
+		totalCommands += len(stage.Commands)
+	}
+	shlex := NewShellLex(escapeToken)
+	for _, meta := range metaArgs {
+		currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta)
 
-		opts := dispatchOptions{
-			state:   state,
-			stepMsg: formatStep(i, total),
-			node:    n,
-			shlex:   shlex,
-			source:  source,
-		}
-		if state, err = b.dispatch(opts); err != nil {
-			if b.options.ForceRemove {
-				b.containerManager.RemoveAll(b.Stdout)
-			}
+		err := processMetaArg(meta, shlex, buildArgs)
+		if err != nil {
 			return nil, err
 		}
+	}
 
-		fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(state.imageID))
-		if b.options.Remove {
-			b.containerManager.RemoveAll(b.Stdout)
+	stagesResults := newStagesBuildResults()
+
+	for _, stage := range parseResult {
+		if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil {
+			return nil, err
+		}
+		dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults)
+
+		currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode)
+		if err := initializeStage(dispatchRequest, &stage); err != nil {
+			return nil, err
+		}
+		dispatchRequest.state.updateRunConfig()
+		fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
+		for _, cmd := range stage.Commands {
+			select {
+			case <-b.clientCtx.Done():
+				logrus.Debug("Builder: build cancelled!")
+				fmt.Fprint(b.Stdout, "Build cancelled\n")
+				buildsFailed.WithValues(metricsBuildCanceled).Inc()
+				return nil, errors.New("Build cancelled")
+			default:
+				// Not cancelled yet, keep going...
+			}
+
+			currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd)
+
+			if err := dispatch(dispatchRequest, cmd); err != nil {
+				return nil, err
+			}
+
+			dispatchRequest.state.updateRunConfig()
+			fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
+
+		}
+		if err := emitImageID(b.Aux, dispatchRequest.state); err != nil {
+			return nil, err
+		}
+		buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs)
+		if err := commitStage(dispatchRequest.state, stagesResults); err != nil {
+			return nil, err
 		}
 	}
-
-	// Emit a final aux message for the final image
-	if err := emitImageID(b.Aux, state); err != nil {
-		return nil, err
+	if b.options.Remove {
+		b.containerManager.RemoveAll(b.Stdout)
 	}
-
-	return state, nil
+	buildArgs.WarnOnUnusedBuildArgs(b.Stdout)
+	return dispatchRequest.state, nil
 }
 
 func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) {
@@ -380,39 +413,33 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
 	b.Stderr = ioutil.Discard
 	b.disableCommit = true
 
-	if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
-		return nil, validationError{err}
+	commands := []instructions.Command{}
+	for _, n := range dockerfile.AST.Children {
+		cmd, err := instructions.ParseCommand(n)
+		if err != nil {
+			return nil, validationError{err}
+		}
+		commands = append(commands, cmd)
 	}
-	dispatchState := newDispatchState()
-	dispatchState.runConfig = config
-	return dispatchFromDockerfile(b, dockerfile, dispatchState, nil)
+
+	dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, newBuildArgs(b.options.BuildArgs), newStagesBuildResults())
+	dispatchRequest.state.runConfig = config
+	dispatchRequest.state.imageID = config.Image
+	for _, cmd := range commands {
+		err := dispatch(dispatchRequest, cmd)
+		if err != nil {
+			return nil, validationError{err}
+		}
+		dispatchRequest.state.updateRunConfig()
+	}
+
+	return dispatchRequest.state.runConfig, nil
 }
 
-func checkDispatchDockerfile(dockerfile *parser.Node) error {
-	for _, n := range dockerfile.Children {
-		if err := checkDispatch(n); err != nil {
-			return errors.Wrapf(err, "Dockerfile parse error line %d", n.StartLine)
-		}
+func convertMapToEnvList(m map[string]string) []string {
+	result := []string{}
+	for k, v := range m {
+		result = append(result, k+"="+v)
 	}
-	return nil
-}
-
-func dispatchFromDockerfile(b *Builder, result *parser.Result, dispatchState *dispatchState, source builder.Source) (*container.Config, error) {
-	shlex := NewShellLex(result.EscapeToken)
-	ast := result.AST
-	total := len(ast.Children)
-
-	for i, n := range ast.Children {
-		opts := dispatchOptions{
-			state:   dispatchState,
-			stepMsg: formatStep(i, total),
-			node:    n,
-			shlex:   shlex,
-			source:  source,
-		}
-		if _, err := b.dispatch(opts); err != nil {
-			return nil, err
-		}
-	}
-	return dispatchState.runConfig, nil
+	return result
 }

components/engine/builder/dockerfile/dispatchers_test.go

@@ -1,60 +1,29 @@
 package dockerfile
 
 import (
-	"fmt"
-	"runtime"
-	"testing"
-
 	"bytes"
 	"context"
+	"runtime"
+	"testing"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/strslice"
 	"github.com/docker/docker/builder"
-	"github.com/docker/docker/builder/dockerfile/parser"
-	"github.com/docker/docker/internal/testutil"
+	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-connections/nat"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-type commandWithFunction struct {
-	name     string
-	function func(args []string) error
-}
-
-func withArgs(f dispatcher) func([]string) error {
-	return func(args []string) error {
-		return f(dispatchRequest{args: args})
-	}
-}
-
-func withBuilderAndArgs(builder *Builder, f dispatcher) func([]string) error {
-	return func(args []string) error {
-		return f(defaultDispatchReq(builder, args...))
-	}
-}
-
-func defaultDispatchReq(builder *Builder, args ...string) dispatchRequest {
-	return dispatchRequest{
-		builder: builder,
-		args:    args,
-		flags:   NewBFlags(),
-		shlex:   NewShellLex(parser.DefaultEscapeToken),
-		state:   &dispatchState{runConfig: &container.Config{}},
-	}
-}
-
 func newBuilderWithMockBackend() *Builder {
 	mockBackend := &MockBackend{}
 	ctx := context.Background()
 	b := &Builder{
 		options:       &types.ImageBuildOptions{},
 		docker:        mockBackend,
-		buildArgs:     newBuildArgs(make(map[string]*string)),
 		Stdout:        new(bytes.Buffer),
 		clientCtx:     ctx,
 		disableCommit: true,
@@ -62,137 +31,84 @@ func newBuilderWithMockBackend() *Builder {
 			Options: &types.ImageBuildOptions{},
 			Backend: mockBackend,
 		}),
-		buildStages:      newBuildStages(),
 		imageProber:      newImageProber(mockBackend, nil, runtime.GOOS, false),
 		containerManager: newContainerManager(mockBackend),
 	}
 	return b
 }
 
-func TestCommandsExactlyOneArgument(t *testing.T) {
-	commands := []commandWithFunction{
-		{"MAINTAINER", withArgs(maintainer)},
-		{"WORKDIR", withArgs(workdir)},
-		{"USER", withArgs(user)},
-		{"STOPSIGNAL", withArgs(stopSignal)},
-	}
-
-	for _, command := range commands {
-		err := command.function([]string{})
-		assert.EqualError(t, err, errExactlyOneArgument(command.name).Error())
-	}
-}
-
-func TestCommandsAtLeastOneArgument(t *testing.T) {
-	commands := []commandWithFunction{
-		{"ENV", withArgs(env)},
-		{"LABEL", withArgs(label)},
-		{"ONBUILD", withArgs(onbuild)},
-		{"HEALTHCHECK", withArgs(healthcheck)},
-		{"EXPOSE", withArgs(expose)},
-		{"VOLUME", withArgs(volume)},
-	}
-
-	for _, command := range commands {
-		err := command.function([]string{})
-		assert.EqualError(t, err, errAtLeastOneArgument(command.name).Error())
-	}
-}
-
-func TestCommandsAtLeastTwoArguments(t *testing.T) {
-	commands := []commandWithFunction{
-		{"ADD", withArgs(add)},
-		{"COPY", withArgs(dispatchCopy)}}
-
-	for _, command := range commands {
-		err := command.function([]string{"arg1"})
-		assert.EqualError(t, err, errAtLeastTwoArguments(command.name).Error())
-	}
-}
-
-func TestCommandsTooManyArguments(t *testing.T) {
-	commands := []commandWithFunction{
-		{"ENV", withArgs(env)},
-		{"LABEL", withArgs(label)}}
-
-	for _, command := range commands {
-		err := command.function([]string{"arg1", "arg2", "arg3"})
-		assert.EqualError(t, err, errTooManyArguments(command.name).Error())
-	}
-}
-
-func TestCommandsBlankNames(t *testing.T) {
-	builder := newBuilderWithMockBackend()
-	commands := []commandWithFunction{
-		{"ENV", withBuilderAndArgs(builder, env)},
-		{"LABEL", withBuilderAndArgs(builder, label)},
-	}
-
-	for _, command := range commands {
-		err := command.function([]string{"", ""})
-		assert.EqualError(t, err, errBlankCommandNames(command.name).Error())
-	}
-}
-
 func TestEnv2Variables(t *testing.T) {
 	b := newBuilderWithMockBackend()
-
-	args := []string{"var1", "val1", "var2", "val2"}
-	req := defaultDispatchReq(b, args...)
-	err := env(req)
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	envCommand := &instructions.EnvCommand{
+		Env: instructions.KeyValuePairs{
+			instructions.KeyValuePair{Key: "var1", Value: "val1"},
+			instructions.KeyValuePair{Key: "var2", Value: "val2"},
+		},
+	}
+	err := dispatch(sb, envCommand)
 	require.NoError(t, err)
 
 	expected := []string{
-		fmt.Sprintf("%s=%s", args[0], args[1]),
-		fmt.Sprintf("%s=%s", args[2], args[3]),
+		"var1=val1",
+		"var2=val2",
 	}
-	assert.Equal(t, expected, req.state.runConfig.Env)
+	assert.Equal(t, expected, sb.state.runConfig.Env)
 }
 
 func TestEnvValueWithExistingRunConfigEnv(t *testing.T) {
 	b := newBuilderWithMockBackend()
-
-	args := []string{"var1", "val1"}
-	req := defaultDispatchReq(b, args...)
-	req.state.runConfig.Env = []string{"var1=old", "var2=fromenv"}
-	err := env(req)
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	sb.state.runConfig.Env = []string{"var1=old", "var2=fromenv"}
+	envCommand := &instructions.EnvCommand{
+		Env: instructions.KeyValuePairs{
+			instructions.KeyValuePair{Key: "var1", Value: "val1"},
+		},
+	}
+	err := dispatch(sb, envCommand)
 	require.NoError(t, err)
-
 	expected := []string{
-		fmt.Sprintf("%s=%s", args[0], args[1]),
+		"var1=val1",
 		"var2=fromenv",
 	}
-	assert.Equal(t, expected, req.state.runConfig.Env)
+	assert.Equal(t, expected, sb.state.runConfig.Env)
 }
 
 func TestMaintainer(t *testing.T) {
 	maintainerEntry := "Some Maintainer <maintainer@example.com>"
-
 	b := newBuilderWithMockBackend()
-	req := defaultDispatchReq(b, maintainerEntry)
-	err := maintainer(req)
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	cmd := &instructions.MaintainerCommand{Maintainer: maintainerEntry}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-	assert.Equal(t, maintainerEntry, req.state.maintainer)
+	assert.Equal(t, maintainerEntry, sb.state.maintainer)
 }
 
 func TestLabel(t *testing.T) {
 	labelName := "label"
 	labelValue := "value"
 
-	labelEntry := []string{labelName, labelValue}
 	b := newBuilderWithMockBackend()
-	req := defaultDispatchReq(b, labelEntry...)
-	err := label(req)
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	cmd := &instructions.LabelCommand{
+		Labels: instructions.KeyValuePairs{
+			instructions.KeyValuePair{Key: labelName, Value: labelValue},
+		},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
-	require.Contains(t, req.state.runConfig.Labels, labelName)
-	assert.Equal(t, req.state.runConfig.Labels[labelName], labelValue)
+	require.Contains(t, sb.state.runConfig.Labels, labelName)
+	assert.Equal(t, sb.state.runConfig.Labels[labelName], labelValue)
 }
 
 func TestFromScratch(t *testing.T) {
 	b := newBuilderWithMockBackend()
-	req := defaultDispatchReq(b, "scratch")
-	err := from(req)
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	cmd := &instructions.Stage{
+		BaseName: "scratch",
+	}
+	err := initializeStage(sb, cmd)
 
 	if runtime.GOOS == "windows" && !system.LCOWSupported() {
 		assert.EqualError(t, err, "Windows does not support FROM scratch")
@@ -200,14 +116,14 @@ func TestFromScratch(t *testing.T) {
 	}
 
 	require.NoError(t, err)
-	assert.True(t, req.state.hasFromImage())
-	assert.Equal(t, "", req.state.imageID)
+	assert.True(t, sb.state.hasFromImage())
+	assert.Equal(t, "", sb.state.imageID)
 	// Windows does not set the default path. TODO @jhowardmsft LCOW support. This will need revisiting as we get further into the implementation
 	expected := "PATH=" + system.DefaultPathEnv(runtime.GOOS)
 	if runtime.GOOS == "windows" {
 		expected = ""
 	}
-	assert.Equal(t, []string{expected}, req.state.runConfig.Env)
+	assert.Equal(t, []string{expected}, sb.state.runConfig.Env)
 }
 
 func TestFromWithArg(t *testing.T) {
@@ -219,16 +135,27 @@ func TestFromWithArg(t *testing.T) {
 	}
 	b := newBuilderWithMockBackend()
 	b.docker.(*MockBackend).getImageFunc = getImage
+	args := newBuildArgs(make(map[string]*string))
 
-	require.NoError(t, arg(defaultDispatchReq(b, "THETAG="+tag)))
-	req := defaultDispatchReq(b, "alpine${THETAG}")
-	err := from(req)
+	val := "sometag"
+	metaArg := instructions.ArgCommand{
+		Key:   "THETAG",
+		Value: &val,
+	}
+	cmd := &instructions.Stage{
+		BaseName: "alpine:${THETAG}",
+	}
+	err := processMetaArg(metaArg, NewShellLex('\\'), args)
 
+	sb := newDispatchRequest(b, '\\', nil, args, newStagesBuildResults())
 	require.NoError(t, err)
-	assert.Equal(t, expected, req.state.imageID)
-	assert.Equal(t, expected, req.state.baseImage.ImageID())
-	assert.Len(t, b.buildArgs.GetAllAllowed(), 0)
-	assert.Len(t, b.buildArgs.GetAllMeta(), 1)
+	err = initializeStage(sb, cmd)
+	require.NoError(t, err)
+
+	assert.Equal(t, expected, sb.state.imageID)
+	assert.Equal(t, expected, sb.state.baseImage.ImageID())
+	assert.Len(t, sb.state.buildArgs.GetAllAllowed(), 0)
+	assert.Len(t, sb.state.buildArgs.GetAllMeta(), 1)
 }
 
 func TestFromWithUndefinedArg(t *testing.T) {
@@ -240,74 +167,74 @@ func TestFromWithUndefinedArg(t *testing.T) {
 	}
 	b := newBuilderWithMockBackend()
 	b.docker.(*MockBackend).getImageFunc = getImage
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+
 	b.options.BuildArgs = map[string]*string{"THETAG": &tag}
 
-	req := defaultDispatchReq(b, "alpine${THETAG}")
-	err := from(req)
+	cmd := &instructions.Stage{
+		BaseName: "alpine${THETAG}",
+	}
+	err := initializeStage(sb, cmd)
 	require.NoError(t, err)
-	assert.Equal(t, expected, req.state.imageID)
+	assert.Equal(t, expected, sb.state.imageID)
 }
 
-func TestFromMultiStageWithScratchNamedStage(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("Windows does not support scratch")
-	}
+func TestFromMultiStageWithNamedStage(t *testing.T) {
 	b := newBuilderWithMockBackend()
-	req := defaultDispatchReq(b, "scratch", "AS", "base")
-
-	require.NoError(t, from(req))
-	assert.True(t, req.state.hasFromImage())
-
-	req.args = []string{"base"}
-	require.NoError(t, from(req))
-	assert.True(t, req.state.hasFromImage())
-}
-
-func TestOnbuildIllegalTriggers(t *testing.T) {
-	triggers := []struct{ command, expectedError string }{
-		{"ONBUILD", "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed"},
-		{"MAINTAINER", "MAINTAINER isn't allowed as an ONBUILD trigger"},
-		{"FROM", "FROM isn't allowed as an ONBUILD trigger"}}
-
-	for _, trigger := range triggers {
-		b := newBuilderWithMockBackend()
-
-		err := onbuild(defaultDispatchReq(b, trigger.command))
-		testutil.ErrorContains(t, err, trigger.expectedError)
-	}
+	firstFrom := &instructions.Stage{BaseName: "someimg", Name: "base"}
+	secondFrom := &instructions.Stage{BaseName: "base"}
+	previousResults := newStagesBuildResults()
+	firstSB := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), previousResults)
+	secondSB := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), previousResults)
+	err := initializeStage(firstSB, firstFrom)
+	require.NoError(t, err)
+	assert.True(t, firstSB.state.hasFromImage())
+	previousResults.indexed["base"] = firstSB.state.runConfig
+	previousResults.flat = append(previousResults.flat, firstSB.state.runConfig)
+	err = initializeStage(secondSB, secondFrom)
+	require.NoError(t, err)
+	assert.True(t, secondSB.state.hasFromImage())
 }
 
 func TestOnbuild(t *testing.T) {
 	b := newBuilderWithMockBackend()
-
-	req := defaultDispatchReq(b, "ADD", ".", "/app/src")
-	req.original = "ONBUILD ADD . /app/src"
-	req.state.runConfig = &container.Config{}
-
-	err := onbuild(req)
+	sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	cmd := &instructions.OnbuildCommand{
+		Expression: "ADD . /app/src",
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-	assert.Equal(t, "ADD . /app/src", req.state.runConfig.OnBuild[0])
+	assert.Equal(t, "ADD . /app/src", sb.state.runConfig.OnBuild[0])
 }
 
 func TestWorkdir(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 	workingDir := "/app"
 	if runtime.GOOS == "windows" {
-		workingDir = "C:\app"
+		workingDir = "C:\\app"
+	}
+	cmd := &instructions.WorkdirCommand{
+		Path: workingDir,
 	}
 
-	req := defaultDispatchReq(b, workingDir)
-	err := workdir(req)
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-	assert.Equal(t, workingDir, req.state.runConfig.WorkingDir)
+	assert.Equal(t, workingDir, sb.state.runConfig.WorkingDir)
 }
 
 func TestCmd(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 	command := "./executable"
 
-	req := defaultDispatchReq(b, command)
-	err := cmd(req)
+	cmd := &instructions.CmdCommand{
+		ShellDependantCmdLine: instructions.ShellDependantCmdLine{
+			CmdLine:      strslice.StrSlice{command},
+			PrependShell: true,
+		},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
 	var expectedCommand strslice.StrSlice
@@ -317,42 +244,56 @@ func TestCmd(t *testing.T) {
 		expectedCommand = strslice.StrSlice(append([]string{"/bin/sh"}, "-c", command))
 	}
 
-	assert.Equal(t, expectedCommand, req.state.runConfig.Cmd)
-	assert.True(t, req.state.cmdSet)
+	assert.Equal(t, expectedCommand, sb.state.runConfig.Cmd)
+	assert.True(t, sb.state.cmdSet)
 }
 
 func TestHealthcheckNone(t *testing.T) {
 	b := newBuilderWithMockBackend()
-
-	req := defaultDispatchReq(b, "NONE")
-	err := healthcheck(req)
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	cmd := &instructions.HealthCheckCommand{
+		Health: &container.HealthConfig{
+			Test: []string{"NONE"},
+		},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
-	require.NotNil(t, req.state.runConfig.Healthcheck)
-	assert.Equal(t, []string{"NONE"}, req.state.runConfig.Healthcheck.Test)
+	require.NotNil(t, sb.state.runConfig.Healthcheck)
+	assert.Equal(t, []string{"NONE"}, sb.state.runConfig.Healthcheck.Test)
 }
 
 func TestHealthcheckCmd(t *testing.T) {
-	b := newBuilderWithMockBackend()
 
-	args := []string{"CMD", "curl", "-f", "http://localhost/", "||", "exit", "1"}
-	req := defaultDispatchReq(b, args...)
-	err := healthcheck(req)
+	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	expectedTest := []string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"}
+	cmd := &instructions.HealthCheckCommand{
+		Health: &container.HealthConfig{
+			Test: expectedTest,
+		},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
-	require.NotNil(t, req.state.runConfig.Healthcheck)
-	expectedTest := []string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"}
-	assert.Equal(t, expectedTest, req.state.runConfig.Healthcheck.Test)
+	require.NotNil(t, sb.state.runConfig.Healthcheck)
+	assert.Equal(t, expectedTest, sb.state.runConfig.Healthcheck.Test)
 }
 
 func TestEntrypoint(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 	entrypointCmd := "/usr/sbin/nginx"
 
-	req := defaultDispatchReq(b, entrypointCmd)
-	err := entrypoint(req)
+	cmd := &instructions.EntrypointCommand{
+		ShellDependantCmdLine: instructions.ShellDependantCmdLine{
+			CmdLine:      strslice.StrSlice{entrypointCmd},
+			PrependShell: true,
+		},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-	require.NotNil(t, req.state.runConfig.Entrypoint)
+	require.NotNil(t, sb.state.runConfig.Entrypoint)
 
 	var expectedEntrypoint strslice.StrSlice
 	if runtime.GOOS == "windows" {
@@ -360,99 +301,99 @@ func TestEntrypoint(t *testing.T) {
 	} else {
 		expectedEntrypoint = strslice.StrSlice(append([]string{"/bin/sh"}, "-c", entrypointCmd))
 	}
-	assert.Equal(t, expectedEntrypoint, req.state.runConfig.Entrypoint)
+	assert.Equal(t, expectedEntrypoint, sb.state.runConfig.Entrypoint)
 }
 
 func TestExpose(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 
 	exposedPort := "80"
-	req := defaultDispatchReq(b, exposedPort)
-	err := expose(req)
+	cmd := &instructions.ExposeCommand{
+		Ports: []string{exposedPort},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
-	require.NotNil(t, req.state.runConfig.ExposedPorts)
-	require.Len(t, req.state.runConfig.ExposedPorts, 1)
+	require.NotNil(t, sb.state.runConfig.ExposedPorts)
+	require.Len(t, sb.state.runConfig.ExposedPorts, 1)
 
 	portsMapping, err := nat.ParsePortSpec(exposedPort)
 	require.NoError(t, err)
-	assert.Contains(t, req.state.runConfig.ExposedPorts, portsMapping[0].Port)
+	assert.Contains(t, sb.state.runConfig.ExposedPorts, portsMapping[0].Port)
 }
 
 func TestUser(t *testing.T) {
 	b := newBuilderWithMockBackend()
-	userCommand := "foo"
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 
-	req := defaultDispatchReq(b, userCommand)
-	err := user(req)
+	cmd := &instructions.UserCommand{
+		User: "test",
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-	assert.Equal(t, userCommand, req.state.runConfig.User)
+	assert.Equal(t, "test", sb.state.runConfig.User)
 }
 
 func TestVolume(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 
 	exposedVolume := "/foo"
 
-	req := defaultDispatchReq(b, exposedVolume)
-	err := volume(req)
+	cmd := &instructions.VolumeCommand{
+		Volumes: []string{exposedVolume},
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-
-	require.NotNil(t, req.state.runConfig.Volumes)
-	assert.Len(t, req.state.runConfig.Volumes, 1)
-	assert.Contains(t, req.state.runConfig.Volumes, exposedVolume)
+	require.NotNil(t, sb.state.runConfig.Volumes)
+	assert.Len(t, sb.state.runConfig.Volumes, 1)
+	assert.Contains(t, sb.state.runConfig.Volumes, exposedVolume)
 }
 
 func TestStopSignal(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Windows does not support stopsignal")
+		return
+	}
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 	signal := "SIGKILL"
 
-	req := defaultDispatchReq(b, signal)
-	err := stopSignal(req)
+	cmd := &instructions.StopSignalCommand{
+		Signal: signal,
+	}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
-	assert.Equal(t, signal, req.state.runConfig.StopSignal)
+	assert.Equal(t, signal, sb.state.runConfig.StopSignal)
 }
 
 func TestArg(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 
 	argName := "foo"
 	argVal := "bar"
-	argDef := fmt.Sprintf("%s=%s", argName, argVal)
-
-	err := arg(defaultDispatchReq(b, argDef))
+	cmd := &instructions.ArgCommand{Key: argName, Value: &argVal}
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
 	expected := map[string]string{argName: argVal}
-	assert.Equal(t, expected, b.buildArgs.GetAllAllowed())
+	assert.Equal(t, expected, sb.state.buildArgs.GetAllAllowed())
 }
 
 func TestShell(t *testing.T) {
 	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
 
 	shellCmd := "powershell"
-	req := defaultDispatchReq(b, shellCmd)
-	req.attributes = map[string]bool{"json": true}
+	cmd := &instructions.ShellCommand{Shell: strslice.StrSlice{shellCmd}}
 
-	err := shell(req)
+	err := dispatch(sb, cmd)
 	require.NoError(t, err)
 
 	expectedShell := strslice.StrSlice([]string{shellCmd})
-	assert.Equal(t, expectedShell, req.state.runConfig.Shell)
-}
-
-func TestParseOptInterval(t *testing.T) {
-	flInterval := &Flag{
-		name:     "interval",
-		flagType: stringType,
-		Value:    "50ns",
-	}
-	_, err := parseOptInterval(flInterval)
-	testutil.ErrorContains(t, err, "cannot be less than 1ms")
-
-	flInterval.Value = "1ms"
-	_, err = parseOptInterval(flInterval)
-	require.NoError(t, err)
+	assert.Equal(t, expectedShell, sb.state.runConfig.Shell)
 }
 
 func TestPrependEnvOnCmd(t *testing.T) {
@@ -469,8 +410,10 @@ func TestPrependEnvOnCmd(t *testing.T) {
 
 func TestRunWithBuildArgs(t *testing.T) {
 	b := newBuilderWithMockBackend()
-	b.buildArgs.argsFromOptions["HTTP_PROXY"] = strPtr("FOO")
+	args := newBuildArgs(make(map[string]*string))
+	args.argsFromOptions["HTTP_PROXY"] = strPtr("FOO")
 	b.disableCommit = false
+	sb := newDispatchRequest(b, '`', nil, args, newStagesBuildResults())
 
 	runConfig := &container.Config{}
 	origCmd := strslice.StrSlice([]string{"cmd", "in", "from", "image"})
@@ -512,14 +455,18 @@ func TestRunWithBuildArgs(t *testing.T) {
 		assert.Equal(t, strslice.StrSlice(nil), cfg.Config.Entrypoint)
 		return "", nil
 	}
-
-	req := defaultDispatchReq(b, "abcdef")
-	require.NoError(t, from(req))
-	b.buildArgs.AddArg("one", strPtr("two"))
-
-	req.args = []string{"echo foo"}
-	require.NoError(t, run(req))
+	from := &instructions.Stage{BaseName: "abcdef"}
+	err := initializeStage(sb, from)
+	require.NoError(t, err)
+	sb.state.buildArgs.AddArg("one", strPtr("two"))
+	run := &instructions.RunCommand{
+		ShellDependantCmdLine: instructions.ShellDependantCmdLine{
+			CmdLine:      strslice.StrSlice{"echo foo"},
+			PrependShell: true,
+		},
+	}
+	require.NoError(t, dispatch(sb, run))
 
 	// Check that runConfig.Cmd has not been modified by run
-	assert.Equal(t, origCmd, req.state.runConfig.Cmd)
+	assert.Equal(t, origCmd, sb.state.runConfig.Cmd)
 }

components/engine/builder/dockerfile/dispatchers_unix.go

@@ -4,7 +4,6 @@ package dockerfile
 
 import (
 	"errors"
-	"fmt"
 	"os"
 	"path/filepath"
 )
@@ -23,10 +22,6 @@ func normalizeWorkdir(_ string, current string, requested string) (string, error
 	return requested, nil
 }
 
-func errNotJSON(command, _ string) error {
-	return fmt.Errorf("%s requires the arguments to be in JSON form", command)
-}
-
 // equalEnvKeys compare two strings and returns true if they are equal. On
 // Windows this comparison is case insensitive.
 func equalEnvKeys(from, to string) bool {

components/engine/builder/dockerfile/dispatchers_windows.go

@@ -94,25 +94,6 @@ func normalizeWorkdirWindows(current string, requested string) (string, error) {
 	return (strings.ToUpper(string(requested[0])) + requested[1:]), nil
 }
 
-func errNotJSON(command, original string) error {
-	// For Windows users, give a hint if it looks like it might contain
-	// a path which hasn't been escaped such as ["c:\windows\system32\prog.exe", "-param"],
-	// as JSON must be escaped. Unfortunate...
-	//
-	// Specifically looking for quote-driveletter-colon-backslash, there's no
-	// double backslash and a [] pair. No, this is not perfect, but it doesn't
-	// have to be. It's simply a hint to make life a little easier.
-	extra := ""
-	original = filepath.FromSlash(strings.ToLower(strings.Replace(strings.ToLower(original), strings.ToLower(command)+" ", "", -1)))
-	if len(regexp.MustCompile(`"[a-z]:\\.*`).FindStringSubmatch(original)) > 0 &&
-		!strings.Contains(original, `\\`) &&
-		strings.Contains(original, "[") &&
-		strings.Contains(original, "]") {
-		extra = fmt.Sprintf(`. It looks like '%s' includes a file path without an escaped back-slash. JSON requires back-slashes to be escaped such as ["c:\\path\\to\\file.exe", "/parameter"]`, original)
-	}
-	return fmt.Errorf("%s requires the arguments to be in JSON form%s", command, extra)
-}
-
 // equalEnvKeys compare two strings and returns true if they are equal. On
 // Windows this comparison is case insensitive.
 func equalEnvKeys(from, to string) bool {

components/engine/builder/dockerfile/evaluator.go

@@ -20,169 +20,79 @@
 package dockerfile
 
 import (
-	"bytes"
-	"fmt"
+	"reflect"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder"
-	"github.com/docker/docker/builder/dockerfile/command"
-	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/runconfig/opts"
 	"github.com/pkg/errors"
 )
 
-// Environment variable interpolation will happen on these statements only.
-var replaceEnvAllowed = map[string]bool{
-	command.Env:        true,
-	command.Label:      true,
-	command.Add:        true,
-	command.Copy:       true,
-	command.Workdir:    true,
-	command.Expose:     true,
-	command.Volume:     true,
-	command.User:       true,
-	command.StopSignal: true,
-	command.Arg:        true,
-}
-
-// Certain commands are allowed to have their args split into more
-// words after env var replacements. Meaning:
-//   ENV foo="123 456"
-//   EXPOSE $foo
-// should result in the same thing as:
-//   EXPOSE 123 456
-// and not treat "123 456" as a single word.
-// Note that: EXPOSE "$foo" and EXPOSE $foo are not the same thing.
-// Quotes will cause it to still be treated as single word.
-var allowWordExpansion = map[string]bool{
-	command.Expose: true,
-}
-
-type dispatchRequest struct {
-	builder    *Builder // TODO: replace this with a smaller interface
-	args       []string
-	attributes map[string]bool
-	flags      *BFlags
-	original   string
-	shlex      *ShellLex
-	state      *dispatchState
-	source     builder.Source
-}
-
-func newDispatchRequestFromOptions(options dispatchOptions, builder *Builder, args []string) dispatchRequest {
-	return dispatchRequest{
-		builder:    builder,
-		args:       args,
-		attributes: options.node.Attributes,
-		original:   options.node.Original,
-		flags:      NewBFlagsWithArgs(options.node.Flags),
-		shlex:      options.shlex,
-		state:      options.state,
-		source:     options.source,
-	}
-}
-
-type dispatcher func(dispatchRequest) error
-
-var evaluateTable map[string]dispatcher
-
-func init() {
-	evaluateTable = map[string]dispatcher{
-		command.Add:         add,
-		command.Arg:         arg,
-		command.Cmd:         cmd,
-		command.Copy:        dispatchCopy, // copy() is a go builtin
-		command.Entrypoint:  entrypoint,
-		command.Env:         env,
-		command.Expose:      expose,
-		command.From:        from,
-		command.Healthcheck: healthcheck,
-		command.Label:       label,
-		command.Maintainer:  maintainer,
-		command.Onbuild:     onbuild,
-		command.Run:         run,
-		command.Shell:       shell,
-		command.StopSignal:  stopSignal,
-		command.User:        user,
-		command.Volume:      volume,
-		command.Workdir:     workdir,
-	}
-}
-
-func formatStep(stepN int, stepTotal int) string {
-	return fmt.Sprintf("%d/%d", stepN+1, stepTotal)
-}
-
-// This method is the entrypoint to all statement handling routines.
-//
-// Almost all nodes will have this structure:
-// Child[Node, Node, Node] where Child is from parser.Node.Children and each
-// node comes from parser.Node.Next. This forms a "line" with a statement and
-// arguments and we process them in this normalized form by hitting
-// evaluateTable with the leaf nodes of the command and the Builder object.
-//
-// ONBUILD is a special case; in this case the parser will emit:
-// Child[Node, Child[Node, Node...]] where the first node is the literal
-// "onbuild" and the child entrypoint is the command of the ONBUILD statement,
-// such as `RUN` in ONBUILD RUN foo. There is special case logic in here to
-// deal with that, at least until it becomes more of a general concern with new
-// features.
-func (b *Builder) dispatch(options dispatchOptions) (*dispatchState, error) {
-	node := options.node
-	cmd := node.Value
-	upperCasedCmd := strings.ToUpper(cmd)
-
-	// To ensure the user is given a decent error message if the platform
-	// on which the daemon is running does not support a builder command.
-	if err := platformSupports(strings.ToLower(cmd)); err != nil {
-		buildsFailed.WithValues(metricsCommandNotSupportedError).Inc()
-		return nil, validationError{err}
-	}
-
-	msg := bytes.NewBufferString(fmt.Sprintf("Step %s : %s%s",
-		options.stepMsg, upperCasedCmd, formatFlags(node.Flags)))
-
-	args := []string{}
-	ast := node
-	if cmd == command.Onbuild {
-		var err error
-		ast, args, err = handleOnBuildNode(node, msg)
+func dispatch(d dispatchRequest, cmd instructions.Command) error {
+	if c, ok := cmd.(instructions.PlatformSpecific); ok {
+		err := c.CheckPlatform(d.builder.platform)
 		if err != nil {
-			return nil, validationError{err}
+			return validationError{err}
+		}
+	}
+	runConfigEnv := d.state.runConfig.Env
+	envs := append(runConfigEnv, d.state.buildArgs.FilterAllowed(runConfigEnv)...)
+
+	if ex, ok := cmd.(instructions.SupportsSingleWordExpansion); ok {
+		err := ex.Expand(func(word string) (string, error) {
+			return d.shlex.ProcessWord(word, envs)
+		})
+		if err != nil {
+			return validationError{err}
 		}
 	}
 
-	runConfigEnv := options.state.runConfig.Env
-	envs := append(runConfigEnv, b.buildArgs.FilterAllowed(runConfigEnv)...)
-	processFunc := createProcessWordFunc(options.shlex, cmd, envs)
-	words, err := getDispatchArgsFromNode(ast, processFunc, msg)
-	if err != nil {
-		buildsFailed.WithValues(metricsErrorProcessingCommandsError).Inc()
-		return nil, validationError{err}
+	if d.builder.options.ForceRemove {
+		defer d.builder.containerManager.RemoveAll(d.builder.Stdout)
 	}
-	args = append(args, words...)
 
-	fmt.Fprintln(b.Stdout, msg.String())
-
-	f, ok := evaluateTable[cmd]
-	if !ok {
-		buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
-		return nil, validationError{errors.Errorf("unknown instruction: %s", upperCasedCmd)}
+	switch c := cmd.(type) {
+	case *instructions.EnvCommand:
+		return dispatchEnv(d, c)
+	case *instructions.MaintainerCommand:
+		return dispatchMaintainer(d, c)
+	case *instructions.LabelCommand:
+		return dispatchLabel(d, c)
+	case *instructions.AddCommand:
+		return dispatchAdd(d, c)
+	case *instructions.CopyCommand:
+		return dispatchCopy(d, c)
+	case *instructions.OnbuildCommand:
+		return dispatchOnbuild(d, c)
+	case *instructions.WorkdirCommand:
+		return dispatchWorkdir(d, c)
+	case *instructions.RunCommand:
+		return dispatchRun(d, c)
+	case *instructions.CmdCommand:
+		return dispatchCmd(d, c)
+	case *instructions.HealthCheckCommand:
+		return dispatchHealthcheck(d, c)
+	case *instructions.EntrypointCommand:
+		return dispatchEntrypoint(d, c)
+	case *instructions.ExposeCommand:
+		return dispatchExpose(d, c, envs)
+	case *instructions.UserCommand:
+		return dispatchUser(d, c)
+	case *instructions.VolumeCommand:
+		return dispatchVolume(d, c)
+	case *instructions.StopSignalCommand:
+		return dispatchStopSignal(d, c)
+	case *instructions.ArgCommand:
+		return dispatchArg(d, c)
+	case *instructions.ShellCommand:
+		return dispatchShell(d, c)
 	}
-	options.state.updateRunConfig()
-	err = f(newDispatchRequestFromOptions(options, b, args))
-	return options.state, err
-}
-
-type dispatchOptions struct {
-	state   *dispatchState
-	stepMsg string
-	node    *parser.Node
-	shlex   *ShellLex
-	source  builder.Source
+	return errors.Errorf("unsupported command type: %v", reflect.TypeOf(cmd))
 }
 
 // dispatchState is a data object which is modified by dispatchers
@@ -193,10 +103,95 @@ type dispatchState struct {
 	imageID    string
 	baseImage  builder.Image
 	stageName  string
+	buildArgs  *buildArgs
 }
 
-func newDispatchState() *dispatchState {
-	return &dispatchState{runConfig: &container.Config{}}
+func newDispatchState(baseArgs *buildArgs) *dispatchState {
+	args := baseArgs.Clone()
+	args.ResetAllowed()
+	return &dispatchState{runConfig: &container.Config{}, buildArgs: args}
+}
+
+type stagesBuildResults struct {
+	flat    []*container.Config
+	indexed map[string]*container.Config
+}
+
+func newStagesBuildResults() *stagesBuildResults {
+	return &stagesBuildResults{
+		indexed: make(map[string]*container.Config),
+	}
+}
+
+func (r *stagesBuildResults) getByName(name string) (*container.Config, bool) {
+	c, ok := r.indexed[strings.ToLower(name)]
+	return c, ok
+}
+
+func (r *stagesBuildResults) validateIndex(i int) error {
+	if i == len(r.flat) {
+		return errors.New("refers to current build stage")
+	}
+	if i < 0 || i > len(r.flat) {
+		return errors.New("index out of bounds")
+	}
+	return nil
+}
+
+func (r *stagesBuildResults) get(nameOrIndex string) (*container.Config, error) {
+	if c, ok := r.getByName(nameOrIndex); ok {
+		return c, nil
+	}
+	ix, err := strconv.ParseInt(nameOrIndex, 10, 0)
+	if err != nil {
+		return nil, nil
+	}
+	if err := r.validateIndex(int(ix)); err != nil {
+		return nil, err
+	}
+	return r.flat[ix], nil
+}
+
+func (r *stagesBuildResults) checkStageNameAvailable(name string) error {
+	if name != "" {
+		if _, ok := r.getByName(name); ok {
+			return errors.Errorf("%s stage name already used", name)
+		}
+	}
+	return nil
+}
+
+func (r *stagesBuildResults) commitStage(name string, config *container.Config) error {
+	if name != "" {
+		if _, ok := r.getByName(name); ok {
+			return errors.Errorf("%s stage name already used", name)
+		}
+		r.indexed[strings.ToLower(name)] = config
+	}
+	r.flat = append(r.flat, config)
+	return nil
+}
+
+func commitStage(state *dispatchState, stages *stagesBuildResults) error {
+	return stages.commitStage(state.stageName, state.runConfig)
+}
+
+type dispatchRequest struct {
+	state   *dispatchState
+	shlex   *ShellLex
+	builder *Builder
+	source  builder.Source
+	stages  *stagesBuildResults
+}
+
+func newDispatchRequest(builder *Builder, escapeToken rune, source builder.Source, buildArgs *buildArgs, stages *stagesBuildResults) dispatchRequest {
+	return dispatchRequest{
+		state:   newDispatchState(buildArgs),
+		shlex:   NewShellLex(escapeToken),
+		builder: builder,
+		source:  source,
+		stages:  stages,
+	}
 }
 
 func (s *dispatchState) updateRunConfig() {
@@ -220,12 +215,14 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) {
 	s.imageID = image.ImageID()
 
 	if image.RunConfig() != nil {
-		s.runConfig = image.RunConfig()
+		s.runConfig = copyRunConfig(image.RunConfig()) // copy avoids referencing the same instance when 2 stages have the same base
 	} else {
 		s.runConfig = &container.Config{}
 	}
 	s.baseImage = image
 	s.setDefaultPath()
+	s.runConfig.OpenStdin = false
+	s.runConfig.StdinOnce = false
 }
 
 // Add the default PATH to runConfig.ENV if one exists for the platform and there
@@ -244,84 +241,3 @@ func (s *dispatchState) setDefaultPath() {
 		s.runConfig.Env = append(s.runConfig.Env, "PATH="+system.DefaultPathEnv(platform))
 	}
 }
-
-func handleOnBuildNode(ast *parser.Node, msg *bytes.Buffer) (*parser.Node, []string, error) {
-	if ast.Next == nil {
-		return nil, nil, validationError{errors.New("ONBUILD requires at least one argument")}
-	}
-	ast = ast.Next.Children[0]
-	msg.WriteString(" " + ast.Value + formatFlags(ast.Flags))
-	return ast, []string{ast.Value}, nil
-}
-
-func formatFlags(flags []string) string {
-	if len(flags) > 0 {
-		return " " + strings.Join(flags, " ")
-	}
-	return ""
-}
-
-func getDispatchArgsFromNode(ast *parser.Node, processFunc processWordFunc, msg *bytes.Buffer) ([]string, error) {
-	args := []string{}
-	for i := 0; ast.Next != nil; i++ {
-		ast = ast.Next
-		words, err := processFunc(ast.Value)
-		if err != nil {
-			return nil, err
-		}
-		args = append(args, words...)
-		msg.WriteString(" " + ast.Value)
-	}
-	return args, nil
-}
-
-type processWordFunc func(string) ([]string, error)
-
-func createProcessWordFunc(shlex *ShellLex, cmd string, envs []string) processWordFunc {
-	switch {
-	case !replaceEnvAllowed[cmd]:
-		return func(word string) ([]string, error) {
-			return []string{word}, nil
-		}
-	case allowWordExpansion[cmd]:
-		return func(word string) ([]string, error) {
-			return shlex.ProcessWords(word, envs)
-		}
-	default:
-		return func(word string) ([]string, error) {
-			word, err := shlex.ProcessWord(word, envs)
-			return []string{word}, err
-		}
-	}
-}
-
-// checkDispatch does a simple check for syntax errors of the Dockerfile.
-// Because some of the instructions can only be validated through runtime,
-// arg, env, etc., this syntax check will not be complete and could not replace
-// the runtime check. Instead, this function is only a helper that allows
-// user to find out the obvious error in Dockerfile earlier on.
-func checkDispatch(ast *parser.Node) error {
-	cmd := ast.Value
-	upperCasedCmd := strings.ToUpper(cmd)
-
-	// To ensure the user is given a decent error message if the platform
-	// on which the daemon is running does not support a builder command.
-	if err := platformSupports(strings.ToLower(cmd)); err != nil {
-		return err
-	}
-
-	// The instruction itself is ONBUILD, we will make sure it follows with at
-	// least one argument
-	if upperCasedCmd == "ONBUILD" {
-		if ast.Next == nil {
-			buildsFailed.WithValues(metricsMissingOnbuildArgumentsError).Inc()
-			return errors.New("ONBUILD requires at least one argument")
-		}
-	}
-
-	if _, ok := evaluateTable[cmd]; ok {
-		return nil
-	}
-	buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
-	return errors.Errorf("unknown instruction: %s", upperCasedCmd)
-}

components/engine/builder/dockerfile/evaluator_test.go

@@ -1,13 +1,9 @@
 package dockerfile
 
 import (
-	"io/ioutil"
-	"strings"
 	"testing"
 
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/remotecontext"
 	"github.com/docker/docker/internal/testutil"
 	"github.com/docker/docker/pkg/archive"
@@ -15,8 +11,9 @@ import (
 )
 
 type dispatchTestCase struct {
-	name, dockerfile, expectedError string
-	files                           map[string]string
+	name, expectedError string
+	cmd                 instructions.Command
+	files               map[string]string
 }
 
 func init() {
@@ -24,108 +21,73 @@ func init() {
 }
 
 func initDispatchTestCases() []dispatchTestCase {
-	dispatchTestCases := []dispatchTestCase{{
-		name: "copyEmptyWhitespace",
-		dockerfile: `COPY
-	quux \
-      bar`,
-		expectedError: "COPY requires at least two arguments",
-	},
+	dispatchTestCases := []dispatchTestCase{
 		{
-			name:          "ONBUILD forbidden FROM",
-			dockerfile:    "ONBUILD FROM scratch",
-			expectedError: "FROM isn't allowed as an ONBUILD trigger",
-			files:         nil,
-		},
-		{
-			name:          "ONBUILD forbidden MAINTAINER",
-			dockerfile:    "ONBUILD MAINTAINER docker.io",
-			expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger",
-			files:         nil,
-		},
-		{
-			name:          "ARG two arguments",
-			dockerfile:    "ARG foo bar",
-			expectedError: "ARG requires exactly one argument",
-			files:         nil,
-		},
-		{
-			name:          "MAINTAINER unknown flag",
-			dockerfile:    "MAINTAINER --boo joe@example.com",
-			expectedError: "Unknown flag: boo",
-			files:         nil,
-		},
-		{
-			name:          "ADD multiple files to file",
-			dockerfile:    "ADD file1.txt file2.txt test",
+			name: "ADD multiple files to file",
+			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"file1.txt",
+				"file2.txt",
+				"test",
+			}},
 			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
 			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
 		},
 		{
-			name:          "JSON ADD multiple files to file",
-			dockerfile:    `ADD ["file1.txt", "file2.txt", "test"]`,
+			name: "Wildcard ADD multiple files to file",
+			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"file*.txt",
+				"test",
+			}},
 			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
 			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
 		},
 		{
-			name:          "Wildcard ADD multiple files to file",
-			dockerfile:    "ADD file*.txt test",
-			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
-			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
-		},
-		{
-			name:          "Wildcard JSON ADD multiple files to file",
-			dockerfile:    `ADD ["file*.txt", "test"]`,
-			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
-			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
-		},
-		{
-			name:          "COPY multiple files to file",
-			dockerfile:    "COPY file1.txt file2.txt test",
+			name: "COPY multiple files to file",
+			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"file1.txt",
+				"file2.txt",
+				"test",
+			}},
 			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
 			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
 		},
 		{
-			name:          "JSON COPY multiple files to file",
-			dockerfile:    `COPY ["file1.txt", "file2.txt", "test"]`,
-			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
-			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
-		},
-		{
-			name:          "ADD multiple files to file with whitespace",
-			dockerfile:    `ADD [ "test file1.txt", "test file2.txt", "test" ]`,
+			name: "ADD multiple files to file with whitespace",
+			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"test file1.txt",
+				"test file2.txt",
+				"test",
+			}},
 			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
 			files:         map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
 		},
 		{
-			name:          "COPY multiple files to file with whitespace",
-			dockerfile:    `COPY [ "test file1.txt", "test file2.txt", "test" ]`,
+			name: "COPY multiple files to file with whitespace",
+			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"test file1.txt",
+				"test file2.txt",
+				"test",
+			}},
 			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
 			files:         map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
 		},
 		{
-			name:          "COPY wildcard no files",
-			dockerfile:    `COPY file*.txt /tmp/`,
+			name: "COPY wildcard no files",
+			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"file*.txt",
+				"/tmp/",
+			}},
 			expectedError: "COPY failed: no source files were specified",
 			files:         nil,
 		},
 		{
-			name:          "COPY url",
-			dockerfile:    `COPY https://index.docker.io/robots.txt /`,
+			name: "COPY url",
+			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
+				"https://index.docker.io/robots.txt",
+				"/",
+			}},
 			expectedError: "source can't be a URL for COPY",
 			files:         nil,
-		},
-		{
-			name:          "Chaining ONBUILD",
-			dockerfile:    `ONBUILD ONBUILD RUN touch foobar`,
-			expectedError: "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed",
-			files:         nil,
-		},
-		{
-			name:          "Invalid instruction",
-			dockerfile:    `foo bar`,
-			expectedError: "unknown instruction: FOO",
-			files:         nil,
 		}}
 
 	return dispatchTestCases
@@ -171,33 +133,8 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
 		}
 	}()
 
-	r := strings.NewReader(testCase.dockerfile)
-	result, err := parser.Parse(r)
-
-	if err != nil {
-		t.Fatalf("Error when parsing Dockerfile: %s", err)
-	}
-
-	options := &types.ImageBuildOptions{
-		BuildArgs: make(map[string]*string),
-	}
-
-	b := &Builder{
-		options:   options,
-		Stdout:    ioutil.Discard,
-		buildArgs: newBuildArgs(options.BuildArgs),
-	}
-
-	shlex := NewShellLex(parser.DefaultEscapeToken)
-	n := result.AST
-	state := &dispatchState{runConfig: &container.Config{}}
-	opts := dispatchOptions{
-		state:   state,
-		stepMsg: formatStep(0, len(n.Children)),
-		node:    n.Children[0],
-		shlex:   shlex,
-		source:  context,
-	}
-	_, err = b.dispatch(opts)
+	b := newBuilderWithMockBackend()
+	sb := newDispatchRequest(b, '`', context, newBuildArgs(make(map[string]*string)), newStagesBuildResults())
+	err = dispatch(sb, testCase.cmd)
 	testutil.ErrorContains(t, err, testCase.expectedError)
 }

components/engine/builder/dockerfile/evaluator_unix.go

@@ -1,9 +0,0 @@
-// +build !windows
-
-package dockerfile
-
-// platformSupports is a short-term function to give users a quality error
-// message if a Dockerfile uses a command not supported on the platform.
-func platformSupports(command string) error {
-	return nil
-}

components/engine/builder/dockerfile/evaluator_windows.go

@@ -1,13 +0,0 @@
-package dockerfile
-
-import "fmt"
-
-// platformSupports is gives users a quality error message if a Dockerfile uses
-// a command not supported on the platform.
-func platformSupports(command string) error {
-	switch command {
-	case "stopsignal":
-		return fmt.Errorf("The daemon on this platform does not support the command '%s'", command)
-	}
-	return nil
-}

components/engine/builder/dockerfile/imagecontext.go

@@ -1,9 +1,6 @@
 package dockerfile
 
 import (
-	"strconv"
-	"strings"
-
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
@@ -13,79 +10,6 @@ import (
 	"golang.org/x/net/context"
 )
 
-type buildStage struct {
-	id string
-}
-
-func newBuildStage(imageID string) *buildStage {
-	return &buildStage{id: imageID}
-}
-
-func (b *buildStage) ImageID() string {
-	return b.id
-}
-
-func (b *buildStage) update(imageID string) {
-	b.id = imageID
-}
-
-// buildStages tracks each stage of a build so they can be retrieved by index
-// or by name.
-type buildStages struct {
-	sequence []*buildStage
-	byName   map[string]*buildStage
-}
-
-func newBuildStages() *buildStages {
-	return &buildStages{byName: make(map[string]*buildStage)}
-}
-
-func (s *buildStages) getByName(name string) (*buildStage, bool) {
-	stage, ok := s.byName[strings.ToLower(name)]
-	return stage, ok
-}
-
-func (s *buildStages) get(indexOrName string) (*buildStage, error) {
-	index, err := strconv.Atoi(indexOrName)
-	if err == nil {
-		if err := s.validateIndex(index); err != nil {
-			return nil, err
-		}
-		return s.sequence[index], nil
-	}
-	if im, ok := s.byName[strings.ToLower(indexOrName)]; ok {
-		return im, nil
-	}
-	return nil, nil
-}
-
-func (s *buildStages) validateIndex(i int) error {
-	if i < 0 || i >= len(s.sequence)-1 {
-		if i == len(s.sequence)-1 {
-			return errors.New("refers to current build stage")
-		}
-		return errors.New("index out of bounds")
-	}
-	return nil
-}
-
-func (s *buildStages) add(name string, image builder.Image) error {
-	stage := newBuildStage(image.ImageID())
-	name = strings.ToLower(name)
-	if len(name) > 0 {
-		if _, ok := s.byName[name]; ok {
-			return errors.Errorf("duplicate name %s", name)
-		}
-		s.byName[name] = stage
-	}
-	s.sequence = append(s.sequence, stage)
-	return nil
-}
-
-func (s *buildStages) update(imageID string) {
-	s.sequence[len(s.sequence)-1].update(imageID)
-}
-
 type getAndMountFunc func(string, bool) (builder.Image, builder.ReleaseableLayer, error)
 
 // imageSources mounts images and provides a cache for mounted images. It tracks

components/engine/builder/dockerfile/instructions/bflag.go

@@ -1,4 +1,4 @@
-package dockerfile
+package instructions
 
 import (
 	"fmt"

components/engine/builder/dockerfile/instructions/bflag_test.go

@@ -1,4 +1,4 @@
-package dockerfile
+package instructions
 
 import (
 	"testing"

components/engine/builder/dockerfile/instructions/commands.go

@@ -0,0 +1,396 @@
+package instructions
+
+import (
+	"errors"
+
+	"strings"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/strslice"
+)
+
+// KeyValuePair represent an arbitrary named value (usefull in slice insted of map[string] string to preserve ordering)
+type KeyValuePair struct {
+	Key   string
+	Value string
+}
+
+func (kvp *KeyValuePair) String() string {
+	return kvp.Key + "=" + kvp.Value
+}
+
+// Command is implemented by every command present in a dockerfile
+type Command interface {
+	Name() string
+}
+
+// KeyValuePairs is a slice of KeyValuePair
+type KeyValuePairs []KeyValuePair
+
+// withNameAndCode is the base of every command in a Dockerfile (String() returns its source code)
+type withNameAndCode struct {
+	code string
+	name string
+}
+
+func (c *withNameAndCode) String() string {
+	return c.code
+}
+
+// Name of the command
+func (c *withNameAndCode) Name() string {
+	return c.name
+}
+
+func newWithNameAndCode(req parseRequest) withNameAndCode {
+	return withNameAndCode{code: strings.TrimSpace(req.original), name: req.command}
+}
+
+// SingleWordExpander is a provider for variable expansion where 1 word => 1 output
+type SingleWordExpander func(word string) (string, error)
+
+// SupportsSingleWordExpansion interface marks a command as supporting variable expansion
+type SupportsSingleWordExpansion interface {
+	Expand(expander SingleWordExpander) error
+}
+
+// PlatformSpecific adds platform checks to a command
+type PlatformSpecific interface {
+	CheckPlatform(platform string) error
+}
+
+func expandKvp(kvp KeyValuePair, expander SingleWordExpander) (KeyValuePair, error) {
+	key, err := expander(kvp.Key)
+	if err != nil {
+		return KeyValuePair{}, err
+	}
+	value, err := expander(kvp.Value)
+	if err != nil {
+		return KeyValuePair{}, err
+	}
+	return KeyValuePair{Key: key, Value: value}, nil
+}
+func expandKvpsInPlace(kvps KeyValuePairs, expander SingleWordExpander) error {
+	for i, kvp := range kvps {
+		newKvp, err := expandKvp(kvp, expander)
+		if err != nil {
+			return err
+		}
+		kvps[i] = newKvp
+	}
+	return nil
+}
+
+func expandSliceInPlace(values []string, expander SingleWordExpander) error {
+	for i, v := range values {
+		newValue, err := expander(v)
+		if err != nil {
+			return err
+		}
+		values[i] = newValue
+	}
+	return nil
+}
+
+// EnvCommand : ENV key1 value1 [keyN valueN...]
+type EnvCommand struct {
+	withNameAndCode
+	Env KeyValuePairs // kvp slice instead of map to preserve ordering
+}
+
+// Expand variables
+func (c *EnvCommand) Expand(expander SingleWordExpander) error {
+	return expandKvpsInPlace(c.Env, expander)
+}
+
+// MaintainerCommand : MAINTAINER maintainer_name
+type MaintainerCommand struct {
+	withNameAndCode
+	Maintainer string
+}
+
+// LabelCommand : LABEL some json data describing the image
+//
+// Sets the Label variable foo to bar,
+//
+type LabelCommand struct {
+	withNameAndCode
+	Labels KeyValuePairs // kvp slice instead of map to preserve ordering
+}
+
+// Expand variables
+func (c *LabelCommand) Expand(expander SingleWordExpander) error {
+	return expandKvpsInPlace(c.Labels, expander)
+}
+
+// SourcesAndDest represent a list of source files and a destination
+type SourcesAndDest []string
+
+// Sources list the source paths
+func (s SourcesAndDest) Sources() []string {
+	res := make([]string, len(s)-1)
+	copy(res, s[:len(s)-1])
+	return res
+}
+
+// Dest path of the operation
+func (s SourcesAndDest) Dest() string {
+	return s[len(s)-1]
+}
+
+// AddCommand : ADD foo /path
+//
+// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling
+// exist here. If you do not wish to have this automatic handling, use COPY.
+//
+type AddCommand struct {
+	withNameAndCode
+	SourcesAndDest
+	Chown string
+}
+
+// Expand variables
+func (c *AddCommand) Expand(expander SingleWordExpander) error {
+	return expandSliceInPlace(c.SourcesAndDest, expander)
+}
+
+// CopyCommand : COPY foo /path
+//
+// Same as 'ADD' but without the tar and remote url handling.
+//
+type CopyCommand struct {
+	withNameAndCode
+	SourcesAndDest
+	From  string
+	Chown string
+}
+
+// Expand variables
+func (c *CopyCommand) Expand(expander SingleWordExpander) error {
+	return expandSliceInPlace(c.SourcesAndDest, expander)
+}
+
+// OnbuildCommand : ONBUILD <some other command>
+type OnbuildCommand struct {
+	withNameAndCode
+	Expression string
+}
+
+// WorkdirCommand : WORKDIR /tmp
+//
+// Set the working directory for future RUN/CMD/etc statements.
+//
+type WorkdirCommand struct {
+	withNameAndCode
+	Path string
+}
+
+// Expand variables
+func (c *WorkdirCommand) Expand(expander SingleWordExpander) error {
+	p, err := expander(c.Path)
+	if err != nil {
+		return err
+	}
+	c.Path = p
+	return nil
+}
+
+// ShellDependantCmdLine represents a cmdline optionaly prepended with the shell
+type ShellDependantCmdLine struct {
+	CmdLine      strslice.StrSlice
+	PrependShell bool
+}
+
+// RunCommand : RUN some command yo
+//
+// run a command and commit the image. Args are automatically prepended with
+// the current SHELL which defaults to 'sh -c' under linux or 'cmd /S /C' under
+// Windows, in the event there is only one argument The difference in processing:
+//
+// RUN echo hi          # sh -c echo hi       (Linux)
+// RUN echo hi          # cmd /S /C echo hi   (Windows)
+// RUN [ "echo", "hi" ] # echo hi
+//
+type RunCommand struct {
+	withNameAndCode
+	ShellDependantCmdLine
+}
+
+// CmdCommand : CMD foo
+//
+// Set the default command to run in the container (which may be empty).
+// Argument handling is the same as RUN.
+//
+type CmdCommand struct {
+	withNameAndCode
+	ShellDependantCmdLine
+}
+
+// HealthCheckCommand : HEALTHCHECK foo
+//
+// Set the default healthcheck command to run in the container (which may be empty).
+// Argument handling is the same as RUN.
+//
+type HealthCheckCommand struct {
+	withNameAndCode
+	Health *container.HealthConfig
+}
+
+// EntrypointCommand : ENTRYPOINT /usr/sbin/nginx
+//
+// Set the entrypoint to /usr/sbin/nginx. Will accept the CMD as the arguments
+// to /usr/sbin/nginx. Uses the default shell if not in JSON format.
+//
+// Handles command processing similar to CMD and RUN, only req.runConfig.Entrypoint
+// is initialized at newBuilder time instead of through argument parsing.
+//
+type EntrypointCommand struct {
+	withNameAndCode
+	ShellDependantCmdLine
+}
+
+// ExposeCommand : EXPOSE 6667/tcp 7000/tcp
+//
+// Expose ports for links and port mappings. This all ends up in
+// req.runConfig.ExposedPorts for runconfig.
+//
+type ExposeCommand struct {
+	withNameAndCode
+	Ports []string
+}
+
+// UserCommand : USER foo
+//
+// Set the user to 'foo' for future commands and when running the
+// ENTRYPOINT/CMD at container run time.
+//
+type UserCommand struct {
+	withNameAndCode
+	User string
+}
+
+// Expand variables
+func (c *UserCommand) Expand(expander SingleWordExpander) error {
+	p, err := expander(c.User)
+	if err != nil {
+		return err
+	}
+	c.User = p
+	return nil
+}
+
+// VolumeCommand : VOLUME /foo
+//
+// Expose the volume /foo for use. Will also accept the JSON array form.
+//
+type VolumeCommand struct {
+	withNameAndCode
+	Volumes []string
+}
+
+// Expand variables
+func (c *VolumeCommand) Expand(expander SingleWordExpander) error {
+	return expandSliceInPlace(c.Volumes, expander)
+}
+
+// StopSignalCommand : STOPSIGNAL signal
+//
+// Set the signal that will be used to kill the container.
+type StopSignalCommand struct {
+	withNameAndCode
+	Signal string
+}
+
+// Expand variables
+func (c *StopSignalCommand) Expand(expander SingleWordExpander) error {
+	p, err := expander(c.Signal)
+	if err != nil {
+		return err
+	}
+	c.Signal = p
+	return nil
+}
+
+// CheckPlatform checks that the command is supported in the target platform
+func (c *StopSignalCommand) CheckPlatform(platform string) error {
+	if platform == "windows" {
+		return errors.New("The daemon on this platform does not support the command stopsignal")
+	}
+	return nil
+}
+
+// ArgCommand : ARG name[=value]
+//
+// Adds the variable foo to the trusted list of variables that can be passed
+// to builder using the --build-arg flag for expansion/substitution or passing to 'run'.
+// Dockerfile author may optionally set a default value of this variable.
+type ArgCommand struct {
+	withNameAndCode
+	Key   string
+	Value *string
+}
+
+// Expand variables
+func (c *ArgCommand) Expand(expander SingleWordExpander) error {
+	p, err := expander(c.Key)
+	if err != nil {
+		return err
+	}
+	c.Key = p
+	if c.Value != nil {
+		p, err = expander(*c.Value)
+		if err != nil {
+			return err
+		}
+		c.Value = &p
+	}
+	return nil
+}
+
+// ShellCommand : SHELL powershell -command
+//
+// Set the non-default shell to use.
+type ShellCommand struct {
+	withNameAndCode
+	Shell strslice.StrSlice
+}
+
+// Stage represents a single stage in a multi-stage build
+type Stage struct {
+	Name       string
+	Commands   []Command
+	BaseName   string
+	SourceCode string
+}
+
+// AddCommand to the stage
+func (s *Stage) AddCommand(cmd Command) {
+	// todo: validate cmd type
+	s.Commands = append(s.Commands, cmd)
+}
+
+// IsCurrentStage check if the stage name is the current stage
+func IsCurrentStage(s []Stage, name string) bool {
+	if len(s) == 0 {
+		return false
+	}
+	return s[len(s)-1].Name == name
+}
+
+// CurrentStage return the last stage in a slice
+func CurrentStage(s []Stage) (*Stage, error) {
+	if len(s) == 0 {
+		return nil, errors.New("No build stage in current context")
+	}
+	return &s[len(s)-1], nil
+}
+
+// HasStage looks for the presence of a given stage name
+func HasStage(s []Stage, name string) (int, bool) {
+	for i, stage := range s {
+		if stage.Name == name {
+			return i, true
+		}
+	}
+	return -1, false
+}

components/engine/builder/dockerfile/instructions/errors_unix.go

@@ -0,0 +1,9 @@
+// +build !windows
+
+package instructions
+
+import "fmt"
+
+func errNotJSON(command, _ string) error {
+	return fmt.Errorf("%s requires the arguments to be in JSON form", command)
+}

components/engine/builder/dockerfile/instructions/errors_windows.go

@@ -0,0 +1,27 @@
+package instructions
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+func errNotJSON(command, original string) error {
+	// For Windows users, give a hint if it looks like it might contain
+	// a path which hasn't been escaped such as ["c:\windows\system32\prog.exe", "-param"],
+	// as JSON must be escaped. Unfortunate...
+	//
+	// Specifically looking for quote-driveletter-colon-backslash, there's no
+	// double backslash and a [] pair. No, this is not perfect, but it doesn't
+	// have to be. It's simply a hint to make life a little easier.
+	extra := ""
+	original = filepath.FromSlash(strings.ToLower(strings.Replace(strings.ToLower(original), strings.ToLower(command)+" ", "", -1)))
+	if len(regexp.MustCompile(`"[a-z]:\\.*`).FindStringSubmatch(original)) > 0 &&
+		!strings.Contains(original, `\\`) &&
+		strings.Contains(original, "[") &&
+		strings.Contains(original, "]") {
+		extra = fmt.Sprintf(`. It looks like '%s' includes a file path without an escaped back-slash. JSON requires back-slashes to be escaped such as ["c:\\path\\to\\file.exe", "/parameter"]`, original)
+	}
+	return fmt.Errorf("%s requires the arguments to be in JSON form%s", command, extra)
+}

components/engine/builder/dockerfile/instructions/parse.go

@@ -0,0 +1,635 @@
+package instructions
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/builder/dockerfile/command"
+	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/pkg/errors"
+)
+
+type parseRequest struct {
+	command    string
+	args       []string
+	attributes map[string]bool
+	flags      *BFlags
+	original   string
+}
+
+func nodeArgs(node *parser.Node) []string {
+	result := []string{}
+	for ; node.Next != nil; node = node.Next {
+		arg := node.Next
+		if len(arg.Children) == 0 {
+			result = append(result, arg.Value)
+		} else if len(arg.Children) == 1 {
+			//sub command
+			result = append(result, arg.Children[0].Value)
+			result = append(result, nodeArgs(arg.Children[0])...)
+		}
+	}
+	return result
+}
+
+func newParseRequestFromNode(node *parser.Node) parseRequest {
+	return parseRequest{
+		command:    node.Value,
+		args:       nodeArgs(node),
+		attributes: node.Attributes,
+		original:   node.Original,
+		flags:      NewBFlagsWithArgs(node.Flags),
+	}
+}
+
+// ParseInstruction converts an AST to a typed instruction (either a command or a build stage beginning when encountering a `FROM` statement)
+func ParseInstruction(node *parser.Node) (interface{}, error) {
+	req := newParseRequestFromNode(node)
+	switch node.Value {
+	case command.Env:
+		return parseEnv(req)
+	case command.Maintainer:
+		return parseMaintainer(req)
+	case command.Label:
+		return parseLabel(req)
+	case command.Add:
+		return parseAdd(req)
+	case command.Copy:
+		return parseCopy(req)
+	case command.From:
+		return parseFrom(req)
+	case command.Onbuild:
+		return parseOnBuild(req)
+	case command.Workdir:
+		return parseWorkdir(req)
+	case command.Run:
+		return parseRun(req)
+	case command.Cmd:
+		return parseCmd(req)
+	case command.Healthcheck:
+		return parseHealthcheck(req)
+	case command.Entrypoint:
+		return parseEntrypoint(req)
+	case command.Expose:
+		return parseExpose(req)
+	case command.User:
+		return parseUser(req)
+	case command.Volume:
+		return parseVolume(req)
+	case command.StopSignal:
+		return parseStopSignal(req)
+	case command.Arg:
+		return parseArg(req)
+	case command.Shell:
+		return parseShell(req)
+	}
+
+	return nil, &UnknownInstruction{Instruction: node.Value, Line: node.StartLine}
+}
+
+// ParseCommand converts an AST to a typed Command
+func ParseCommand(node *parser.Node) (Command, error) {
+	s, err := ParseInstruction(node)
+	if err != nil {
+		return nil, err
+	}
+	if c, ok := s.(Command); ok {
+		return c, nil
+	}
+	return nil, errors.Errorf("%T is not a command type", s)
+}
+
+// UnknownInstruction represents an error occuring when a command is unresolvable
+type UnknownInstruction struct {
+	Line        int
+	Instruction string
+}
+
+func (e *UnknownInstruction) Error() string {
+	return fmt.Sprintf("unknown instruction: %s", strings.ToUpper(e.Instruction))
+}
+
+// IsUnknownInstruction checks if the error is an UnknownInstruction or a parseError containing an UnknownInstruction
+func IsUnknownInstruction(err error) bool {
+	_, ok := err.(*UnknownInstruction)
+	if !ok {
+		var pe *parseError
+		if pe, ok = err.(*parseError); ok {
+			_, ok = pe.inner.(*UnknownInstruction)
+		}
+	}
+	return ok
+}
+
+type parseError struct {
+	inner error
+	node  *parser.Node
+}
+
+func (e *parseError) Error() string {
+	return fmt.Sprintf("Dockerfile parse error line %d: %v", e.node.StartLine, e.inner.Error())
+}
+
+// Parse a docker file into a collection of buildable stages
+func Parse(ast *parser.Node) (stages []Stage, metaArgs []ArgCommand, err error) {
+	for _, n := range ast.Children {
+		cmd, err := ParseInstruction(n)
+		if err != nil {
+			return nil, nil, &parseError{inner: err, node: n}
+		}
+		if len(stages) == 0 {
+			// meta arg case
+			if a, isArg := cmd.(*ArgCommand); isArg {
+				metaArgs = append(metaArgs, *a)
+				continue
+			}
+		}
+		switch c := cmd.(type) {
+		case *Stage:
+			stages = append(stages, *c)
+		case Command:
+			stage, err := CurrentStage(stages)
+			if err != nil {
+				return nil, nil, err
+			}
+			stage.AddCommand(c)
+		default:
+			return nil, nil, errors.Errorf("%T is not a command type", cmd)
+		}
+
+	}
+	return stages, metaArgs, nil
+}
+
+func parseKvps(args []string, cmdName string) (KeyValuePairs, error) {
+	if len(args) == 0 {
+		return nil, errAtLeastOneArgument(cmdName)
+	}
+	if len(args)%2 != 0 {
+		// should never get here, but just in case
+		return nil, errTooManyArguments(cmdName)
+	}
+	var res KeyValuePairs
+	for j := 0; j < len(args); j += 2 {
+		if len(args[j]) == 0 {
+			return nil, errBlankCommandNames(cmdName)
+		}
+		name := args[j]
+		value := args[j+1]
+		res = append(res, KeyValuePair{Key: name, Value: value})
+	}
+	return res, nil
+}
+
+func parseEnv(req parseRequest) (*EnvCommand, error) {
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	envs, err := parseKvps(req.args, "ENV")
+	if err != nil {
+		return nil, err
+	}
+	return &EnvCommand{
+		Env:             envs,
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+}
+
+func parseMaintainer(req parseRequest) (*MaintainerCommand, error) {
+	if len(req.args) != 1 {
+		return nil, errExactlyOneArgument("MAINTAINER")
+	}
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	return &MaintainerCommand{
+		Maintainer:      req.args[0],
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+}
+
+func parseLabel(req parseRequest) (*LabelCommand, error) {
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+
+	labels, err := parseKvps(req.args, "LABEL")
+	if err != nil {
+		return nil, err
+	}
+
+	return &LabelCommand{
+		Labels:          labels,
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+}
+
+func parseAdd(req parseRequest) (*AddCommand, error) {
+	if len(req.args) < 2 {
+		return nil, errNoDestinationArgument("ADD")
+	}
+	flChown := req.flags.AddString("chown", "")
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	return &AddCommand{
+		SourcesAndDest:  SourcesAndDest(req.args),
+		withNameAndCode: newWithNameAndCode(req),
+		Chown:           flChown.Value,
+	}, nil
+}
+
+func parseCopy(req parseRequest) (*CopyCommand, error) {
+	if len(req.args) < 2 {
+		return nil, errNoDestinationArgument("COPY")
+	}
+	flChown := req.flags.AddString("chown", "")
+	flFrom := req.flags.AddString("from", "")
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	return &CopyCommand{
+		SourcesAndDest:  SourcesAndDest(req.args),
+		From:            flFrom.Value,
+		withNameAndCode: newWithNameAndCode(req),
+		Chown:           flChown.Value,
+	}, nil
+}
+
+func parseFrom(req parseRequest) (*Stage, error) {
+	stageName, err := parseBuildStageName(req.args)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	code := strings.TrimSpace(req.original)
+
+	return &Stage{
+		BaseName:   req.args[0],
+		Name:       stageName,
+		SourceCode: code,
+		Commands:   []Command{},
+	}, nil
+
+}
+
+func parseBuildStageName(args []string) (string, error) {
+	stageName := ""
+	switch {
+	case len(args) == 3 && strings.EqualFold(args[1], "as"):
+		stageName = strings.ToLower(args[2])
+		if ok, _ := regexp.MatchString("^[a-z][a-z0-9-_\\.]*$", stageName); !ok {
+			return "", errors.Errorf("invalid name for build stage: %q, name can't start with a number or contain symbols", stageName)
+		}
+	case len(args) != 1:
+		return "", errors.New("FROM requires either one or three arguments")
+	}
+
+	return stageName, nil
+}
+
+func parseOnBuild(req parseRequest) (*OnbuildCommand, error) {
+	if len(req.args) == 0 {
+		return nil, errAtLeastOneArgument("ONBUILD")
+	}
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+
+	triggerInstruction := strings.ToUpper(strings.TrimSpace(req.args[0]))
+	switch strings.ToUpper(triggerInstruction) {
+	case "ONBUILD":
+		return nil, errors.New("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed")
+	case "MAINTAINER", "FROM":
+		return nil, fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction)
+	}
+
+	original := regexp.MustCompile(`(?i)^\s*ONBUILD\s*`).ReplaceAllString(req.original, "")
+	return &OnbuildCommand{
+		Expression:      original,
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+
+}
+
+func parseWorkdir(req parseRequest) (*WorkdirCommand, error) {
+	if len(req.args) != 1 {
+		return nil, errExactlyOneArgument("WORKDIR")
+	}
+
+	err := req.flags.Parse()
+	if err != nil {
+		return nil, err
+	}
+	return &WorkdirCommand{
+		Path:            req.args[0],
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+
+}
+
+func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependantCmdLine {
+	args := handleJSONArgs(req.args, req.attributes)
+	cmd := strslice.StrSlice(args)
+	if emptyAsNil && len(cmd) == 0 {
+		cmd = nil
+	}
+	return ShellDependantCmdLine{
+		CmdLine:      cmd,
+		PrependShell: !req.attributes["json"],
+	}
+}
+
+func parseRun(req parseRequest) (*RunCommand, error) {
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	return &RunCommand{
+		ShellDependantCmdLine: parseShellDependentCommand(req, false),
+		withNameAndCode:       newWithNameAndCode(req),
+	}, nil
+
+}
+
+func parseCmd(req parseRequest) (*CmdCommand, error) {
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	return &CmdCommand{
+		ShellDependantCmdLine: parseShellDependentCommand(req, false),
+		withNameAndCode:       newWithNameAndCode(req),
+	}, nil
+
+}
+
+func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+
+	cmd := &EntrypointCommand{
+		ShellDependantCmdLine: parseShellDependentCommand(req, true),
+		withNameAndCode:       newWithNameAndCode(req),
+	}
+
+	return cmd, nil
+}
+
+// parseOptInterval(flag) is the duration of flag.Value, or 0 if
+// empty. An error is reported if the value is given and less than minimum duration.
+func parseOptInterval(f *Flag) (time.Duration, error) {
+	s := f.Value
+	if s == "" {
+		return 0, nil
+	}
+	d, err := time.ParseDuration(s)
+	if err != nil {
+		return 0, err
+	}
+	if d < container.MinimumDuration {
+		return 0, fmt.Errorf("Interval %#v cannot be less than %s", f.name, container.MinimumDuration)
+	}
+	return d, nil
+}
+func parseHealthcheck(req parseRequest) (*HealthCheckCommand, error) {
+	if len(req.args) == 0 {
+		return nil, errAtLeastOneArgument("HEALTHCHECK")
+	}
+	cmd := &HealthCheckCommand{
+		withNameAndCode: newWithNameAndCode(req),
+	}
+
+	typ := strings.ToUpper(req.args[0])
+	args := req.args[1:]
+	if typ == "NONE" {
+		if len(args) != 0 {
+			return nil, errors.New("HEALTHCHECK NONE takes no arguments")
+		}
+		test := strslice.StrSlice{typ}
+		cmd.Health = &container.HealthConfig{
+			Test: test,
+		}
+	} else {
+
+		healthcheck := container.HealthConfig{}
+
+		flInterval := req.flags.AddString("interval", "")
+		flTimeout := req.flags.AddString("timeout", "")
+		flStartPeriod := req.flags.AddString("start-period", "")
+		flRetries := req.flags.AddString("retries", "")
+
+		if err := req.flags.Parse(); err != nil {
+			return nil, err
+		}
+
+		switch typ {
+		case "CMD":
+			cmdSlice := handleJSONArgs(args, req.attributes)
+			if len(cmdSlice) == 0 {
+				return nil, errors.New("Missing command after HEALTHCHECK CMD")
+			}
+
+			if !req.attributes["json"] {
+				typ = "CMD-SHELL"
+			}
+
+			healthcheck.Test = strslice.StrSlice(append([]string{typ}, cmdSlice...))
+		default:
+			return nil, fmt.Errorf("Unknown type %#v in HEALTHCHECK (try CMD)", typ)
+		}
+
+		interval, err := parseOptInterval(flInterval)
+		if err != nil {
+			return nil, err
+		}
+		healthcheck.Interval = interval
+
+		timeout, err := parseOptInterval(flTimeout)
+		if err != nil {
+			return nil, err
+		}
+		healthcheck.Timeout = timeout
+
+		startPeriod, err := parseOptInterval(flStartPeriod)
+		if err != nil {
+			return nil, err
+		}
+		healthcheck.StartPeriod = startPeriod
+
+		if flRetries.Value != "" {
+			retries, err := strconv.ParseInt(flRetries.Value, 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			if retries < 1 {
+				return nil, fmt.Errorf("--retries must be at least 1 (not %d)", retries)
+			}
+			healthcheck.Retries = int(retries)
+		} else {
+			healthcheck.Retries = 0
+		}
+
+		cmd.Health = &healthcheck
+	}
+	return cmd, nil
+}
+
+func parseExpose(req parseRequest) (*ExposeCommand, error) {
+	portsTab := req.args
+
+	if len(req.args) == 0 {
+		return nil, errAtLeastOneArgument("EXPOSE")
+	}
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+
+	sort.Strings(portsTab)
+	return &ExposeCommand{
+		Ports:           portsTab,
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+}
+
+func parseUser(req parseRequest) (*UserCommand, error) {
+	if len(req.args) != 1 {
+		return nil, errExactlyOneArgument("USER")
+	}
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	return &UserCommand{
+		User:            req.args[0],
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+}
+
+func parseVolume(req parseRequest) (*VolumeCommand, error) {
+	if len(req.args) == 0 {
+		return nil, errAtLeastOneArgument("VOLUME")
+	}
+
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+
+	cmd := &VolumeCommand{
+		withNameAndCode: newWithNameAndCode(req),
+	}
+
+	for _, v := range req.args {
+		v = strings.TrimSpace(v)
+		if v == "" {
+			return nil, errors.New("VOLUME specified can not be an empty string")
+		}
+		cmd.Volumes = append(cmd.Volumes, v)
+	}
+	return cmd, nil
+
+}
+
+func parseStopSignal(req parseRequest) (*StopSignalCommand, error) {
+	if len(req.args) != 1 {
+		return nil, errExactlyOneArgument("STOPSIGNAL")
+	}
+	sig := req.args[0]
+
+	cmd := &StopSignalCommand{
+		Signal:          sig,
+		withNameAndCode: newWithNameAndCode(req),
+	}
+	return cmd, nil
+
+}
+
+func parseArg(req parseRequest) (*ArgCommand, error) {
+	if len(req.args) != 1 {
+		return nil, errExactlyOneArgument("ARG")
+	}
+
+	var (
+		name     string
+		newValue *string
+	)
+
+	arg := req.args[0]
+	// 'arg' can just be a name or name-value pair. Note that this is different
+	// from 'env' that handles the split of name and value at the parser level.
+	// The reason for doing it differently for 'arg' is that we support just
+	// defining an arg and not assign it a value (while 'env' always expects a
+	// name-value pair). If possible, it will be good to harmonize the two.
+	if strings.Contains(arg, "=") {
+		parts := strings.SplitN(arg, "=", 2)
+		if len(parts[0]) == 0 {
+			return nil, errBlankCommandNames("ARG")
+		}
+
+		name = parts[0]
+		newValue = &parts[1]
+	} else {
+		name = arg
+	}
+
+	return &ArgCommand{
+		Key:             name,
+		Value:           newValue,
+		withNameAndCode: newWithNameAndCode(req),
+	}, nil
+}
+
+func parseShell(req parseRequest) (*ShellCommand, error) {
+	if err := req.flags.Parse(); err != nil {
+		return nil, err
+	}
+	shellSlice := handleJSONArgs(req.args, req.attributes)
+	switch {
+	case len(shellSlice) == 0:
+		// SHELL []
+		return nil, errAtLeastOneArgument("SHELL")
+	case req.attributes["json"]:
+		// SHELL ["powershell", "-command"]
+
+		return &ShellCommand{
+			Shell:           strslice.StrSlice(shellSlice),
+			withNameAndCode: newWithNameAndCode(req),
+		}, nil
+	default:
+		// SHELL powershell -command - not JSON
+		return nil, errNotJSON("SHELL", req.original)
+	}
+}
+
+func errAtLeastOneArgument(command string) error {
+	return errors.Errorf("%s requires at least one argument", command)
+}
+
+func errExactlyOneArgument(command string) error {
+	return errors.Errorf("%s requires exactly one argument", command)
+}
+
+func errNoDestinationArgument(command string) error {
+	return errors.Errorf("%s requires at least two arguments, but only one was provided. Destination could not be determined.", command)
+}
+
+func errBlankCommandNames(command string) error {
+	return errors.Errorf("%s names can not be blank", command)
+}
+
+func errTooManyArguments(command string) error {
+	return errors.Errorf("Bad input to %s, too many arguments", command)
+}

components/engine/builder/dockerfile/instructions/parse_test.go

@@ -0,0 +1,204 @@
+package instructions
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/docker/docker/builder/dockerfile/command"
+	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/internal/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommandsExactlyOneArgument(t *testing.T) {
+	commands := []string{
+		"MAINTAINER",
+		"WORKDIR",
+		"USER",
+		"STOPSIGNAL",
+	}
+
+	for _, command := range commands {
+		ast, err := parser.Parse(strings.NewReader(command))
+		require.NoError(t, err)
+		_, err = ParseInstruction(ast.AST.Children[0])
+		assert.EqualError(t, err, errExactlyOneArgument(command).Error())
+	}
+}
+
+func TestCommandsAtLeastOneArgument(t *testing.T) {
+	commands := []string{
+		"ENV",
+		"LABEL",
+		"ONBUILD",
+		"HEALTHCHECK",
+		"EXPOSE",
+		"VOLUME",
+	}
+
+	for _, command := range commands {
+		ast, err := parser.Parse(strings.NewReader(command))
+		require.NoError(t, err)
+		_, err = ParseInstruction(ast.AST.Children[0])
+		assert.EqualError(t, err, errAtLeastOneArgument(command).Error())
+	}
+}
+
+func TestCommandsNoDestinationArgument(t *testing.T) {
+	commands := []string{
+		"ADD",
+		"COPY",
+	}
+
+	for _, command := range commands {
+		ast, err := parser.Parse(strings.NewReader(command + " arg1"))
+		require.NoError(t, err)
+		_, err = ParseInstruction(ast.AST.Children[0])
+		assert.EqualError(t, err, errNoDestinationArgument(command).Error())
+	}
+}
+
+func TestCommandsTooManyArguments(t *testing.T) {
+	commands := []string{
+		"ENV",
+		"LABEL",
+	}
+
+	for _, command := range commands {
+		node := &parser.Node{
+			Original: command + "arg1 arg2 arg3",
+			Value:    strings.ToLower(command),
+			Next: &parser.Node{
+				Value: "arg1",
+				Next: &parser.Node{
+					Value: "arg2",
+					Next: &parser.Node{
+						Value: "arg3",
+					},
+				},
+			},
+		}
+		_, err := ParseInstruction(node)
+		assert.EqualError(t, err, errTooManyArguments(command).Error())
+	}
+}
+
+func TestCommandsBlankNames(t *testing.T) {
+	commands := []string{
+		"ENV",
+		"LABEL",
+	}
+
+	for _, command := range commands {
+		node := &parser.Node{
+			Original: command + " =arg2",
+			Value:    strings.ToLower(command),
+			Next: &parser.Node{
+				Value: "",
+				Next: &parser.Node{
+					Value: "arg2",
+				},
+			},
+		}
+		_, err := ParseInstruction(node)
+		assert.EqualError(t, err, errBlankCommandNames(command).Error())
+	}
+}
+
+func TestHealthCheckCmd(t *testing.T) {
+	node := &parser.Node{
+		Value: command.Healthcheck,
+		Next: &parser.Node{
+			Value: "CMD",
+			Next: &parser.Node{
+				Value: "hello",
+				Next: &parser.Node{
+					Value: "world",
+				},
+			},
+		},
+	}
+	cmd, err := ParseInstruction(node)
+	assert.NoError(t, err)
+	hc, ok := cmd.(*HealthCheckCommand)
+	assert.True(t, ok)
+	expected := []string{"CMD-SHELL", "hello world"}
+	assert.Equal(t, expected, hc.Health.Test)
+}
+
+func TestParseOptInterval(t *testing.T) {
+	flInterval := &Flag{
+		name:     "interval",
+		flagType: stringType,
+		Value:    "50ns",
+	}
+	_, err := parseOptInterval(flInterval)
+	testutil.ErrorContains(t, err, "cannot be less than 1ms")
+
+	flInterval.Value = "1ms"
+	_, err = parseOptInterval(flInterval)
+	require.NoError(t, err)
+}
+
+func TestErrorCases(t *testing.T) {
+	cases := []struct {
+		name          string
+		dockerfile    string
+		expectedError string
+	}{
+		{
+			name: "copyEmptyWhitespace",
+			dockerfile: `COPY	
+		quux \
+      bar`,
+			expectedError: "COPY requires at least two arguments",
+		},
+		{
+			name:          "ONBUILD forbidden FROM",
+			dockerfile:    "ONBUILD FROM scratch",
+			expectedError: "FROM isn't allowed as an ONBUILD trigger",
+		},
+		{
+			name:          "ONBUILD forbidden MAINTAINER",
+			dockerfile:    "ONBUILD MAINTAINER docker.io",
+			expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger",
+		},
+		{
+			name:          "ARG two arguments",
+			dockerfile:    "ARG foo bar",
+			expectedError: "ARG requires exactly one argument",
+		},
+		{
+			name:          "MAINTAINER unknown flag",
+			dockerfile:    "MAINTAINER --boo joe@example.com",
+			expectedError: "Unknown flag: boo",
+		},
+		{
+			name:          "Chaining ONBUILD",
+			dockerfile:    `ONBUILD ONBUILD RUN touch foobar`,
+			expectedError: "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed",
+		},
+		{
+			name:          "Invalid instruction",
+			dockerfile:    `foo bar`,
+			expectedError: "unknown instruction: FOO",
+		},
+	}
+	for _, c := range cases {
+		r := strings.NewReader(c.dockerfile)
+		ast, err := parser.Parse(r)
+
+		if err != nil {
+			t.Fatalf("Error when parsing Dockerfile: %s", err)
+		}
+		n := ast.AST.Children[0]
+		_, err = ParseInstruction(n)
+		if err != nil {
+			testutil.ErrorContains(t, err, c.expectedError)
+			return
+		}
+		t.Fatalf("No error when executing test %s", c.name)
+	}
+
+}

components/engine/builder/dockerfile/instructions/support.go

@@ -1,4 +1,4 @@
-package dockerfile
+package instructions
 
 import "strings"
 

components/engine/builder/dockerfile/instructions/support_test.go

@@ -1,4 +1,4 @@
-package dockerfile
+package instructions
 
 import "testing"
 

components/engine/builder/dockerfile/internals.go

@@ -124,7 +124,6 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta
 	}
 
 	dispatchState.imageID = imageID
-	b.buildStages.update(imageID)
 	return nil
 }
 
@@ -164,7 +163,6 @@ func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runC
 
 	state.imageID = exportedImage.ImageID()
 	b.imageSources.Add(newImageMount(exportedImage, newLayer))
-	b.buildStages.update(state.imageID)
 	return nil
 }
 
@@ -460,7 +458,6 @@ func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.
 	fmt.Fprint(b.Stdout, " ---> Using cache\n")
 
 	dispatchState.imageID = cachedID
-	b.buildStages.update(dispatchState.imageID)
 	return true, nil
 }
 

components/engine/builder/fscache/fscache.go

@@ -1,7 +1,10 @@
 package fscache
 
 import (
+	"archive/tar"
+	"crypto/sha256"
 	"encoding/json"
+	"hash"
 	"os"
 	"path/filepath"
 	"sort"
@@ -11,8 +14,10 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/directory"
 	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/pkg/tarsum"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -578,6 +583,10 @@ func (dc *detectChanges) MarkSupported(v bool) {
 	dc.supported = v
 }
 
+func (dc *detectChanges) ContentHasher() fsutil.ContentHasher {
+	return newTarsumHash
+}
+
 type wrappedContext struct {
 	builder.Source
 	closer func() error
@@ -607,3 +616,40 @@ func (s sortableCacheSources) Less(i, j int) bool {
 func (s sortableCacheSources) Swap(i, j int) {
 	s[i], s[j] = s[j], s[i]
 }
+
+func newTarsumHash(stat *fsutil.Stat) (hash.Hash, error) {
+	fi := &fsutil.StatInfo{stat}
+	p := stat.Path
+	if fi.IsDir() {
+		p += string(os.PathSeparator)
+	}
+	h, err := archive.FileInfoHeader(p, fi, stat.Linkname)
+	if err != nil {
+		return nil, err
+	}
+	h.Name = p
+	h.Uid = int(stat.Uid)
+	h.Gid = int(stat.Gid)
+	h.Linkname = stat.Linkname
+	if stat.Xattrs != nil {
+		h.Xattrs = make(map[string]string)
+		for k, v := range stat.Xattrs {
+			h.Xattrs[k] = string(v)
+		}
+	}
+
+	tsh := &tarsumHash{h: h, Hash: sha256.New()}
+	tsh.Reset()
+	return tsh, nil
+}
+
+// Reset resets the Hash to its initial state.
+func (tsh *tarsumHash) Reset() {
+	tsh.Hash.Reset()
+	tarsum.WriteV1Header(tsh.h, tsh.Hash)
+}
+
+type tarsumHash struct {
+	hash.Hash
+	h *tar.Header
+}

components/engine/builder/remotecontext/tarsum.go

@@ -5,15 +5,15 @@ import (
 	"os"
 	"sync"
 
-	iradix "github.com/hashicorp/go-immutable-radix"
-
 	"github.com/docker/docker/pkg/containerfs"
+	iradix "github.com/hashicorp/go-immutable-radix"
+	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
 )
 
 type hashed interface {
-	Hash() string
+	Digest() digest.Digest
 }
 
 // CachableSource is a source that contains cache records for its contents
@@ -110,7 +110,7 @@ func (cs *CachableSource) HandleChange(kind fsutil.ChangeKind, p string, fi os.F
 	}
 
 	hfi := &fileInfo{
-		sum: h.Hash(),
+		sum: h.Digest().Hex(),
 	}
 	cs.txn.Insert([]byte(p), hfi)
 	cs.mu.Unlock()

components/engine/client/errors.go

@@ -64,6 +64,8 @@ func wrapResponseError(err error, resp serverResponse, object, id string) error
 		return nil
 	case resp.statusCode == http.StatusNotFound:
 		return objectNotFoundError{object: object, id: id}
+	case resp.statusCode == http.StatusNotImplemented:
+		return notImplementedError{message: err.Error()}
 	default:
 		return err
 	}
@@ -157,6 +159,26 @@ func IsErrPluginPermissionDenied(err error) bool {
 	return ok
 }
 
+type notImplementedError struct {
+	message string
+}
+
+func (e notImplementedError) Error() string {
+	return e.message
+}
+
+func (e notImplementedError) NotImplemented() bool {
+	return true
+}
+
+// IsErrNotImplemented returns true if the error is a NotImplemented error.
+// This is returned by the API when a requested feature has not been
+// implemented.
+func IsErrNotImplemented(err error) bool {
+	te, ok := err.(notImplementedError)
+	return ok && te.NotImplemented()
+}
+
 // NewVersionError returns an error if the APIVersion required
 // if less than the current supported version
 func (cli *Client) NewVersionError(APIrequired, feature string) error {

components/engine/client/plugin_list.go

@@ -23,7 +23,7 @@ func (cli *Client) PluginList(ctx context.Context, filter filters.Args) (types.P
 	}
 	resp, err := cli.get(ctx, "/plugins", query, nil)
 	if err != nil {
-		return plugins, err
+		return plugins, wrapResponseError(err, resp, "plugin", "")
 	}
 
 	err = json.NewDecoder(resp.body).Decode(&plugins)

components/engine/cmd/dockerd/daemon.go

@@ -38,7 +38,7 @@ import (
 	"github.com/docker/docker/libcontainerd"
 	dopts "github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/authorization"
-	"github.com/docker/docker/pkg/jsonlog"
+	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/pidfile"
 	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/docker/pkg/signal"
@@ -94,7 +94,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 	}
 
 	logrus.SetFormatter(&logrus.TextFormatter{
-		TimestampFormat: jsonlog.RFC3339NanoFixed,
+		TimestampFormat: jsonmessage.RFC3339NanoFixed,
 		DisableColors:   cli.Config.RawLogs,
 		FullTimestamp:   true,
 	})

components/engine/container/state.go

@@ -357,6 +357,15 @@ func (s *State) ResetRemovalInProgress() {
 	s.Unlock()
 }
 
+// IsRemovalInProgress returns whether the RemovalInProgress flag is set.
+// Used by Container to check whether a container is being removed.
+func (s *State) IsRemovalInProgress() bool {
+	s.Lock()
+	res := s.RemovalInProgress
+	s.Unlock()
+	return res
+}
+
 // SetDead sets the container state to "dead"
 func (s *State) SetDead() {
 	s.Lock()
@@ -364,6 +373,14 @@ func (s *State) SetDead() {
 	s.Unlock()
 }
 
+// IsDead returns whether the Dead flag is set. Used by Container to check whether a container is dead.
+func (s *State) IsDead() bool {
+	s.Lock()
+	res := s.Dead
+	s.Unlock()
+	return res
+}
+
 // SetRemoved assumes this container is already in the "dead" state and
 // closes the internal waitRemove channel to unblock callers waiting for a
 // container to be removed.

components/engine/container/stream/attach.go

@@ -7,7 +7,6 @@ import (
 	"golang.org/x/net/context"
 
 	"github.com/docker/docker/pkg/pools"
-	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/pkg/term"
 	"github.com/sirupsen/logrus"
 )
@@ -58,7 +57,7 @@ func (c *Config) AttachStreams(cfg *AttachConfig) {
 }
 
 // CopyStreams starts goroutines to copy data in and out to/from the container
-func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) chan error {
+func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) <-chan error {
 	var (
 		wg     sync.WaitGroup
 		errors = make(chan error, 3)
@@ -137,35 +136,42 @@ func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) chan error
 	go attachStream("stdout", cfg.Stdout, cfg.CStdout)
 	go attachStream("stderr", cfg.Stderr, cfg.CStderr)
 
-	return promise.Go(func() error {
-		done := make(chan struct{})
-		go func() {
-			wg.Wait()
-			close(done)
+	errs := make(chan error, 1)
+
+	go func() {
+		defer close(errs)
+		errs <- func() error {
+			done := make(chan struct{})
+			go func() {
+				wg.Wait()
+				close(done)
+			}()
+			select {
+			case <-done:
+			case <-ctx.Done():
+				// close all pipes
+				if cfg.CStdin != nil {
+					cfg.CStdin.Close()
+				}
+				if cfg.CStdout != nil {
+					cfg.CStdout.Close()
+				}
+				if cfg.CStderr != nil {
+					cfg.CStderr.Close()
+				}
+				<-done
+			}
+			close(errors)
+			for err := range errors {
+				if err != nil {
+					return err
+				}
+			}
+			return nil
 		}()
-		select {
-		case <-done:
-		case <-ctx.Done():
-			// close all pipes
-			if cfg.CStdin != nil {
-				cfg.CStdin.Close()
-			}
-			if cfg.CStdout != nil {
-				cfg.CStdout.Close()
-			}
-			if cfg.CStderr != nil {
-				cfg.CStderr.Close()
-			}
-			<-done
-		}
-		close(errors)
-		for err := range errors {
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
+	}()
+
+	return errs
 }
 
 func copyEscapable(dst io.Writer, src io.ReadCloser, keys []byte) (written int64, err error) {

components/engine/daemon/commit.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"runtime"
 	"strings"
@@ -133,6 +134,16 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
 		return "", errors.Errorf("%+v does not support commit of a running container", runtime.GOOS)
 	}
 
+	if container.IsDead() {
+		err := fmt.Errorf("You cannot commit container %s which is Dead", container.ID)
+		return "", stateConflictError{err}
+	}
+
+	if container.IsRemovalInProgress() {
+		err := fmt.Errorf("You cannot commit container %s which is being removed", container.ID)
+		return "", stateConflictError{err}
+	}
+
 	if c.Pause && !container.IsPaused() {
 		daemon.containerPause(container)
 		defer daemon.containerUnpause(container)
@@ -234,19 +245,36 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
 	return id.String(), nil
 }
 
-func (daemon *Daemon) exportContainerRw(container *container.Container) (io.ReadCloser, error) {
-	if err := daemon.Mount(container); err != nil {
+func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io.ReadCloser, err error) {
+	rwlayer, err := daemon.stores[container.Platform].layerStore.GetRWLayer(container.ID)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer)
+		}
+	}()
+
+	// TODO: this mount call is not necessary as we assume that TarStream() should
+	// mount the layer if needed. But the Diff() function for windows requests that
+	// the layer should be mounted when calling it. So we reserve this mount call
+	// until windows driver can implement Diff() interface correctly.
+	_, err = rwlayer.Mount(container.GetMountLabel())
+	if err != nil {
 		return nil, err
 	}
 
-	archive, err := container.RWLayer.TarStream()
+	archive, err := rwlayer.TarStream()
 	if err != nil {
-		daemon.Unmount(container) // logging is already handled in the `Unmount` function
+		rwlayer.Unmount()
 		return nil, err
 	}
 	return ioutils.NewReadCloserWrapper(archive, func() error {
 			archive.Close()
-			return container.RWLayer.Unmount()
+			err = rwlayer.Unmount()
+			daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer)
+			return err
 		}),
 		nil
 }

components/engine/daemon/daemon.go

@@ -48,6 +48,7 @@ import (
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/truncindex"
 	"github.com/docker/docker/plugin"
+	pluginexec "github.com/docker/docker/plugin/executor/containerd"
 	refstore "github.com/docker/docker/reference"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
@@ -646,12 +647,16 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
 	}
 	registerMetricsPluginCallback(d.PluginStore, metricsSockPath)
 
+	createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) {
+		return pluginexec.New(containerdRemote, m)
+	}
+
 	// Plugin system initialization should happen before restore. Do not change order.
 	d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
 		Root:               filepath.Join(config.Root, "plugins"),
 		ExecRoot:           getPluginExecRoot(config.Root),
 		Store:              d.PluginStore,
-		Executor:           containerdRemote,
+		CreateExecutor:     createPluginExec,
 		RegistryService:    registryService,
 		LiveRestoreEnabled: config.LiveRestoreEnabled,
 		LogPluginEvent:     d.LogPluginEvent, // todo: make private
@@ -1039,7 +1044,7 @@ func prepareTempDir(rootDir string, rootIDs idtools.IDPair) (string, error) {
 					logrus.Warnf("failed to delete old tmp directory: %s", newName)
 				}
 			}()
-		} else {
+		} else if !os.IsNotExist(err) {
 			logrus.Warnf("failed to rename %s for background deletion: %s. Deleting synchronously", tmpDir, err)
 			if err := os.RemoveAll(tmpDir); err != nil {
 				logrus.Warnf("failed to delete old tmp directory: %s", tmpDir)

components/engine/daemon/daemon_unix.go

@@ -1297,7 +1297,39 @@ func rootFSToAPIType(rootfs *image.RootFS) types.RootFS {
 // setupDaemonProcess sets various settings for the daemon's process
 func setupDaemonProcess(config *config.Config) error {
 	// setup the daemons oom_score_adj
-	return setupOOMScoreAdj(config.OOMScoreAdjust)
+	if err := setupOOMScoreAdj(config.OOMScoreAdjust); err != nil {
+		return err
+	}
+	return setMayDetachMounts()
+}
+
+// This is used to allow removal of mountpoints that may be mounted in other
+// namespaces on RHEL based kernels starting from RHEL 7.4.
+// Without this setting, removals on these RHEL based kernels may fail with
+// "device or resource busy".
+// This setting is not available in upstream kernels as it is not configurable,
+// but has been in the upstream kernels since 3.15.
+func setMayDetachMounts() error {
+	f, err := os.OpenFile("/proc/sys/fs/may_detach_mounts", os.O_WRONLY, 0)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return errors.Wrap(err, "error opening may_detach_mounts kernel config file")
+	}
+	defer f.Close()
+
+	_, err = f.WriteString("1")
+	if os.IsPermission(err) {
+		// Setting may_detach_mounts does not work in an
+		// unprivileged container. Ignore the error, but log
+		// it if we appear not to be in that situation.
+		if !rsystem.RunningInUserNS() {
+			logrus.Debugf("Permission denied writing %q to /proc/sys/fs/may_detach_mounts", "1")
+		}
+		return nil
+	}
+	return err
 }
 
 func setupOOMScoreAdj(score int) error {

components/engine/daemon/daemon_windows.go

@@ -235,7 +235,7 @@ func checkSystem() error {
 
 	vmcompute := windows.NewLazySystemDLL("vmcompute.dll")
 	if vmcompute.Load() != nil {
-		return fmt.Errorf("Failed to load vmcompute.dll. Ensure that the Containers role is installed.")
+		return fmt.Errorf("failed to load vmcompute.dll, ensure that the Containers feature is installed")
 	}
 
 	return nil

components/engine/daemon/export.go

@@ -22,6 +22,16 @@ func (daemon *Daemon) ContainerExport(name string, out io.Writer) error {
 		return fmt.Errorf("the daemon on this platform does not support exporting Windows containers")
 	}
 
+	if container.IsDead() {
+		err := fmt.Errorf("You cannot export container %s which is Dead", container.ID)
+		return stateConflictError{err}
+	}
+
+	if container.IsRemovalInProgress() {
+		err := fmt.Errorf("You cannot export container %s which is being removed", container.ID)
+		return stateConflictError{err}
+	}
+
 	data, err := daemon.containerExport(container)
 	if err != nil {
 		return fmt.Errorf("Error exporting container %s: %v", name, err)
@@ -35,8 +45,19 @@ func (daemon *Daemon) ContainerExport(name string, out io.Writer) error {
 	return nil
 }
 
-func (daemon *Daemon) containerExport(container *container.Container) (io.ReadCloser, error) {
-	if err := daemon.Mount(container); err != nil {
+func (daemon *Daemon) containerExport(container *container.Container) (arch io.ReadCloser, err error) {
+	rwlayer, err := daemon.stores[container.Platform].layerStore.GetRWLayer(container.ID)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer)
+		}
+	}()
+
+	_, err = rwlayer.Mount(container.GetMountLabel())
+	if err != nil {
 		return nil, err
 	}
 
@@ -46,12 +67,13 @@ func (daemon *Daemon) containerExport(container *container.Container) (io.ReadCl
 		GIDMaps:     daemon.idMappings.GIDs(),
 	})
 	if err != nil {
-		daemon.Unmount(container)
+		rwlayer.Unmount()
 		return nil, err
 	}
-	arch := ioutils.NewReadCloserWrapper(archive, func() error {
+	arch = ioutils.NewReadCloserWrapper(archive, func() error {
 		err := archive.Close()
-		daemon.Unmount(container)
+		rwlayer.Unmount()
+		daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer)
 		return err
 	})
 	daemon.LogContainerEvent(container, "export")

components/engine/daemon/graphdriver/driver_solaris.go

@@ -81,17 +81,16 @@ func (c *defaultChecker) IsMounted(path string) bool {
 func Mounted(fsType FsMagic, mountPath string) (bool, error) {
 
 	cs := C.CString(filepath.Dir(mountPath))
+	defer C.free(unsafe.Pointer(cs))
 	buf := C.getstatfs(cs)
+	defer C.free(unsafe.Pointer(buf))
 
 	// on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ]
 	if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) ||
 		(buf.f_basetype[3] != 0) {
 		logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath)
-		C.free(unsafe.Pointer(buf))
 		return false, ErrPrerequisites
 	}
 
-	C.free(unsafe.Pointer(buf))
-	C.free(unsafe.Pointer(cs))
 	return true, nil
 }

components/engine/daemon/graphdriver/overlay2/mount.go

@@ -49,18 +49,19 @@ func mountFrom(dir, device, target, mType string, flags uintptr, label string) e
 	output := bytes.NewBuffer(nil)
 	cmd.Stdout = output
 	cmd.Stderr = output
-
 	if err := cmd.Start(); err != nil {
+		w.Close()
 		return fmt.Errorf("mountfrom error on re-exec cmd: %v", err)
 	}
 	//write the options to the pipe for the untar exec to read
 	if err := json.NewEncoder(w).Encode(options); err != nil {
+		w.Close()
 		return fmt.Errorf("mountfrom json encode to pipe failed: %v", err)
 	}
 	w.Close()
 
 	if err := cmd.Wait(); err != nil {
-		return fmt.Errorf("mountfrom re-exec error: %v: output: %s", err, output)
+		return fmt.Errorf("mountfrom re-exec error: %v: output: %v", err, output)
 	}
 	return nil
 }

components/engine/daemon/graphdriver/overlay2/overlay.go

@@ -515,7 +515,7 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get creates and mounts the required file system for the given id and returns the mount path.
-func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
+func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr error) {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
 	dir := d.dir(id)
@@ -538,9 +538,11 @@ func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 		return containerfs.NewLocalContainerFS(mergedDir), nil
 	}
 	defer func() {
-		if err != nil {
+		if retErr != nil {
 			if c := d.ctr.Decrement(mergedDir); c <= 0 {
-				unix.Unmount(mergedDir, 0)
+				if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil {
+					logrus.Errorf("error unmounting %v: %v", mergedDir, mntErr)
+				}
 			}
 		}
 	}()

components/engine/daemon/graphdriver/zfs/zfs_solaris.go

@@ -27,18 +27,17 @@ import (
 func checkRootdirFs(rootdir string) error {
 
 	cs := C.CString(filepath.Dir(rootdir))
+	defer C.free(unsafe.Pointer(cs))
 	buf := C.getstatfs(cs)
+	defer C.free(unsafe.Pointer(buf))
 
 	// on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ]
 	if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) ||
 		(buf.f_basetype[3] != 0) {
 		logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir)
-		C.free(unsafe.Pointer(buf))
 		return graphdriver.ErrPrerequisites
 	}
 
-	C.free(unsafe.Pointer(buf))
-	C.free(unsafe.Pointer(cs))
 	return nil
 }
 

components/engine/daemon/logger/jsonfilelog/jsonfilelog.go

@@ -12,9 +12,9 @@ import (
 	"sync"
 
 	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog"
 	"github.com/docker/docker/daemon/logger/loggerutils"
-	"github.com/docker/docker/pkg/jsonlog"
-	"github.com/docker/go-units"
+	units "github.com/docker/go-units"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -106,25 +106,19 @@ func writeMessageBuf(w io.Writer, m *logger.Message, extra json.RawMessage, buf
 		return err
 	}
 	logger.PutMessage(m)
-	if _, err := w.Write(buf.Bytes()); err != nil {
-		return errors.Wrap(err, "error writing log entry")
-	}
-	return nil
+	_, err := w.Write(buf.Bytes())
+	return errors.Wrap(err, "error writing log entry")
 }
 
 func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error {
-	timestamp, err := jsonlog.FastTimeMarshalJSON(msg.Timestamp)
-	if err != nil {
-		return err
-	}
 	logLine := msg.Line
 	if !msg.Partial {
 		logLine = append(msg.Line, '\n')
 	}
-	err = (&jsonlog.JSONLogs{
+	err := (&jsonlog.JSONLogs{
 		Log:      logLine,
 		Stream:   msg.Source,
-		Created:  timestamp,
+		Created:  msg.Timestamp,
 		RawAttrs: extra,
 	}).MarshalJSONBuf(buf)
 	if err != nil {

components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go

@@ -1,6 +1,7 @@
 package jsonfilelog
 
 import (
+	"bytes"
 	"encoding/json"
 	"io/ioutil"
 	"os"
@@ -11,7 +12,9 @@ import (
 	"time"
 
 	"github.com/docker/docker/daemon/logger"
-	"github.com/docker/docker/pkg/jsonlog"
+	"github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog"
+	"github.com/gotestyourself/gotestyourself/fs"
+	"github.com/stretchr/testify/require"
 )
 
 func TestJSONFileLogger(t *testing.T) {
@@ -54,36 +57,38 @@ func TestJSONFileLogger(t *testing.T) {
 	}
 }
 
-func BenchmarkJSONFileLogger(b *testing.B) {
-	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
-	tmp, err := ioutil.TempDir("", "docker-logger-")
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-	filename := filepath.Join(tmp, "container.log")
-	l, err := New(logger.Info{
-		ContainerID: cid,
-		LogPath:     filename,
-	})
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer l.Close()
+func BenchmarkJSONFileLoggerLog(b *testing.B) {
+	tmp := fs.NewDir(b, "bench-jsonfilelog")
+	defer tmp.Remove()
 
-	testLine := "Line that thinks that it is log line from docker\n"
-	msg := &logger.Message{Line: []byte(testLine), Source: "stderr", Timestamp: time.Now().UTC()}
-	jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON()
-	if err != nil {
-		b.Fatal(err)
+	jsonlogger, err := New(logger.Info{
+		ContainerID: "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657",
+		LogPath:     tmp.Join("container.log"),
+		Config: map[string]string{
+			"labels": "first,second",
+		},
+		ContainerLabels: map[string]string{
+			"first":  "label_value",
+			"second": "label_foo",
+		},
+	})
+	require.NoError(b, err)
+	defer jsonlogger.Close()
+
+	msg := &logger.Message{
+		Line:      []byte("Line that thinks that it is log line from docker\n"),
+		Source:    "stderr",
+		Timestamp: time.Now().UTC(),
 	}
-	b.SetBytes(int64(len(jsonlog)+1) * 30)
+
+	buf := bytes.NewBuffer(nil)
+	require.NoError(b, marshalMessage(msg, jsonlogger.(*JSONFileLogger).extra, buf))
+	b.SetBytes(int64(buf.Len()))
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		for j := 0; j < 30; j++ {
-			if err := l.Log(msg); err != nil {
-				b.Fatal(err)
-			}
+		if err := jsonlogger.Log(msg); err != nil {
+			b.Fatal(err)
 		}
 	}
 }
@@ -200,50 +205,3 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
 		t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected)
 	}
 }
-
-func BenchmarkJSONFileLoggerWithReader(b *testing.B) {
-	b.StopTimer()
-	b.ResetTimer()
-	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
-	dir, err := ioutil.TempDir("", "json-logger-bench")
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	l, err := New(logger.Info{
-		ContainerID: cid,
-		LogPath:     filepath.Join(dir, "container.log"),
-	})
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer l.Close()
-	msg := &logger.Message{Line: []byte("line"), Source: "src1"}
-	jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON()
-	if err != nil {
-		b.Fatal(err)
-	}
-	b.SetBytes(int64(len(jsonlog)+1) * 30)
-
-	b.StartTimer()
-
-	go func() {
-		for i := 0; i < b.N; i++ {
-			for j := 0; j < 30; j++ {
-				l.Log(msg)
-			}
-		}
-		l.Close()
-	}()
-
-	lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Follow: true})
-	watchClose := lw.WatchClose()
-	for {
-		select {
-		case <-lw.Msg:
-		case <-watchClose:
-			return
-		}
-	}
-}

components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go

@@ -0,0 +1,25 @@
+package jsonlog
+
+import (
+	"time"
+)
+
+// JSONLog is a log message, typically a single entry from a given log stream.
+type JSONLog struct {
+	// Log is the log message
+	Log string `json:"log,omitempty"`
+	// Stream is the log source
+	Stream string `json:"stream,omitempty"`
+	// Created is the created timestamp of log
+	Created time.Time `json:"time"`
+	// Attrs is the list of extra attributes provided by the user
+	Attrs map[string]string `json:"attrs,omitempty"`
+}
+
+// Reset all fields to their zero value.
+func (jl *JSONLog) Reset() {
+	jl.Log = ""
+	jl.Stream = ""
+	jl.Created = time.Time{}
+	jl.Attrs = make(map[string]string)
+}

components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go

@@ -3,23 +3,22 @@ package jsonlog
 import (
 	"bytes"
 	"encoding/json"
+	"time"
 	"unicode/utf8"
 )
 
-// JSONLogs is based on JSONLog.
-// It allows marshalling JSONLog from Log as []byte
-// and an already marshalled Created timestamp.
+// JSONLogs marshals encoded JSONLog objects
 type JSONLogs struct {
-	Log     []byte `json:"log,omitempty"`
-	Stream  string `json:"stream,omitempty"`
-	Created string `json:"time"`
+	Log     []byte    `json:"log,omitempty"`
+	Stream  string    `json:"stream,omitempty"`
+	Created time.Time `json:"time"`
 
 	// json-encoded bytes
 	RawAttrs json.RawMessage `json:"attrs,omitempty"`
 }
 
-// MarshalJSONBuf is based on the same method from JSONLog
-// It has been modified to take into account the necessary changes.
+// MarshalJSONBuf is an optimized JSON marshaller that avoids reflection
+// and unnecessary allocation.
 func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
 	var first = true
 
@@ -36,7 +35,7 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
 			buf.WriteString(`,`)
 		}
 		buf.WriteString(`"stream":`)
-		ffjsonWriteJSONString(buf, mj.Stream)
+		ffjsonWriteJSONBytesAsString(buf, []byte(mj.Stream))
 	}
 	if len(mj.RawAttrs) > 0 {
 		if first {
@@ -50,14 +49,18 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
 	if !first {
 		buf.WriteString(`,`)
 	}
+
+	created, err := fastTimeMarshalJSON(mj.Created)
+	if err != nil {
+		return err
+	}
+
 	buf.WriteString(`"time":`)
-	buf.WriteString(mj.Created)
+	buf.WriteString(created)
 	buf.WriteString(`}`)
 	return nil
 }
 
-// This is based on ffjsonWriteJSONBytesAsString. It has been changed
-// to accept a string passed as a slice of bytes.
 func ffjsonWriteJSONBytesAsString(buf *bytes.Buffer, s []byte) {
 	const hex = "0123456789abcdef"
 

components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go

@@ -0,0 +1,40 @@
+package jsonlog
+
+import (
+	"bytes"
+	"encoding/json"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestJSONLogsMarshalJSONBuf(t *testing.T) {
+	logs := map[*JSONLogs]string{
+		{Log: []byte(`"A log line with \\"`)}:                  `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":`,
+		{Log: []byte("A log line")}:                            `^{\"log\":\"A log line\",\"time\":`,
+		{Log: []byte("A log line with \r")}:                    `^{\"log\":\"A log line with \\r\",\"time\":`,
+		{Log: []byte("A log line with & < >")}:                 `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":`,
+		{Log: []byte("A log line with utf8 : 🚀 ψ ω β")}:        `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":`,
+		{Stream: "stdout"}:                                     `^{\"stream\":\"stdout\",\"time\":`,
+		{Stream: "stdout", Log: []byte("A log line")}:          `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":`,
+		{Created: time.Date(2017, 9, 1, 1, 1, 1, 1, time.UTC)}: `^{\"time\":"2017-09-01T01:01:01.000000001Z"}$`,
+
+		{}: `^{\"time\":"0001-01-01T00:00:00Z"}$`,
+		// These ones are a little weird
+		{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":`,
+		{Log: []byte{0xaF}}:            `^{\"log\":\"\\ufffd\",\"time\":`,
+		{Log: []byte{0x7F}}:            `^{\"log\":\"\x7f\",\"time\":`,
+		// with raw attributes
+		{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`,
+	}
+	for jsonLog, expression := range logs {
+		var buf bytes.Buffer
+		err := jsonLog.MarshalJSONBuf(&buf)
+		require.NoError(t, err)
+		assert.Regexp(t, regexp.MustCompile(expression), buf.String())
+		assert.NoError(t, json.Unmarshal(buf.Bytes(), &map[string]interface{}{}))
+	}
+}

components/engine/daemon/logger/jsonfilelog/jsonlog/time_marshalling.go

@@ -0,0 +1,20 @@
+package jsonlog
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+const jsonFormat = `"` + time.RFC3339Nano + `"`
+
+// fastTimeMarshalJSON avoids one of the extra allocations that
+// time.MarshalJSON is making.
+func fastTimeMarshalJSON(t time.Time) (string, error) {
+	if y := t.Year(); y < 0 || y >= 10000 {
+		// RFC 3339 is clear that years are 4 digits exactly.
+		// See golang.org/issue/4556#c15 for more discussion.
+		return "", errors.New("time.MarshalJSON: year outside of range [0,9999]")
+	}
+	return t.Format(jsonFormat), nil
+}

components/engine/daemon/logger/jsonfilelog/jsonlog/time_marshalling_test.go

@@ -0,0 +1,35 @@
+package jsonlog
+
+import (
+	"testing"
+	"time"
+
+	"github.com/docker/docker/internal/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFastTimeMarshalJSONWithInvalidYear(t *testing.T) {
+	aTime := time.Date(-1, 1, 1, 0, 0, 0, 0, time.Local)
+	_, err := fastTimeMarshalJSON(aTime)
+	testutil.ErrorContains(t, err, "year outside of range")
+
+	anotherTime := time.Date(10000, 1, 1, 0, 0, 0, 0, time.Local)
+	_, err = fastTimeMarshalJSON(anotherTime)
+	testutil.ErrorContains(t, err, "year outside of range")
+}
+
+func TestFastTimeMarshalJSON(t *testing.T) {
+	aTime := time.Date(2015, 5, 29, 11, 1, 2, 3, time.UTC)
+	json, err := fastTimeMarshalJSON(aTime)
+	require.NoError(t, err)
+	assert.Equal(t, "\"2015-05-29T11:01:02.000000003Z\"", json)
+
+	location, err := time.LoadLocation("Europe/Paris")
+	require.NoError(t, err)
+
+	aTime = time.Date(2015, 5, 29, 11, 1, 2, 3, location)
+	json, err = fastTimeMarshalJSON(aTime)
+	require.NoError(t, err)
+	assert.Equal(t, "\"2015-05-29T11:01:02.000000003+02:00\"", json)
+}

components/engine/daemon/logger/jsonfilelog/read.go

@@ -13,9 +13,9 @@ import (
 
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog"
 	"github.com/docker/docker/daemon/logger/jsonfilelog/multireader"
 	"github.com/docker/docker/pkg/filenotify"
-	"github.com/docker/docker/pkg/jsonlog"
 	"github.com/docker/docker/pkg/tailfile"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -147,9 +147,8 @@ func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since ti
 		rdr = bytes.NewBuffer(bytes.Join(ls, []byte("\n")))
 	}
 	dec := json.NewDecoder(rdr)
-	l := &jsonlog.JSONLog{}
 	for {
-		msg, err := decodeLogLine(dec, l)
+		msg, err := decodeLogLine(dec, &jsonlog.JSONLog{})
 		if err != nil {
 			if err != io.EOF {
 				logWatcher.Err <- err

components/engine/daemon/logger/jsonfilelog/read_test.go

@@ -0,0 +1,64 @@
+package jsonfilelog
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/daemon/logger"
+	"github.com/gotestyourself/gotestyourself/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkJSONFileLoggerReadLogs(b *testing.B) {
+	tmp := fs.NewDir(b, "bench-jsonfilelog")
+	defer tmp.Remove()
+
+	jsonlogger, err := New(logger.Info{
+		ContainerID: "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657",
+		LogPath:     tmp.Join("container.log"),
+		Config: map[string]string{
+			"labels": "first,second",
+		},
+		ContainerLabels: map[string]string{
+			"first":  "label_value",
+			"second": "label_foo",
+		},
+	})
+	require.NoError(b, err)
+	defer jsonlogger.Close()
+
+	msg := &logger.Message{
+		Line:      []byte("Line that thinks that it is log line from docker\n"),
+		Source:    "stderr",
+		Timestamp: time.Now().UTC(),
+	}
+
+	buf := bytes.NewBuffer(nil)
+	require.NoError(b, marshalMessage(msg, jsonlogger.(*JSONFileLogger).extra, buf))
+	b.SetBytes(int64(buf.Len()))
+
+	b.ResetTimer()
+
+	chError := make(chan error, b.N+1)
+	go func() {
+		for i := 0; i < b.N; i++ {
+			chError <- jsonlogger.Log(msg)
+		}
+		chError <- jsonlogger.Close()
+	}()
+
+	lw := jsonlogger.(*JSONFileLogger).ReadLogs(logger.ReadConfig{Follow: true})
+	watchClose := lw.WatchClose()
+	for {
+		select {
+		case <-lw.Msg:
+		case <-watchClose:
+			return
+		case err := <-chError:
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}

components/engine/daemon/logger/logger.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types/backend"
-	"github.com/docker/docker/pkg/jsonlog"
 )
 
 // ErrReadLogsNotSupported is returned when the underlying log driver does not support reading
@@ -26,8 +25,6 @@ func (ErrReadLogsNotSupported) Error() string {
 func (ErrReadLogsNotSupported) NotImplemented() {}
 
 const (
-	// TimeFormat is the time format used for timestamps sent to log readers.
-	TimeFormat           = jsonlog.RFC3339NanoFixed
 	logWatcherBufferSize = 4096
 )
 

components/engine/daemon/volumes.go

@@ -207,6 +207,9 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo
 			}); ok {
 				mp.Source = cv.CachedPath()
 			}
+			if mp.Driver == volume.DefaultDriverName {
+				setBindModeIfNull(mp)
+			}
 		}
 
 		binds[mp.Destination] = true

components/engine/docs/api/version-history.md

@@ -13,6 +13,13 @@ keywords: "API, Docker, rcli, REST, documentation"
      will be rejected.
 -->
 
+
+## v1.33 API changes
+
+[Docker Engine API v1.33](https://docs.docker.com/engine/api/v1.33/) documentation
+
+
+
 ## v1.32 API changes
 
 [Docker Engine API v1.32](https://docs.docker.com/engine/api/v1.32/) documentation

components/engine/image/image_test.go

@@ -2,6 +2,7 @@ package image
 
 import (
 	"encoding/json"
+	"runtime"
 	"sort"
 	"strings"
 	"testing"
@@ -54,6 +55,39 @@ func TestMarshalKeyOrder(t *testing.T) {
 	}
 }
 
+func TestImage(t *testing.T) {
+	cid := "50a16564e727"
+	config := &container.Config{
+		Hostname:   "hostname",
+		Domainname: "domain",
+		User:       "root",
+	}
+	platform := runtime.GOOS
+
+	img := &Image{
+		V1Image: V1Image{
+			Config: config,
+		},
+		computedID: ID(cid),
+	}
+
+	assert.Equal(t, cid, img.ImageID())
+	assert.Equal(t, cid, img.ID().String())
+	assert.Equal(t, platform, img.Platform())
+	assert.Equal(t, config, img.RunConfig())
+}
+
+func TestImagePlatformNotEmpty(t *testing.T) {
+	platform := "platform"
+	img := &Image{
+		V1Image: V1Image{
+			OS: platform,
+		},
+		OSVersion: "osversion",
+	}
+	assert.Equal(t, platform, img.Platform())
+}
+
 func TestNewChildImageFromImageWithRootFS(t *testing.T) {
 	rootFS := NewRootFS()
 	rootFS.Append(layer.DiffID("ba5e"))

components/engine/integration-cli/cli/build/fakestorage/fixtures.go

@@ -30,7 +30,7 @@ func ensureHTTPServerImage(t testingT) {
 	}
 	defer os.RemoveAll(tmp)
 
-	goos := testEnv.DaemonInfo.OSType
+	goos := testEnv.OSType
 	if goos == "" {
 		goos = "linux"
 	}

components/engine/integration-cli/cli/cli.go

@@ -115,7 +115,7 @@ func Docker(cmd icmd.Cmd, cmdOperators ...CmdOperator) *icmd.Result {
 // validateArgs is a checker to ensure tests are not running commands which are
 // not supported on platforms. Specifically on Windows this is 'busybox top'.
 func validateArgs(args ...string) error {
-	if testEnv.DaemonInfo.OSType != "windows" {
+	if testEnv.OSType != "windows" {
 		return nil
 	}
 	foundBusybox := -1

components/engine/integration-cli/docker_api_build_test.go

@@ -439,6 +439,82 @@ func (s *DockerSuite) TestBuildChownOnCopy(c *check.C) {
 	assert.Contains(c, string(out), "Successfully built")
 }
 
+func (s *DockerSuite) TestBuildCopyCacheOnFileChange(c *check.C) {
+
+	dockerfile := `FROM busybox
+COPY file /file`
+
+	ctx1 := fakecontext.New(c, "",
+		fakecontext.WithDockerfile(dockerfile),
+		fakecontext.WithFile("file", "foo"))
+	ctx2 := fakecontext.New(c, "",
+		fakecontext.WithDockerfile(dockerfile),
+		fakecontext.WithFile("file", "bar"))
+
+	var build = func(ctx *fakecontext.Fake) string {
+		res, body, err := request.Post("/build",
+			request.RawContent(ctx.AsTarReader(c)),
+			request.ContentType("application/x-tar"))
+
+		require.NoError(c, err)
+		assert.Equal(c, http.StatusOK, res.StatusCode)
+
+		out, err := request.ReadBody(body)
+
+		ids := getImageIDsFromBuild(c, out)
+		return ids[len(ids)-1]
+	}
+
+	id1 := build(ctx1)
+	id2 := build(ctx1)
+	id3 := build(ctx2)
+
+	if id1 != id2 {
+		c.Fatal("didn't use the cache")
+	}
+	if id1 == id3 {
+		c.Fatal("COPY With different source file should not share same cache")
+	}
+}
+
+func (s *DockerSuite) TestBuildAddCacheOnFileChange(c *check.C) {
+
+	dockerfile := `FROM busybox
+ADD file /file`
+
+	ctx1 := fakecontext.New(c, "",
+		fakecontext.WithDockerfile(dockerfile),
+		fakecontext.WithFile("file", "foo"))
+	ctx2 := fakecontext.New(c, "",
+		fakecontext.WithDockerfile(dockerfile),
+		fakecontext.WithFile("file", "bar"))
+
+	var build = func(ctx *fakecontext.Fake) string {
+		res, body, err := request.Post("/build",
+			request.RawContent(ctx.AsTarReader(c)),
+			request.ContentType("application/x-tar"))
+
+		require.NoError(c, err)
+		assert.Equal(c, http.StatusOK, res.StatusCode)
+
+		out, err := request.ReadBody(body)
+
+		ids := getImageIDsFromBuild(c, out)
+		return ids[len(ids)-1]
+	}
+
+	id1 := build(ctx1)
+	id2 := build(ctx1)
+	id3 := build(ctx2)
+
+	if id1 != id2 {
+		c.Fatal("didn't use the cache")
+	}
+	if id1 == id3 {
+		c.Fatal("COPY With different source file should not share same cache")
+	}
+}
+
 func (s *DockerSuite) TestBuildWithSession(c *check.C) {
 	testRequires(c, ExperimentalDaemon)
 
@@ -510,7 +586,9 @@ func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) {
 	sess, err := session.NewSession("foo1", "foo")
 	assert.Nil(c, err)
 
-	fsProvider := filesync.NewFSSyncProvider(dir, nil)
+	fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{
+		{Dir: dir},
+	})
 	sess.Allow(fsProvider)
 
 	g, ctx := errgroup.WithContext(context.Background())
@@ -520,7 +598,7 @@ func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) {
 	})
 
 	g.Go(func() error {
-		res, body, err := request.Post("/build?remote=client-session&session="+sess.UUID(), func(req *http.Request) error {
+		res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error {
 			req.Body = ioutil.NopCloser(strings.NewReader(dockerfile))
 			return nil
 		})

components/engine/integration-cli/docker_api_containers_test.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -1901,33 +1902,37 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) {
 	}
 
 	type testCase struct {
-		cfg      mounttypes.Mount
+		spec     mounttypes.Mount
 		expected types.MountPoint
 	}
 
+	var selinuxSharedLabel string
+	if runtime.GOOS == "linux" {
+		selinuxSharedLabel = "z"
+	}
+
 	cases := []testCase{
 		// use literal strings here for `Type` instead of the defined constants in the volume package to keep this honest
 		// Validation of the actual `Mount` struct is done in another test is not needed here
-		{mounttypes.Mount{Type: "volume", Target: destPath}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}},
-		{mounttypes.Mount{Type: "volume", Target: destPath + slash}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}},
-		{mounttypes.Mount{Type: "volume", Target: destPath, Source: "test1"}, types.MountPoint{Type: "volume", Name: "test1", RW: true, Destination: destPath}},
-		{mounttypes.Mount{Type: "volume", Target: destPath, ReadOnly: true, Source: "test2"}, types.MountPoint{Type: "volume", Name: "test2", RW: false, Destination: destPath}},
 		{
-			mounttypes.Mount{
-				Type:   "volume",
-				Target: destPath,
-				Source: "test3",
-				VolumeOptions: &mounttypes.VolumeOptions{
-					DriverConfig: &mounttypes.Driver{Name: volume.DefaultDriverName},
-				},
-			},
-			types.MountPoint{
-				Driver:      volume.DefaultDriverName,
-				Type:        "volume",
-				Name:        "test3",
-				RW:          true,
-				Destination: destPath,
-			},
+			spec:     mounttypes.Mount{Type: "volume", Target: destPath},
+			expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
+		},
+		{
+			spec:     mounttypes.Mount{Type: "volume", Target: destPath + slash},
+			expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
+		},
+		{
+			spec:     mounttypes.Mount{Type: "volume", Target: destPath, Source: "test1"},
+			expected: types.MountPoint{Type: "volume", Name: "test1", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
+		},
+		{
+			spec:     mounttypes.Mount{Type: "volume", Target: destPath, ReadOnly: true, Source: "test2"},
+			expected: types.MountPoint{Type: "volume", Name: "test2", RW: false, Destination: destPath, Mode: selinuxSharedLabel},
+		},
+		{
+			spec:     mounttypes.Mount{Type: "volume", Target: destPath, Source: "test3", VolumeOptions: &mounttypes.VolumeOptions{DriverConfig: &mounttypes.Driver{Name: volume.DefaultDriverName}}},
+			expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", Name: "test3", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
 		},
 	}
 
@@ -1938,19 +1943,22 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) {
 		defer os.RemoveAll(tmpDir1)
 		cases = append(cases, []testCase{
 			{
-				mounttypes.Mount{
+				spec: mounttypes.Mount{
 					Type:   "bind",
 					Source: tmpDir1,
 					Target: destPath,
 				},
-				types.MountPoint{
+				expected: types.MountPoint{
 					Type:        "bind",
 					RW:          true,
 					Destination: destPath,
 					Source:      tmpDir1,
 				},
 			},
-			{mounttypes.Mount{Type: "bind", Source: tmpDir1, Target: destPath, ReadOnly: true}, types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir1}},
+			{
+				spec:     mounttypes.Mount{Type: "bind", Source: tmpDir1, Target: destPath, ReadOnly: true},
+				expected: types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir1},
+			},
 		}...)
 
 		// for modes only supported on Linux
@@ -1963,19 +1971,40 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) {
 			c.Assert(mount.ForceMount("", tmpDir3, "none", "shared"), checker.IsNil)
 
 			cases = append(cases, []testCase{
-				{mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath}, types.MountPoint{Type: "bind", RW: true, Destination: destPath, Source: tmpDir3}},
-				{mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true}, types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3}},
-				{mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true, BindOptions: &mounttypes.BindOptions{Propagation: "shared"}}, types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3, Propagation: "shared"}},
+				{
+					spec:     mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath},
+					expected: types.MountPoint{Type: "bind", RW: true, Destination: destPath, Source: tmpDir3},
+				},
+				{
+					spec:     mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true},
+					expected: types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3},
+				},
+				{
+					spec:     mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true, BindOptions: &mounttypes.BindOptions{Propagation: "shared"}},
+					expected: types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3, Propagation: "shared"},
+				},
 			}...)
 		}
 	}
 
 	if testEnv.DaemonPlatform() != "windows" { // Windows does not support volume populate
 		cases = append(cases, []testCase{
-			{mounttypes.Mount{Type: "volume", Target: destPath, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}},
-			{mounttypes.Mount{Type: "volume", Target: destPath + slash, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}},
-			{mounttypes.Mount{Type: "volume", Target: destPath, Source: "test4", VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Type: "volume", Name: "test4", RW: true, Destination: destPath}},
-			{mounttypes.Mount{Type: "volume", Target: destPath, Source: "test5", ReadOnly: true, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Type: "volume", Name: "test5", RW: false, Destination: destPath}},
+			{
+				spec:     mounttypes.Mount{Type: "volume", Target: destPath, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}},
+				expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
+			},
+			{
+				spec:     mounttypes.Mount{Type: "volume", Target: destPath + slash, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}},
+				expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
+			},
+			{
+				spec:     mounttypes.Mount{Type: "volume", Target: destPath, Source: "test4", VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}},
+				expected: types.MountPoint{Type: "volume", Name: "test4", RW: true, Destination: destPath, Mode: selinuxSharedLabel},
+			},
+			{
+				spec:     mounttypes.Mount{Type: "volume", Target: destPath, Source: "test5", ReadOnly: true, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}},
+				expected: types.MountPoint{Type: "volume", Name: "test5", RW: false, Destination: destPath, Mode: selinuxSharedLabel},
+			},
 		}...)
 	}
 
@@ -1990,11 +2019,11 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) {
 	ctx := context.Background()
 	apiclient := testEnv.APIClient()
 	for i, x := range cases {
-		c.Logf("case %d - config: %v", i, x.cfg)
+		c.Logf("case %d - config: %v", i, x.spec)
 		container, err := apiclient.ContainerCreate(
 			ctx,
 			&containertypes.Config{Image: testImg},
-			&containertypes.HostConfig{Mounts: []mounttypes.Mount{x.cfg}},
+			&containertypes.HostConfig{Mounts: []mounttypes.Mount{x.spec}},
 			&networktypes.NetworkingConfig{},
 			"")
 		require.NoError(c, err)
@@ -2035,12 +2064,12 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) {
 		switch {
 
 		// Named volumes still exist after the container is removed
-		case x.cfg.Type == "volume" && len(x.cfg.Source) > 0:
+		case x.spec.Type == "volume" && len(x.spec.Source) > 0:
 			_, err := apiclient.VolumeInspect(ctx, mountPoint.Name)
 			require.NoError(c, err)
 
 		// Bind mounts are never removed with the container
-		case x.cfg.Type == "bind":
+		case x.spec.Type == "bind":
 
 		// anonymous volumes are removed
 		default:

components/engine/integration-cli/docker_cli_build_test.go

@@ -1173,12 +1173,13 @@ func (s *DockerSuite) TestBuildForceRm(c *check.C) {
 	containerCountBefore := getContainerCount(c)
 	name := "testbuildforcerm"
 
-	buildImage(name, cli.WithFlags("--force-rm"), build.WithBuildContext(c,
-		build.WithFile("Dockerfile", `FROM `+minimalBaseImage()+`
+	r := buildImage(name, cli.WithFlags("--force-rm"), build.WithBuildContext(c,
+		build.WithFile("Dockerfile", `FROM busybox
 	RUN true
-	RUN thiswillfail`))).Assert(c, icmd.Expected{
-		ExitCode: 1,
-	})
+	RUN thiswillfail`)))
+	if r.ExitCode != 1 && r.ExitCode != 127 { // different on Linux / Windows
+		c.Fatalf("Wrong exit code")
+	}
 
 	containerCountAfter := getContainerCount(c)
 	if containerCountBefore != containerCountAfter {
@@ -4542,7 +4543,6 @@ func (s *DockerSuite) TestBuildBuildTimeArgOverrideEnvDefinedBeforeArg(c *check.
 }
 
 func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) {
-	testRequires(c, DaemonIsLinux) // Windows does not support ARG
 	imgName := "bldvarstest"
 
 	wdVar := "WDIR"
@@ -4559,6 +4559,10 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) {
 	userVal := "testUser"
 	volVar := "VOL"
 	volVal := "/testVol/"
+	if DaemonIsWindows() {
+		volVal = "C:\\testVol"
+		wdVal = "C:\\tmp"
+	}
 
 	buildImageSuccessfully(c, imgName,
 		cli.WithFlags(
@@ -4594,7 +4598,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) {
 	)
 
 	res := inspectField(c, imgName, "Config.WorkingDir")
-	c.Check(res, check.Equals, filepath.ToSlash(wdVal))
+	c.Check(filepath.ToSlash(res), check.Equals, filepath.ToSlash(wdVal))
 
 	var resArr []string
 	inspectFieldAndUnmarshall(c, imgName, "Config.Env", &resArr)

components/engine/integration-cli/docker_cli_logs_test.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/cli"
-	"github.com/docker/docker/pkg/jsonlog"
+	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/go-check/check"
 	"github.com/gotestyourself/gotestyourself/icmd"
 )
@@ -55,7 +55,7 @@ func (s *DockerSuite) TestLogsTimestamps(c *check.C) {
 
 	for _, l := range lines {
 		if l != "" {
-			_, err := time.Parse(jsonlog.RFC3339NanoFixed+" ", ts.FindString(l))
+			_, err := time.Parse(jsonmessage.RFC3339NanoFixed+" ", ts.FindString(l))
 			c.Assert(err, checker.IsNil, check.Commentf("Failed to parse timestamp from %v", l))
 			// ensure we have padded 0's
 			c.Assert(l[29], checker.Equals, uint8('Z'))

components/engine/integration-cli/environment/environment.go

@@ -67,9 +67,9 @@ func (e *Execution) ExperimentalDaemon() bool {
 // decisions on how to configure themselves according to the platform
 // of the daemon. This is initialized in docker_utils by sending
 // a version call to the daemon and examining the response header.
-// Deprecated: use Execution.DaemonInfo.OSType
+// Deprecated: use Execution.OSType
 func (e *Execution) DaemonPlatform() string {
-	return e.DaemonInfo.OSType
+	return e.OSType
 }
 
 // MinimalBaseImage is the image used for minimal builds (it depends on the platform)

components/engine/integration-cli/fixtures/plugin/plugin.go

@@ -1,9 +1,19 @@
 package plugin
 
 import (
+	"encoding/json"
 	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/plugin"
+	"github.com/docker/docker/registry"
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -32,3 +42,142 @@ func WithBinary(bin string) CreateOpt {
 type CreateClient interface {
 	PluginCreate(context.Context, io.Reader, types.PluginCreateOptions) error
 }
+
+// Create creates a new plugin with the specified name
+func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error {
+	tmpDir, err := ioutil.TempDir("", "create-test-plugin")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	tar, err := makePluginBundle(tmpDir, opts...)
+	if err != nil {
+		return err
+	}
+	defer tar.Close()
+
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	return c.PluginCreate(ctx, tar, types.PluginCreateOptions{RepoName: name})
+}
+
+// CreateInRegistry makes a plugin (locally) and pushes it to a registry.
+// This does not use a dockerd instance to create or push the plugin.
+// If you just want to create a plugin in some daemon, use `Create`.
+//
+// This can be useful when testing plugins on swarm where you don't really want
+// the plugin to exist on any of the daemons (immediately) and there needs to be
+// some way to distribute the plugin.
+func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error {
+	tmpDir, err := ioutil.TempDir("", "create-test-plugin-local")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	inPath := filepath.Join(tmpDir, "plugin")
+	if err := os.MkdirAll(inPath, 0755); err != nil {
+		return errors.Wrap(err, "error creating plugin root")
+	}
+
+	tar, err := makePluginBundle(inPath, opts...)
+	if err != nil {
+		return err
+	}
+	defer tar.Close()
+
+	dummyExec := func(m *plugin.Manager) (plugin.Executor, error) {
+		return nil, nil
+	}
+
+	regService, err := registry.NewService(registry.ServiceOptions{V2Only: true})
+	if err != nil {
+		return err
+	}
+
+	managerConfig := plugin.ManagerConfig{
+		Store:           plugin.NewStore(),
+		RegistryService: regService,
+		Root:            filepath.Join(tmpDir, "root"),
+		ExecRoot:        "/run/docker", // manager init fails if not set
+		CreateExecutor:  dummyExec,
+		LogPluginEvent:  func(id, name, action string) {}, // panics when not set
+	}
+	manager, err := plugin.NewManager(managerConfig)
+	if err != nil {
+		return errors.Wrap(err, "error creating plugin manager")
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	if err := manager.CreateFromContext(ctx, tar, &types.PluginCreateOptions{RepoName: repo}); err != nil {
+		return err
+	}
+
+	if auth == nil {
+		auth = &types.AuthConfig{}
+	}
+	err = manager.Push(ctx, repo, nil, auth, ioutil.Discard)
+	return errors.Wrap(err, "error pushing plugin")
+}
+
+func makePluginBundle(inPath string, opts ...CreateOpt) (io.ReadCloser, error) {
+	p := &types.PluginConfig{
+		Interface: types.PluginConfigInterface{
+			Socket: "basic.sock",
+			Types:  []types.PluginInterfaceType{{Capability: "docker.dummy/1.0"}},
+		},
+		Entrypoint: []string{"/basic"},
+	}
+	cfg := &Config{
+		PluginConfig: p,
+	}
+	for _, o := range opts {
+		o(cfg)
+	}
+	if cfg.binPath == "" {
+		binPath, err := ensureBasicPluginBin()
+		if err != nil {
+			return nil, err
+		}
+		cfg.binPath = binPath
+	}
+
+	configJSON, err := json.Marshal(p)
+	if err != nil {
+		return nil, err
+	}
+	if err := ioutil.WriteFile(filepath.Join(inPath, "config.json"), configJSON, 0644); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(filepath.Join(inPath, "rootfs", filepath.Dir(p.Entrypoint[0])), 0755); err != nil {
+		return nil, errors.Wrap(err, "error creating plugin rootfs dir")
+	}
+	if err := archive.NewDefaultArchiver().CopyFileWithTar(cfg.binPath, filepath.Join(inPath, "rootfs", p.Entrypoint[0])); err != nil {
+		return nil, errors.Wrap(err, "error copying plugin binary to rootfs path")
+	}
+	tar, err := archive.Tar(inPath, archive.Uncompressed)
+	return tar, errors.Wrap(err, "error making plugin archive")
+}
+
+func ensureBasicPluginBin() (string, error) {
+	name := "docker-basic-plugin"
+	p, err := exec.LookPath(name)
+	if err == nil {
+		return p, nil
+	}
+
+	goBin, err := exec.LookPath("go")
+	if err != nil {
+		return "", err
+	}
+	installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name)
+	cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("fixtures", "plugin", "basic"))
+	cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return "", errors.Wrapf(err, "error building basic plugin bin: %s", string(out))
+	}
+	return installPath, nil
+}

components/engine/integration-cli/fixtures/plugin/plugin_linux.go

@@ -1,162 +0,0 @@
-package plugin
-
-import (
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"time"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/libcontainerd"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/plugin"
-	"github.com/docker/docker/registry"
-	"github.com/pkg/errors"
-	"golang.org/x/net/context"
-)
-
-// Create creates a new plugin with the specified name
-func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error {
-	tmpDir, err := ioutil.TempDir("", "create-test-plugin")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tar, err := makePluginBundle(tmpDir, opts...)
-	if err != nil {
-		return err
-	}
-	defer tar.Close()
-
-	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
-	defer cancel()
-
-	return c.PluginCreate(ctx, tar, types.PluginCreateOptions{RepoName: name})
-}
-
-// TODO(@cpuguy83): we really shouldn't have to do this...
-// The manager panics on init when `Executor` is not set.
-type dummyExecutor struct{}
-
-func (dummyExecutor) Client(libcontainerd.Backend) (libcontainerd.Client, error) { return nil, nil }
-func (dummyExecutor) Cleanup()                                                   {}
-func (dummyExecutor) UpdateOptions(...libcontainerd.RemoteOption) error          { return nil }
-
-// CreateInRegistry makes a plugin (locally) and pushes it to a registry.
-// This does not use a dockerd instance to create or push the plugin.
-// If you just want to create a plugin in some daemon, use `Create`.
-//
-// This can be useful when testing plugins on swarm where you don't really want
-// the plugin to exist on any of the daemons (immediately) and there needs to be
-// some way to distribute the plugin.
-func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error {
-	tmpDir, err := ioutil.TempDir("", "create-test-plugin-local")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpDir)
-
-	inPath := filepath.Join(tmpDir, "plugin")
-	if err := os.MkdirAll(inPath, 0755); err != nil {
-		return errors.Wrap(err, "error creating plugin root")
-	}
-
-	tar, err := makePluginBundle(inPath, opts...)
-	if err != nil {
-		return err
-	}
-	defer tar.Close()
-
-	regService, err := registry.NewService(registry.ServiceOptions{V2Only: true})
-	if err != nil {
-		return err
-	}
-
-	managerConfig := plugin.ManagerConfig{
-		Store:           plugin.NewStore(),
-		RegistryService: regService,
-		Root:            filepath.Join(tmpDir, "root"),
-		ExecRoot:        "/run/docker", // manager init fails if not set
-		Executor:        dummyExecutor{},
-		LogPluginEvent:  func(id, name, action string) {}, // panics when not set
-	}
-	manager, err := plugin.NewManager(managerConfig)
-	if err != nil {
-		return errors.Wrap(err, "error creating plugin manager")
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
-	defer cancel()
-	if err := manager.CreateFromContext(ctx, tar, &types.PluginCreateOptions{RepoName: repo}); err != nil {
-		return err
-	}
-
-	if auth == nil {
-		auth = &types.AuthConfig{}
-	}
-	err = manager.Push(ctx, repo, nil, auth, ioutil.Discard)
-	return errors.Wrap(err, "error pushing plugin")
-}
-
-func makePluginBundle(inPath string, opts ...CreateOpt) (io.ReadCloser, error) {
-	p := &types.PluginConfig{
-		Interface: types.PluginConfigInterface{
-			Socket: "basic.sock",
-			Types:  []types.PluginInterfaceType{{Capability: "docker.dummy/1.0"}},
-		},
-		Entrypoint: []string{"/basic"},
-	}
-	cfg := &Config{
-		PluginConfig: p,
-	}
-	for _, o := range opts {
-		o(cfg)
-	}
-	if cfg.binPath == "" {
-		binPath, err := ensureBasicPluginBin()
-		if err != nil {
-			return nil, err
-		}
-		cfg.binPath = binPath
-	}
-
-	configJSON, err := json.Marshal(p)
-	if err != nil {
-		return nil, err
-	}
-	if err := ioutil.WriteFile(filepath.Join(inPath, "config.json"), configJSON, 0644); err != nil {
-		return nil, err
-	}
-	if err := os.MkdirAll(filepath.Join(inPath, "rootfs", filepath.Dir(p.Entrypoint[0])), 0755); err != nil {
-		return nil, errors.Wrap(err, "error creating plugin rootfs dir")
-	}
-	if err := archive.NewDefaultArchiver().CopyFileWithTar(cfg.binPath, filepath.Join(inPath, "rootfs", p.Entrypoint[0])); err != nil {
-		return nil, errors.Wrap(err, "error copying plugin binary to rootfs path")
-	}
-	tar, err := archive.Tar(inPath, archive.Uncompressed)
-	return tar, errors.Wrap(err, "error making plugin archive")
-}
-
-func ensureBasicPluginBin() (string, error) {
-	name := "docker-basic-plugin"
-	p, err := exec.LookPath(name)
-	if err == nil {
-		return p, nil
-	}
-
-	goBin, err := exec.LookPath("go")
-	if err != nil {
-		return "", err
-	}
-	installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name)
-	cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("fixtures", "plugin", "basic"))
-	cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
-	if out, err := cmd.CombinedOutput(); err != nil {
-		return "", errors.Wrapf(err, "error building basic plugin bin: %s", string(out))
-	}
-	return installPath, nil
-}

components/engine/integration-cli/fixtures/plugin/plugin_unsuported.go

@@ -1,19 +0,0 @@
-// +build !linux
-
-package plugin
-
-import (
-	"github.com/docker/docker/api/types"
-	"github.com/pkg/errors"
-	"golang.org/x/net/context"
-)
-
-// Create is not supported on this platform
-func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error {
-	return errors.New("not supported on this platform")
-}
-
-// CreateInRegistry is not supported on this platform
-func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error {
-	return errors.New("not supported on this platform")
-}

components/engine/integration-cli/requirements_test.go

@@ -21,12 +21,12 @@ func ArchitectureIsNot(arch string) bool {
 }
 
 func DaemonIsWindows() bool {
-	return testEnv.DaemonInfo.OSType == "windows"
+	return testEnv.OSType == "windows"
 }
 
 func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool {
 	return func() bool {
-		if testEnv.DaemonInfo.OSType != "windows" {
+		if testEnv.OSType != "windows" {
 			return false
 		}
 		version := testEnv.DaemonInfo.KernelVersion
@@ -36,7 +36,7 @@ func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool {
 }
 
 func DaemonIsLinux() bool {
-	return testEnv.DaemonInfo.OSType == "linux"
+	return testEnv.OSType == "linux"
 }
 
 func OnlyDefaultNetworks() bool {
@@ -178,21 +178,21 @@ func UserNamespaceInKernel() bool {
 }
 
 func IsPausable() bool {
-	if testEnv.DaemonInfo.OSType == "windows" {
+	if testEnv.OSType == "windows" {
 		return testEnv.DaemonInfo.Isolation == "hyperv"
 	}
 	return true
 }
 
 func NotPausable() bool {
-	if testEnv.DaemonInfo.OSType == "windows" {
+	if testEnv.OSType == "windows" {
 		return testEnv.DaemonInfo.Isolation == "process"
 	}
 	return false
 }
 
 func IsolationIs(expectedIsolation string) bool {
-	return testEnv.DaemonInfo.OSType == "windows" && string(testEnv.DaemonInfo.Isolation) == expectedIsolation
+	return testEnv.OSType == "windows" && string(testEnv.DaemonInfo.Isolation) == expectedIsolation
 }
 
 func IsolationIsHyperv() bool {

components/engine/integration/container/rename_test.go

@@ -0,0 +1,88 @@
+package container
+
+import (
+	"context"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/client"
+	"github.com/docker/docker/integration/util/request"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func runContainer(ctx context.Context, t *testing.T, client client.APIClient, cntCfg *container.Config, hstCfg *container.HostConfig, nwkCfg *network.NetworkingConfig, cntName string) string {
+	cnt, err := client.ContainerCreate(ctx, cntCfg, hstCfg, nwkCfg, cntName)
+	require.NoError(t, err)
+
+	err = client.ContainerStart(ctx, cnt.ID, types.ContainerStartOptions{})
+	require.NoError(t, err)
+	return cnt.ID
+}
+
+// This test simulates the scenario mentioned in #31392:
+// Having two linked container, renaming the target and bringing a replacement
+// and then deleting and recreating the source container linked to the new target.
+// This checks that "rename" updates source container correctly and doesn't set it to null.
+func TestRenameLinkedContainer(t *testing.T) {
+	defer setupTest(t)()
+	ctx := context.Background()
+	client := request.NewAPIClient(t)
+
+	cntConfig := &container.Config{
+		Image: "busybox",
+		Tty:   true,
+		Cmd:   strslice.StrSlice([]string{"top"}),
+	}
+
+	var (
+		aID, bID string
+		cntJSON  types.ContainerJSON
+		err      error
+	)
+
+	aID = runContainer(ctx, t, client,
+		cntConfig,
+		&container.HostConfig{},
+		&network.NetworkingConfig{},
+		"a0",
+	)
+
+	bID = runContainer(ctx, t, client,
+		cntConfig,
+		&container.HostConfig{
+			Links: []string{"a0"},
+		},
+		&network.NetworkingConfig{},
+		"b0",
+	)
+
+	err = client.ContainerRename(ctx, aID, "a1")
+	require.NoError(t, err)
+
+	runContainer(ctx, t, client,
+		cntConfig,
+		&container.HostConfig{},
+		&network.NetworkingConfig{},
+		"a0",
+	)
+
+	err = client.ContainerRemove(ctx, bID, types.ContainerRemoveOptions{Force: true})
+	require.NoError(t, err)
+
+	bID = runContainer(ctx, t, client,
+		cntConfig,
+		&container.HostConfig{
+			Links: []string{"a0"},
+		},
+		&network.NetworkingConfig{},
+		"b0",
+	)
+
+	cntJSON, err = client.ContainerInspect(ctx, bID)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"/a0:/b0/a0"}, cntJSON.HostConfig.Links)
+}

components/engine/integration/system/main_test.go

@@ -23,6 +23,6 @@ func TestMain(m *testing.M) {
 }
 
 func setupTest(t *testing.T) func() {
-	environment.ProtectImages(t, testEnv)
+	environment.ProtectAll(t, testEnv)
 	return func() { testEnv.Clean(t) }
 }

components/engine/integration/system/version_test.go

@@ -3,15 +3,14 @@ package system
 import (
 	"testing"
 
-	"github.com/docker/docker/integration-cli/request"
+	"github.com/docker/docker/integration/util/request"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
 )
 
 func TestVersion(t *testing.T) {
-	client, err := request.NewClient()
-	require.NoError(t, err)
+	client := request.NewAPIClient(t)
 
 	version, err := client.ServerVersion(context.Background())
 	require.NoError(t, err)
@@ -20,5 +19,5 @@ func TestVersion(t *testing.T) {
 	assert.NotNil(t, version.Version)
 	assert.NotNil(t, version.MinAPIVersion)
 	assert.Equal(t, testEnv.DaemonInfo.ExperimentalBuild, version.Experimental)
-	assert.Equal(t, testEnv.DaemonInfo.OSType, version.Os)
+	assert.Equal(t, testEnv.OSType, version.Os)
 }

components/engine/internal/test/environment/clean.go

@@ -28,7 +28,7 @@ type logT interface {
 func (e *Execution) Clean(t testingT) {
 	client := e.APIClient()
 
-	platform := e.DaemonInfo.OSType
+	platform := e.OSType
 	if (platform != "windows") || (platform == "windows" && e.DaemonInfo.Isolation == "hyperv") {
 		unpauseAllContainers(t, client)
 	}
@@ -81,7 +81,7 @@ func deleteAllContainers(t assert.TestingT, apiclient client.ContainerAPIClient,
 			Force:         true,
 			RemoveVolumes: true,
 		})
-		if err == nil || client.IsErrNotFound(err) || alreadyExists.MatchString(err.Error()) {
+		if err == nil || client.IsErrNotFound(err) || alreadyExists.MatchString(err.Error()) || isErrNotFoundSwarmClassic(err) {
 			continue
 		}
 		assert.NoError(t, err, "failed to remove %s", container.ID)
@@ -138,6 +138,10 @@ func deleteAllVolumes(t assert.TestingT, c client.VolumeAPIClient, protectedVolu
 			continue
 		}
 		err := c.VolumeRemove(context.Background(), v.Name, true)
+		// Docker EE may list volumes that no longer exist.
+		if isErrNotFoundSwarmClassic(err) {
+			continue
+		}
 		assert.NoError(t, err, "failed to remove volume %s", v.Name)
 	}
 }
@@ -164,6 +168,10 @@ func deleteAllNetworks(t assert.TestingT, c client.NetworkAPIClient, daemonPlatf
 
 func deleteAllPlugins(t assert.TestingT, c client.PluginAPIClient, protectedPlugins map[string]struct{}) {
 	plugins, err := c.PluginList(context.Background(), filters.Args{})
+	// Docker EE does not allow cluster-wide plugin management.
+	if client.IsErrNotImplemented(err) {
+		return
+	}
 	assert.NoError(t, err, "failed to list plugins")
 
 	for _, p := range plugins {
@@ -174,3 +182,9 @@ func deleteAllPlugins(t assert.TestingT, c client.PluginAPIClient, protectedPlug
 		assert.NoError(t, err, "failed to remove plugin %s", p.ID)
 	}
 }
+
+// Swarm classic aggregates node errors and returns a 500 so we need to check
+// the error string instead of just IsErrNotFound().
+func isErrNotFoundSwarmClassic(err error) bool {
+	return err != nil && strings.Contains(strings.ToLower(err.Error()), "no such")
+}

components/engine/internal/test/environment/environment.go

@@ -17,6 +17,7 @@ import (
 type Execution struct {
 	client            client.APIClient
 	DaemonInfo        types.Info
+	OSType            string
 	PlatformDefaults  PlatformDefaults
 	protectedElements protectedElements
 }
@@ -40,19 +41,31 @@ func New() (*Execution, error) {
 		return nil, errors.Wrapf(err, "failed to get info from daemon")
 	}
 
+	osType := getOSType(info)
+
 	return &Execution{
 		client:            client,
 		DaemonInfo:        info,
-		PlatformDefaults:  getPlatformDefaults(info),
+		OSType:            osType,
+		PlatformDefaults:  getPlatformDefaults(info, osType),
 		protectedElements: newProtectedElements(),
 	}, nil
 }
 
-func getPlatformDefaults(info types.Info) PlatformDefaults {
+func getOSType(info types.Info) string {
+	// Docker EE does not set the OSType so allow the user to override this value.
+	userOsType := os.Getenv("TEST_OSTYPE")
+	if userOsType != "" {
+		return userOsType
+	}
+	return info.OSType
+}
+
+func getPlatformDefaults(info types.Info, osType string) PlatformDefaults {
 	volumesPath := filepath.Join(info.DockerRootDir, "volumes")
 	containersPath := filepath.Join(info.DockerRootDir, "containers")
 
-	switch info.OSType {
+	switch osType {
 	case "linux":
 		return PlatformDefaults{
 			BaseImage:            "scratch",
@@ -71,7 +84,7 @@ func getPlatformDefaults(info types.Info) PlatformDefaults {
 			ContainerStoragePath: filepath.FromSlash(containersPath),
 		}
 	default:
-		panic(fmt.Sprintf("unknown info.OSType for daemon: %s", info.OSType))
+		panic(fmt.Sprintf("unknown OSType for daemon: %s", osType))
 	}
 }
 

components/engine/internal/test/environment/protect.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	dclient "github.com/docker/docker/client"
 	"github.com/docker/docker/integration-cli/fixtures/load"
 	"github.com/stretchr/testify/require"
 )
@@ -35,7 +36,7 @@ func ProtectAll(t testingT, testEnv *Execution) {
 	ProtectImages(t, testEnv)
 	ProtectNetworks(t, testEnv)
 	ProtectVolumes(t, testEnv)
-	if testEnv.DaemonInfo.OSType == "linux" {
+	if testEnv.OSType == "linux" {
 		ProtectPlugins(t, testEnv)
 	}
 }
@@ -81,7 +82,7 @@ func (e *Execution) ProtectImage(t testingT, images ...string) {
 func ProtectImages(t testingT, testEnv *Execution) {
 	images := getExistingImages(t, testEnv)
 
-	if testEnv.DaemonInfo.OSType == "linux" {
+	if testEnv.OSType == "linux" {
 		images = append(images, ensureFrozenImagesLinux(t, testEnv)...)
 	}
 	testEnv.ProtectImage(t, images...)
@@ -172,6 +173,10 @@ func ProtectPlugins(t testingT, testEnv *Execution) {
 func getExistingPlugins(t require.TestingT, testEnv *Execution) []string {
 	client := testEnv.APIClient()
 	pluginList, err := client.PluginList(context.Background(), filters.Args{})
+	// Docker EE does not allow cluster-wide plugin management.
+	if dclient.IsErrNotImplemented(err) {
+		return []string{}
+	}
 	require.NoError(t, err, "failed to list plugins")
 
 	plugins := []string{}

components/engine/libcontainerd/client_windows.go

@@ -530,6 +530,9 @@ func (clnt *client) AddProcess(ctx context.Context, containerID, processFriendly
 	if err != nil {
 		return -1, err
 	}
+
+	defer container.debugGCS()
+
 	// Note we always tell HCS to
 	// create stdout as it's required regardless of '-i' or '-t' options, so that
 	// docker can always grab the output through logs. We also tell HCS to always

components/engine/libcontainerd/container_windows.go

@@ -50,6 +50,7 @@ func (ctr *container) start(attachStdio StdioCallback) error {
 	logrus.Debugln("libcontainerd: starting container ", ctr.containerID)
 	if err = ctr.hcsContainer.Start(); err != nil {
 		logrus.Errorf("libcontainerd: failed to start container: %s", err)
+		ctr.debugGCS() // Before terminating!
 		if err := ctr.terminate(); err != nil {
 			logrus.Errorf("libcontainerd: failed to cleanup after a failed Start. %s", err)
 		} else {
@@ -58,6 +59,8 @@ func (ctr *container) start(attachStdio StdioCallback) error {
 		return err
 	}
 
+	defer ctr.debugGCS()
+
 	// Note we always tell HCS to
 	// create stdout as it's required regardless of '-i' or '-t' options, so that
 	// docker can always grab the output through logs. We also tell HCS to always

components/engine/libcontainerd/utils_windows.go

@@ -1,6 +1,10 @@
 package libcontainerd
 
-import "strings"
+import (
+	"strings"
+
+	opengcs "github.com/Microsoft/opengcs/client"
+)
 
 // setupEnvironmentVariables converts a string array of environment variables
 // into a map as required by the HCS. Source array is in format [v1=k1] [v2=k2] etc.
@@ -19,3 +23,16 @@ func setupEnvironmentVariables(a []string) map[string]string {
 func (s *LCOWOption) Apply(interface{}) error {
 	return nil
 }
+
+// debugGCS is a dirty hack for debugging for Linux Utility VMs. It simply
+// runs a bunch of commands inside the UVM, but seriously aides in advanced debugging.
+func (c *container) debugGCS() {
+	if c == nil || c.isWindows || c.hcsContainer == nil {
+		return
+	}
+	cfg := opengcs.Config{
+		Uvm:               c.hcsContainer,
+		UvmTimeoutSeconds: 600,
+	}
+	cfg.DebugGCS()
+}

components/engine/pkg/archive/archive.go

@@ -20,7 +20,6 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/pools"
-	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/pkg/system"
 	"github.com/sirupsen/logrus"
 )
@@ -1095,36 +1094,42 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 	}
 
 	r, w := io.Pipe()
-	errC := promise.Go(func() error {
-		defer w.Close()
+	errC := make(chan error, 1)
 
-		srcF, err := os.Open(src)
-		if err != nil {
-			return err
-		}
-		defer srcF.Close()
+	go func() {
+		defer close(errC)
 
-		hdr, err := tar.FileInfoHeader(srcSt, "")
-		if err != nil {
-			return err
-		}
-		hdr.Name = filepath.Base(dst)
-		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
+		errC <- func() error {
+			defer w.Close()
 
-		if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
-			return err
-		}
+			srcF, err := os.Open(src)
+			if err != nil {
+				return err
+			}
+			defer srcF.Close()
 
-		tw := tar.NewWriter(w)
-		defer tw.Close()
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if _, err := io.Copy(tw, srcF); err != nil {
-			return err
-		}
-		return nil
-	})
+			hdr, err := tar.FileInfoHeader(srcSt, "")
+			if err != nil {
+				return err
+			}
+			hdr.Name = filepath.Base(dst)
+			hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
+
+			if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
+				return err
+			}
+
+			tw := tar.NewWriter(w)
+			defer tw.Close()
+			if err := tw.WriteHeader(hdr); err != nil {
+				return err
+			}
+			if _, err := io.Copy(tw, srcF); err != nil {
+				return err
+			}
+			return nil
+		}()
+	}()
 	defer func() {
 		if er := <-errC; err == nil && er != nil {
 			err = er

components/engine/pkg/archive/archive_unix_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"syscall"
 	"testing"
 
@@ -261,3 +262,58 @@ func TestTarUntarWithXattr(t *testing.T) {
 		}
 	}
 }
+
+func TestCopyInfoDestinationPathSymlink(t *testing.T) {
+	tmpDir, _ := getTestTempDirs(t)
+	defer removeAllPaths(tmpDir)
+
+	root := strings.TrimRight(tmpDir, "/") + "/"
+
+	type FileTestData struct {
+		resource FileData
+		file     string
+		expected CopyInfo
+	}
+
+	testData := []FileTestData{
+		//Create a directory: /tmp/archive-copy-test*/dir1
+		//Test will "copy" file1 to dir1
+		{resource: FileData{filetype: Dir, path: "dir1", permissions: 0740}, file: "file1", expected: CopyInfo{Path: root + "dir1/file1", Exists: false, IsDir: false}},
+
+		//Create a symlink directory to dir1: /tmp/archive-copy-test*/dirSymlink -> dir1
+		//Test will "copy" file2 to dirSymlink
+		{resource: FileData{filetype: Symlink, path: "dirSymlink", contents: root + "dir1", permissions: 0600}, file: "file2", expected: CopyInfo{Path: root + "dirSymlink/file2", Exists: false, IsDir: false}},
+
+		//Create a file in tmp directory: /tmp/archive-copy-test*/file1
+		//Test to cover when the full file path already exists.
+		{resource: FileData{filetype: Regular, path: "file1", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "file1", Exists: true}},
+
+		//Create a directory: /tmp/archive-copy*/dir2
+		//Test to cover when the full directory path already exists
+		{resource: FileData{filetype: Dir, path: "dir2", permissions: 0740}, file: "", expected: CopyInfo{Path: root + "dir2", Exists: true, IsDir: true}},
+
+		//Create a symlink to a non-existent target: /tmp/archive-copy*/symlink1 -> noSuchTarget
+		//Negative test to cover symlinking to a target that does not exit
+		{resource: FileData{filetype: Symlink, path: "symlink1", contents: "noSuchTarget", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "noSuchTarget", Exists: false}},
+
+		//Create a file in tmp directory for next test: /tmp/existingfile
+		{resource: FileData{filetype: Regular, path: "existingfile", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "existingfile", Exists: true}},
+
+		//Create a symlink to an existing file: /tmp/archive-copy*/symlink2 -> /tmp/existingfile
+		//Test to cover when the parent directory of a new file is a symlink
+		{resource: FileData{filetype: Symlink, path: "symlink2", contents: "existingfile", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "existingfile", Exists: true}},
+	}
+
+	var dirs []FileData
+	for _, data := range testData {
+		dirs = append(dirs, data.resource)
+	}
+	provisionSampleDir(t, tmpDir, dirs)
+
+	for _, info := range testData {
+		p := filepath.Join(tmpDir, info.resource.path, info.file)
+		ci, err := CopyInfoDestinationPath(p)
+		assert.NoError(t, err)
+		assert.Equal(t, info.expected, ci)
+	}
+}

components/engine/pkg/archive/changes_test.go

@@ -50,32 +50,35 @@ type FileData struct {
 
 func createSampleDir(t *testing.T, root string) {
 	files := []FileData{
-		{Regular, "file1", "file1\n", 0600},
-		{Regular, "file2", "file2\n", 0666},
-		{Regular, "file3", "file3\n", 0404},
-		{Regular, "file4", "file4\n", 0600},
-		{Regular, "file5", "file5\n", 0600},
-		{Regular, "file6", "file6\n", 0600},
-		{Regular, "file7", "file7\n", 0600},
-		{Dir, "dir1", "", 0740},
-		{Regular, "dir1/file1-1", "file1-1\n", 01444},
-		{Regular, "dir1/file1-2", "file1-2\n", 0666},
-		{Dir, "dir2", "", 0700},
-		{Regular, "dir2/file2-1", "file2-1\n", 0666},
-		{Regular, "dir2/file2-2", "file2-2\n", 0666},
-		{Dir, "dir3", "", 0700},
-		{Regular, "dir3/file3-1", "file3-1\n", 0666},
-		{Regular, "dir3/file3-2", "file3-2\n", 0666},
-		{Dir, "dir4", "", 0700},
-		{Regular, "dir4/file3-1", "file4-1\n", 0666},
-		{Regular, "dir4/file3-2", "file4-2\n", 0666},
-		{Symlink, "symlink1", "target1", 0666},
-		{Symlink, "symlink2", "target2", 0666},
-		{Symlink, "symlink3", root + "/file1", 0666},
-		{Symlink, "symlink4", root + "/symlink3", 0666},
-		{Symlink, "dirSymlink", root + "/dir1", 0740},
+		{filetype: Regular, path: "file1", contents: "file1\n", permissions: 0600},
+		{filetype: Regular, path: "file2", contents: "file2\n", permissions: 0666},
+		{filetype: Regular, path: "file3", contents: "file3\n", permissions: 0404},
+		{filetype: Regular, path: "file4", contents: "file4\n", permissions: 0600},
+		{filetype: Regular, path: "file5", contents: "file5\n", permissions: 0600},
+		{filetype: Regular, path: "file6", contents: "file6\n", permissions: 0600},
+		{filetype: Regular, path: "file7", contents: "file7\n", permissions: 0600},
+		{filetype: Dir, path: "dir1", contents: "", permissions: 0740},
+		{filetype: Regular, path: "dir1/file1-1", contents: "file1-1\n", permissions: 01444},
+		{filetype: Regular, path: "dir1/file1-2", contents: "file1-2\n", permissions: 0666},
+		{filetype: Dir, path: "dir2", contents: "", permissions: 0700},
+		{filetype: Regular, path: "dir2/file2-1", contents: "file2-1\n", permissions: 0666},
+		{filetype: Regular, path: "dir2/file2-2", contents: "file2-2\n", permissions: 0666},
+		{filetype: Dir, path: "dir3", contents: "", permissions: 0700},
+		{filetype: Regular, path: "dir3/file3-1", contents: "file3-1\n", permissions: 0666},
+		{filetype: Regular, path: "dir3/file3-2", contents: "file3-2\n", permissions: 0666},
+		{filetype: Dir, path: "dir4", contents: "", permissions: 0700},
+		{filetype: Regular, path: "dir4/file3-1", contents: "file4-1\n", permissions: 0666},
+		{filetype: Regular, path: "dir4/file3-2", contents: "file4-2\n", permissions: 0666},
+		{filetype: Symlink, path: "symlink1", contents: "target1", permissions: 0666},
+		{filetype: Symlink, path: "symlink2", contents: "target2", permissions: 0666},
+		{filetype: Symlink, path: "symlink3", contents: root + "/file1", permissions: 0666},
+		{filetype: Symlink, path: "symlink4", contents: root + "/symlink3", permissions: 0666},
+		{filetype: Symlink, path: "dirSymlink", contents: root + "/dir1", permissions: 0740},
 	}
+	provisionSampleDir(t, root, files)
+}
 
+func provisionSampleDir(t *testing.T, root string, files []FileData) {
 	now := time.Now()
 	for _, info := range files {
 		p := path.Join(root, info.path)

components/engine/pkg/archive/copy.go

@@ -223,7 +223,7 @@ func CopyInfoDestinationPath(path string) (info CopyInfo, err error) {
 		// Ensure destination parent dir exists.
 		dstParent, _ := SplitPathDirEntry(path)
 
-		parentDirStat, err := os.Lstat(dstParent)
+		parentDirStat, err := os.Stat(dstParent)
 		if err != nil {
 			return CopyInfo{}, err
 		}

components/engine/pkg/containerfs/archiver.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/idtools"
-	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/pkg/system"
 	"github.com/sirupsen/logrus"
 )
@@ -122,40 +121,45 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 	}
 
 	r, w := io.Pipe()
-	errC := promise.Go(func() error {
-		defer w.Close()
+	errC := make(chan error, 1)
 
-		srcF, err := srcDriver.Open(src)
-		if err != nil {
-			return err
-		}
-		defer srcF.Close()
+	go func() {
+		defer close(errC)
+		errC <- func() error {
+			defer w.Close()
 
-		hdr, err := tar.FileInfoHeader(srcSt, "")
-		if err != nil {
-			return err
-		}
-		hdr.Name = dstDriver.Base(dst)
-		if dstDriver.OS() == "windows" {
-			hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
-		} else {
-			hdr.Mode = int64(os.FileMode(hdr.Mode))
-		}
+			srcF, err := srcDriver.Open(src)
+			if err != nil {
+				return err
+			}
+			defer srcF.Close()
 
-		if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
-			return err
-		}
+			hdr, err := tar.FileInfoHeader(srcSt, "")
+			if err != nil {
+				return err
+			}
+			hdr.Name = dstDriver.Base(dst)
+			if dstDriver.OS() == "windows" {
+				hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
+			} else {
+				hdr.Mode = int64(os.FileMode(hdr.Mode))
+			}
 
-		tw := tar.NewWriter(w)
-		defer tw.Close()
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if _, err := io.Copy(tw, srcF); err != nil {
-			return err
-		}
-		return nil
-	})
+			if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
+				return err
+			}
+
+			tw := tar.NewWriter(w)
+			defer tw.Close()
+			if err := tw.WriteHeader(hdr); err != nil {
+				return err
+			}
+			if _, err := io.Copy(tw, srcF); err != nil {
+				return err
+			}
+			return nil
+		}()
+	}()
 	defer func() {
 		if er := <-errC; err == nil && er != nil {
 			err = er

components/engine/pkg/jsonlog/jsonlog.go

@@ -1,42 +0,0 @@
-package jsonlog
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-)
-
-// JSONLog represents a log message, typically a single entry from a given log stream.
-// JSONLogs can be easily serialized to and from JSON and support custom formatting.
-type JSONLog struct {
-	// Log is the log message
-	Log string `json:"log,omitempty"`
-	// Stream is the log source
-	Stream string `json:"stream,omitempty"`
-	// Created is the created timestamp of log
-	Created time.Time `json:"time"`
-	// Attrs is the list of extra attributes provided by the user
-	Attrs map[string]string `json:"attrs,omitempty"`
-}
-
-// Format returns the log formatted according to format
-// If format is nil, returns the log message
-// If format is json, returns the log marshaled in json format
-// By default, returns the log with the log time formatted according to format.
-func (jl *JSONLog) Format(format string) (string, error) {
-	if format == "" {
-		return jl.Log, nil
-	}
-	if format == "json" {
-		m, err := json.Marshal(jl)
-		return string(m), err
-	}
-	return fmt.Sprintf("%s %s", jl.Created.Format(format), jl.Log), nil
-}
-
-// Reset resets the log to nil.
-func (jl *JSONLog) Reset() {
-	jl.Log = ""
-	jl.Stream = ""
-	jl.Created = time.Time{}
-}

components/engine/pkg/jsonlog/jsonlog_marshalling.go

@@ -1,178 +0,0 @@
-// This code was initially generated by ffjson <https://github.com/pquerna/ffjson>
-// This code was generated via the following steps:
-// $ go get -u github.com/pquerna/ffjson
-// $ make BIND_DIR=. shell
-// $ ffjson pkg/jsonlog/jsonlog.go
-// $ mv pkg/jsonglog/jsonlog_ffjson.go pkg/jsonlog/jsonlog_marshalling.go
-//
-// It has been modified to improve the performance of time marshalling to JSON
-// and to clean it up.
-// Should this code need to be regenerated when the JSONLog struct is changed,
-// the relevant changes which have been made are:
-// import (
-//        "bytes"
-//-
-//        "unicode/utf8"
-// )
-//
-// func (mj *JSONLog) MarshalJSON() ([]byte, error) {
-//@@ -20,13 +16,13 @@ func (mj *JSONLog) MarshalJSON() ([]byte, error) {
-//        }
-//        return buf.Bytes(), nil
-// }
-//+
-// func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
-//-       var err error
-//-       var obj []byte
-//-       var first bool = true
-//-       _ = obj
-//-       _ = err
-//-       _ = first
-//+       var (
-//+               err       error
-//+               timestamp string
-//+               first     bool = true
-//+       )
-//        buf.WriteString(`{`)
-//        if len(mj.Log) != 0 {
-//                if first == true {
-//@@ -52,11 +48,11 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
-//                buf.WriteString(`,`)
-//        }
-//        buf.WriteString(`"time":`)
-//-       obj, err = mj.Created.MarshalJSON()
-//+       timestamp, err = FastTimeMarshalJSON(mj.Created)
-//        if err != nil {
-//                return err
-//        }
-//-       buf.Write(obj)
-//+       buf.WriteString(timestamp)
-//        buf.WriteString(`}`)
-//        return nil
-// }
-// @@ -81,9 +81,10 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
-//         if len(mj.Log) != 0 {
-// -                if first == true {
-// -                       first = false
-// -               } else {
-// -                       buf.WriteString(`,`)
-// -               }
-// +               first = false
-//                 buf.WriteString(`"log":`)
-//                 ffjsonWriteJSONString(buf, mj.Log)
-//         }
-
-package jsonlog
-
-import (
-	"bytes"
-	"unicode/utf8"
-)
-
-// MarshalJSON marshals the JSONLog.
-func (mj *JSONLog) MarshalJSON() ([]byte, error) {
-	var buf bytes.Buffer
-	buf.Grow(1024)
-	if err := mj.MarshalJSONBuf(&buf); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-// MarshalJSONBuf marshals the JSONLog and stores the result to a bytes.Buffer.
-func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
-	var (
-		err       error
-		timestamp string
-		first     = true
-	)
-	buf.WriteString(`{`)
-	if len(mj.Log) != 0 {
-		first = false
-		buf.WriteString(`"log":`)
-		ffjsonWriteJSONString(buf, mj.Log)
-	}
-	if len(mj.Stream) != 0 {
-		if first {
-			first = false
-		} else {
-			buf.WriteString(`,`)
-		}
-		buf.WriteString(`"stream":`)
-		ffjsonWriteJSONString(buf, mj.Stream)
-	}
-	if !first {
-		buf.WriteString(`,`)
-	}
-	buf.WriteString(`"time":`)
-	timestamp, err = FastTimeMarshalJSON(mj.Created)
-	if err != nil {
-		return err
-	}
-	buf.WriteString(timestamp)
-	buf.WriteString(`}`)
-	return nil
-}
-
-func ffjsonWriteJSONString(buf *bytes.Buffer, s string) {
-	const hex = "0123456789abcdef"
-
-	buf.WriteByte('"')
-	start := 0
-	for i := 0; i < len(s); {
-		if b := s[i]; b < utf8.RuneSelf {
-			if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' {
-				i++
-				continue
-			}
-			if start < i {
-				buf.WriteString(s[start:i])
-			}
-			switch b {
-			case '\\', '"':
-				buf.WriteByte('\\')
-				buf.WriteByte(b)
-			case '\n':
-				buf.WriteByte('\\')
-				buf.WriteByte('n')
-			case '\r':
-				buf.WriteByte('\\')
-				buf.WriteByte('r')
-			default:
-
-				buf.WriteString(`\u00`)
-				buf.WriteByte(hex[b>>4])
-				buf.WriteByte(hex[b&0xF])
-			}
-			i++
-			start = i
-			continue
-		}
-		c, size := utf8.DecodeRuneInString(s[i:])
-		if c == utf8.RuneError && size == 1 {
-			if start < i {
-				buf.WriteString(s[start:i])
-			}
-			buf.WriteString(`\ufffd`)
-			i += size
-			start = i
-			continue
-		}
-
-		if c == '\u2028' || c == '\u2029' {
-			if start < i {
-				buf.WriteString(s[start:i])
-			}
-			buf.WriteString(`\u202`)
-			buf.WriteByte(hex[c&0xF])
-			i += size
-			start = i
-			continue
-		}
-		i += size
-	}
-	if start < len(s) {
-		buf.WriteString(s[start:])
-	}
-	buf.WriteByte('"')
-}

components/engine/pkg/jsonlog/jsonlog_marshalling_test.go

@@ -1,34 +0,0 @@
-package jsonlog
-
-import (
-	"regexp"
-	"testing"
-)
-
-func TestJSONLogMarshalJSON(t *testing.T) {
-	logs := map[*JSONLog]string{
-		{Log: `"A log line with \\"`}:           `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`,
-		{Log: "A log line"}:                     `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`,
-		{Log: "A log line with \r"}:             `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`,
-		{Log: "A log line with & < >"}:          `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`,
-		{Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`,
-		{Stream: "stdout"}:                      `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`,
-		{}:                                      `^{\"time\":\".{20,}\"}$`,
-		// These ones are a little weird
-		{Log: "\u2028 \u2029"}:      `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`,
-		{Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`,
-		{Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`,
-	}
-	for jsonLog, expression := range logs {
-		data, err := jsonLog.MarshalJSON()
-		if err != nil {
-			t.Fatal(err)
-		}
-		res := string(data)
-		t.Logf("Result of WriteLog: %q", res)
-		logRe := regexp.MustCompile(expression)
-		if !logRe.MatchString(res) {
-			t.Fatalf("Log line not in expected format [%v]: %q", expression, res)
-		}
-	}
-}

components/engine/pkg/jsonlog/jsonlogbytes_test.go

@@ -1,39 +0,0 @@
-package jsonlog
-
-import (
-	"bytes"
-	"regexp"
-	"testing"
-)
-
-func TestJSONLogsMarshalJSONBuf(t *testing.T) {
-	logs := map[*JSONLogs]string{
-		{Log: []byte(`"A log line with \\"`)}:           `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":}$`,
-		{Log: []byte("A log line")}:                     `^{\"log\":\"A log line\",\"time\":}$`,
-		{Log: []byte("A log line with \r")}:             `^{\"log\":\"A log line with \\r\",\"time\":}$`,
-		{Log: []byte("A log line with & < >")}:          `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":}$`,
-		{Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":}$`,
-		{Stream: "stdout"}:                              `^{\"stream\":\"stdout\",\"time\":}$`,
-		{Stream: "stdout", Log: []byte("A log line")}:   `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":}$`,
-		{Created: "time"}:                               `^{\"time\":time}$`,
-		{}:                                              `^{\"time\":}$`,
-		// These ones are a little weird
-		{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`,
-		{Log: []byte{0xaF}}:            `^{\"log\":\"\\ufffd\",\"time\":}$`,
-		{Log: []byte{0x7F}}:            `^{\"log\":\"\x7f\",\"time\":}$`,
-		// with raw attributes
-		{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`,
-	}
-	for jsonLog, expression := range logs {
-		var buf bytes.Buffer
-		if err := jsonLog.MarshalJSONBuf(&buf); err != nil {
-			t.Fatal(err)
-		}
-		res := buf.String()
-		t.Logf("Result of WriteLog: %q", res)
-		logRe := regexp.MustCompile(expression)
-		if !logRe.MatchString(res) {
-			t.Fatalf("Log line not in expected format [%v]: %q", expression, res)
-		}
-	}
-}

components/engine/pkg/jsonlog/time_marshalling.go

@@ -1,27 +0,0 @@
-// Package jsonlog provides helper functions to parse and print time (time.Time) as JSON.
-package jsonlog
-
-import (
-	"errors"
-	"time"
-)
-
-const (
-	// RFC3339NanoFixed is our own version of RFC339Nano because we want one
-	// that pads the nano seconds part with zeros to ensure
-	// the timestamps are aligned in the logs.
-	RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-	// JSONFormat is the format used by FastMarshalJSON
-	JSONFormat = `"` + time.RFC3339Nano + `"`
-)
-
-// FastTimeMarshalJSON avoids one of the extra allocations that
-// time.MarshalJSON is making.
-func FastTimeMarshalJSON(t time.Time) (string, error) {
-	if y := t.Year(); y < 0 || y >= 10000 {
-		// RFC 3339 is clear that years are 4 digits exactly.
-		// See golang.org/issue/4556#c15 for more discussion.
-		return "", errors.New("time.MarshalJSON: year outside of range [0,9999]")
-	}
-	return t.Format(JSONFormat), nil
-}

components/engine/pkg/jsonlog/time_marshalling_test.go

@@ -1,47 +0,0 @@
-package jsonlog
-
-import (
-	"testing"
-	"time"
-)
-
-// Testing to ensure 'year' fields is between 0 and 9999
-func TestFastTimeMarshalJSONWithInvalidDate(t *testing.T) {
-	aTime := time.Date(-1, 1, 1, 0, 0, 0, 0, time.Local)
-	json, err := FastTimeMarshalJSON(aTime)
-	if err == nil {
-		t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json)
-	}
-	anotherTime := time.Date(10000, 1, 1, 0, 0, 0, 0, time.Local)
-	json, err = FastTimeMarshalJSON(anotherTime)
-	if err == nil {
-		t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json)
-	}
-
-}
-
-func TestFastTimeMarshalJSON(t *testing.T) {
-	aTime := time.Date(2015, 5, 29, 11, 1, 2, 3, time.UTC)
-	json, err := FastTimeMarshalJSON(aTime)
-	if err != nil {
-		t.Fatal(err)
-	}
-	expected := "\"2015-05-29T11:01:02.000000003Z\""
-	if json != expected {
-		t.Fatalf("Expected %v, got %v", expected, json)
-	}
-
-	location, err := time.LoadLocation("Europe/Paris")
-	if err != nil {
-		t.Fatal(err)
-	}
-	aTime = time.Date(2015, 5, 29, 11, 1, 2, 3, location)
-	json, err = FastTimeMarshalJSON(aTime)
-	if err != nil {
-		t.Fatal(err)
-	}
-	expected = "\"2015-05-29T11:01:02.000000003+02:00\""
-	if json != expected {
-		t.Fatalf("Expected %v, got %v", expected, json)
-	}
-}

components/engine/pkg/jsonmessage/jsonmessage.go

@@ -9,11 +9,14 @@ import (
 	"time"
 
 	gotty "github.com/Nvveen/Gotty"
-	"github.com/docker/docker/pkg/jsonlog"
 	"github.com/docker/docker/pkg/term"
 	units "github.com/docker/go-units"
 )
 
+// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to
+// ensure the formatted time isalways the same number of characters.
+const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
 // JSONError wraps a concrete Code and Message, `Code` is
 // is an integer error code, `Message` is the error message.
 type JSONError struct {
@@ -199,9 +202,9 @@ func (jm *JSONMessage) Display(out io.Writer, termInfo termInfo) error {
 		return nil
 	}
 	if jm.TimeNano != 0 {
-		fmt.Fprintf(out, "%s ", time.Unix(0, jm.TimeNano).Format(jsonlog.RFC3339NanoFixed))
+		fmt.Fprintf(out, "%s ", time.Unix(0, jm.TimeNano).Format(RFC3339NanoFixed))
 	} else if jm.Time != 0 {
-		fmt.Fprintf(out, "%s ", time.Unix(jm.Time, 0).Format(jsonlog.RFC3339NanoFixed))
+		fmt.Fprintf(out, "%s ", time.Unix(jm.Time, 0).Format(RFC3339NanoFixed))
 	}
 	if jm.ID != "" {
 		fmt.Fprintf(out, "%s: ", jm.ID)

components/engine/pkg/jsonmessage/jsonmessage_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/docker/docker/pkg/jsonlog"
 	"github.com/docker/docker/pkg/term"
 	"github.com/stretchr/testify/assert"
 )
@@ -115,8 +114,8 @@ func TestJSONMessageDisplay(t *testing.T) {
 			From:   "From",
 			Status: "status",
 		}: {
-			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(jsonlog.RFC3339NanoFixed)),
-			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(jsonlog.RFC3339NanoFixed)),
+			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(RFC3339NanoFixed)),
+			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(RFC3339NanoFixed)),
 		},
 		// General, with nano precision time
 		{
@@ -125,8 +124,8 @@ func TestJSONMessageDisplay(t *testing.T) {
 			From:     "From",
 			Status:   "status",
 		}: {
-			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)),
-			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)),
+			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)),
+			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)),
 		},
 		// General, with both times Nano is preferred
 		{
@@ -136,8 +135,8 @@ func TestJSONMessageDisplay(t *testing.T) {
 			From:     "From",
 			Status:   "status",
 		}: {
-			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)),
-			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)),
+			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)),
+			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)),
 		},
 		// Stream over status
 		{

components/engine/pkg/mount/mounter_linux_test.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"strings"
 	"testing"
+
+	selinux "github.com/opencontainers/selinux/go-selinux"
 )
 
 func TestMount(t *testing.T) {
@@ -101,7 +103,11 @@ func TestMount(t *testing.T) {
 				t.Fatal(err)
 			}
 			defer ensureUnmount(t, target)
-			validateMount(t, target, tc.expectedOpts, tc.expectedOptional, tc.expectedVFS)
+			expectedVFS := tc.expectedVFS
+			if selinux.GetEnabled() && expectedVFS != "" {
+				expectedVFS = expectedVFS + ",seclabel"
+			}
+			validateMount(t, target, tc.expectedOpts, tc.expectedOptional, expectedVFS)
 		})
 	}
 }

components/engine/pkg/mount/mountinfo_solaris.go

@@ -4,16 +4,23 @@ package mount
 
 /*
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/mnttab.h>
 */
 import "C"
 
 import (
 	"fmt"
+	"unsafe"
 )
 
 func parseMountTable() ([]*Info, error) {
-	mnttab := C.fopen(C.CString(C.MNTTAB), C.CString("r"))
+	path := C.CString(C.MNTTAB)
+	defer C.free(unsafe.Pointer(path))
+	mode := C.CString("r")
+	defer C.free(unsafe.Pointer(mode))
+
+	mnttab := C.fopen(path, mode)
 	if mnttab == nil {
 		return nil, fmt.Errorf("Failed to open %s", C.MNTTAB)
 	}

components/engine/pkg/promise/promise.go

@@ -1,11 +0,0 @@
-package promise
-
-// Go is a basic promise implementation: it wraps calls a function in a goroutine,
-// and returns a channel which will later return the function's return value.
-func Go(f func() error) chan error {
-	ch := make(chan error, 1)
-	go func() {
-		ch <- f()
-	}()
-	return ch
-}

components/engine/pkg/promise/promise_test.go

@@ -1,25 +0,0 @@
-package promise
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestGo(t *testing.T) {
-	errCh := Go(functionWithError)
-	er := <-errCh
-	require.EqualValues(t, "Error Occurred", er.Error())
-
-	noErrCh := Go(functionWithNoError)
-	er = <-noErrCh
-	require.Nil(t, er)
-}
-
-func functionWithError() (err error) {
-	return errors.New("Error Occurred")
-}
-func functionWithNoError() (err error) {
-	return nil
-}

components/engine/plugin/executor/containerd/containerd.go

@@ -0,0 +1,77 @@
+package containerd
+
+import (
+	"io"
+
+	"github.com/docker/docker/libcontainerd"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+)
+
+// ExitHandler represents an object that is called when the exit event is received from containerd
+type ExitHandler interface {
+	HandleExitEvent(id string) error
+}
+
+// New creates a new containerd plugin executor
+func New(remote libcontainerd.Remote, exitHandler ExitHandler) (*Executor, error) {
+	e := &Executor{exitHandler: exitHandler}
+	client, err := remote.Client(e)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating containerd exec client")
+	}
+	e.client = client
+	return e, nil
+}
+
+// Executor is the containerd client implementation of a plugin executor
+type Executor struct {
+	client      libcontainerd.Client
+	exitHandler ExitHandler
+}
+
+// Create creates a new container
+func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error {
+	return e.client.Create(id, "", "", spec, attachStreamsFunc(stdout, stderr))
+}
+
+// Restore restores a container
+func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) error {
+	return e.client.Restore(id, attachStreamsFunc(stdout, stderr))
+}
+
+// IsRunning returns if the container with the given id is running
+func (e *Executor) IsRunning(id string) (bool, error) {
+	pids, err := e.client.GetPidsForContainer(id)
+	return len(pids) > 0, err
+}
+
+// Signal sends the specified signal to the container
+func (e *Executor) Signal(id string, signal int) error {
+	return e.client.Signal(id, signal)
+}
+
+// StateChanged handles state changes from containerd
+// All events are ignored except the exit event, which is sent of to the stored handler
+func (e *Executor) StateChanged(id string, event libcontainerd.StateInfo) error {
+	switch event.State {
+	case libcontainerd.StateExit:
+		return e.exitHandler.HandleExitEvent(id)
+	}
+	return nil
+}
+
+func attachStreamsFunc(stdout, stderr io.WriteCloser) func(libcontainerd.IOPipe) error {
+	return func(iop libcontainerd.IOPipe) error {
+		iop.Stdin.Close()
+		go func() {
+			io.Copy(stdout, iop.Stdout)
+			stdout.Close()
+		}()
+		go func() {
+			io.Copy(stderr, iop.Stderr)
+			stderr.Close()
+		}()
+		return nil
+	}
+}

components/engine/plugin/manager.go

@@ -17,7 +17,6 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
-	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/pkg/authorization"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/mount"
@@ -26,6 +25,7 @@ import (
 	"github.com/docker/docker/plugin/v2"
 	"github.com/docker/docker/registry"
 	"github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -35,6 +35,14 @@ const rootFSFileName = "rootfs"
 
 var validFullID = regexp.MustCompile(`^([a-f0-9]{64})$`)
 
+// Executor is the interface that the plugin manager uses to interact with for starting/stopping plugins
+type Executor interface {
+	Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error
+	Restore(id string, stdout, stderr io.WriteCloser) error
+	IsRunning(id string) (bool, error)
+	Signal(id string, signal int) error
+}
+
 func (pm *Manager) restorePlugin(p *v2.Plugin) error {
 	if p.IsEnabled() {
 		return pm.restore(p)
@@ -47,24 +55,27 @@ type eventLogger func(id, name, action string)
 // ManagerConfig defines configuration needed to start new manager.
 type ManagerConfig struct {
 	Store              *Store // remove
-	Executor           libcontainerd.Remote
 	RegistryService    registry.Service
 	LiveRestoreEnabled bool // TODO: remove
 	LogPluginEvent     eventLogger
 	Root               string
 	ExecRoot           string
+	CreateExecutor     ExecutorCreator
 	AuthzMiddleware    *authorization.Middleware
 }
 
+// ExecutorCreator is used in the manager config to pass in an `Executor`
+type ExecutorCreator func(*Manager) (Executor, error)
+
 // Manager controls the plugin subsystem.
 type Manager struct {
-	config           ManagerConfig
-	mu               sync.RWMutex // protects cMap
-	muGC             sync.RWMutex // protects blobstore deletions
-	cMap             map[*v2.Plugin]*controller
-	containerdClient libcontainerd.Client
-	blobStore        *basicBlobStore
-	publisher        *pubsub.Publisher
+	config    ManagerConfig
+	mu        sync.RWMutex // protects cMap
+	muGC      sync.RWMutex // protects blobstore deletions
+	cMap      map[*v2.Plugin]*controller
+	blobStore *basicBlobStore
+	publisher *pubsub.Publisher
+	executor  Executor
 }
 
 // controller represents the manager's control on a plugin.
@@ -111,10 +122,11 @@ func NewManager(config ManagerConfig) (*Manager, error) {
 	}
 
 	var err error
-	manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct
+	manager.executor, err = config.CreateExecutor(manager)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to create containerd client")
+		return nil, err
 	}
+
 	manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))
 	if err != nil {
 		return nil, err
@@ -133,42 +145,37 @@ func (pm *Manager) tmpDir() string {
 	return filepath.Join(pm.config.Root, "tmp")
 }
 
-// StateChanged updates plugin internals using libcontainerd events.
-func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error {
-	logrus.Debugf("plugin state changed %s %#v", id, e)
+// HandleExitEvent is called when the executor receives the exit event
+// In the future we may change this, but for now all we care about is the exit event.
+func (pm *Manager) HandleExitEvent(id string) error {
+	p, err := pm.config.Store.GetV2Plugin(id)
+	if err != nil {
+		return err
+	}
 
-	switch e.State {
-	case libcontainerd.StateExit:
-		p, err := pm.config.Store.GetV2Plugin(id)
-		if err != nil {
-			return err
+	os.RemoveAll(filepath.Join(pm.config.ExecRoot, id))
+
+	if p.PropagatedMount != "" {
+		if err := mount.Unmount(p.PropagatedMount); err != nil {
+			logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
 		}
-
-		os.RemoveAll(filepath.Join(pm.config.ExecRoot, id))
-
-		if p.PropagatedMount != "" {
-			if err := mount.Unmount(p.PropagatedMount); err != nil {
-				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
-			}
-			propRoot := filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")
-			if err := mount.Unmount(propRoot); err != nil {
-				logrus.Warn("Could not unmount %s: %v", propRoot, err)
-			}
-		}
-
-		pm.mu.RLock()
-		c := pm.cMap[p]
-		if c.exitChan != nil {
-			close(c.exitChan)
-		}
-		restart := c.restart
-		pm.mu.RUnlock()
-
-		if restart {
-			pm.enable(p, c, true)
+		propRoot := filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")
+		if err := mount.Unmount(propRoot); err != nil {
+			logrus.Warn("Could not unmount %s: %v", propRoot, err)
 		}
 	}
 
+	pm.mu.RLock()
+	c := pm.cMap[p]
+	if c.exitChan != nil {
+		close(c.exitChan)
+	}
+	restart := c.restart
+	pm.mu.RUnlock()
+
+	if restart {
+		pm.enable(p, c, true)
+	}
 	return nil
 }
 
@@ -333,23 +340,10 @@ func (l logHook) Fire(entry *logrus.Entry) error {
 	return nil
 }
 
-func attachToLog(id string) func(libcontainerd.IOPipe) error {
-	return func(iop libcontainerd.IOPipe) error {
-		iop.Stdin.Close()
-
-		logger := logrus.New()
-		logger.Hooks.Add(logHook{id})
-		// TODO: cache writer per id
-		w := logger.Writer()
-		go func() {
-			io.Copy(w, iop.Stdout)
-		}()
-		go func() {
-			// TODO: update logrus and use logger.WriterLevel
-			io.Copy(w, iop.Stderr)
-		}()
-		return nil
-	}
+func makeLoggerStreams(id string) (stdout, stderr io.WriteCloser) {
+	logger := logrus.New()
+	logger.Hooks.Add(logHook{id})
+	return logger.WriterLevel(logrus.InfoLevel), logger.WriterLevel(logrus.ErrorLevel)
 }
 
 func validatePrivileges(requiredPrivileges, privileges types.PluginPrivileges) error {

components/engine/plugin/manager_linux.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/daemon/initlayer"
-	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
@@ -63,7 +62,8 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
 		return errors.WithStack(err)
 	}
 
-	if err := pm.containerdClient.Create(p.GetID(), "", "", *spec, attachToLog(p.GetID())); err != nil {
+	stdout, stderr := makeLoggerStreams(p.GetID())
+	if err := pm.executor.Create(p.GetID(), *spec, stdout, stderr); err != nil {
 		if p.PropagatedMount != "" {
 			if err := mount.Unmount(p.PropagatedMount); err != nil {
 				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
@@ -83,7 +83,7 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
 	client, err := plugins.NewClientWithTimeout("unix://"+sockAddr, nil, time.Duration(c.timeoutInSecs)*time.Second)
 	if err != nil {
 		c.restart = false
-		shutdownPlugin(p, c, pm.containerdClient)
+		shutdownPlugin(p, c, pm.executor)
 		return errors.WithStack(err)
 	}
 
@@ -109,7 +109,7 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
 			c.restart = false
 			// While restoring plugins, we need to explicitly set the state to disabled
 			pm.config.Store.SetState(p, false)
-			shutdownPlugin(p, c, pm.containerdClient)
+			shutdownPlugin(p, c, pm.executor)
 			return err
 		}
 
@@ -121,13 +121,14 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
 }
 
 func (pm *Manager) restore(p *v2.Plugin) error {
-	if err := pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())); err != nil {
+	stdout, stderr := makeLoggerStreams(p.GetID())
+	if err := pm.executor.Restore(p.GetID(), stdout, stderr); err != nil {
 		return err
 	}
 
 	if pm.config.LiveRestoreEnabled {
 		c := &controller{}
-		if pids, _ := pm.containerdClient.GetPidsForContainer(p.GetID()); len(pids) == 0 {
+		if isRunning, _ := pm.executor.IsRunning(p.GetID()); !isRunning {
 			// plugin is not running, so follow normal startup procedure
 			return pm.enable(p, c, true)
 		}
@@ -143,10 +144,10 @@ func (pm *Manager) restore(p *v2.Plugin) error {
 	return nil
 }
 
-func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.Client) {
+func shutdownPlugin(p *v2.Plugin, c *controller, executor Executor) {
 	pluginID := p.GetID()
 
-	err := containerdClient.Signal(pluginID, int(unix.SIGTERM))
+	err := executor.Signal(pluginID, int(unix.SIGTERM))
 	if err != nil {
 		logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err)
 	} else {
@@ -155,7 +156,7 @@ func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.
 			logrus.Debug("Clean shutdown of plugin")
 		case <-time.After(time.Second * 10):
 			logrus.Debug("Force shutdown plugin")
-			if err := containerdClient.Signal(pluginID, int(unix.SIGKILL)); err != nil {
+			if err := executor.Signal(pluginID, int(unix.SIGKILL)); err != nil {
 				logrus.Errorf("Sending SIGKILL to plugin failed with error: %v", err)
 			}
 		}
@@ -175,7 +176,7 @@ func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
 	}
 
 	c.restart = false
-	shutdownPlugin(p, c, pm.containerdClient)
+	shutdownPlugin(p, c, pm.executor)
 	pm.config.Store.SetState(p, false)
 	return pm.save(p)
 }
@@ -192,9 +193,9 @@ func (pm *Manager) Shutdown() {
 			logrus.Debug("Plugin active when liveRestore is set, skipping shutdown")
 			continue
 		}
-		if pm.containerdClient != nil && p.IsEnabled() {
+		if pm.executor != nil && p.IsEnabled() {
 			c.restart = false
-			shutdownPlugin(p, c, pm.containerdClient)
+			shutdownPlugin(p, c, pm.executor)
 		}
 	}
 	mount.Unmount(pm.config.Root)

components/engine/vendor.conf

@@ -2,13 +2,12 @@
 github.com/Azure/go-ansiterm 19f72df4d05d31cbe1c56bfc8045c96babff6c7e
 github.com/Microsoft/hcsshim v0.6.5
 github.com/Microsoft/go-winio v0.4.5
-github.com/moby/buildkit da2b9dc7dab99e824b2b1067ad7d0523e32dd2d9 https://github.com/dmcgowan/buildkit.git
 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git
 github.com/gorilla/context v1.1
 github.com/gorilla/mux v1.1
-github.com/Microsoft/opengcs v0.3.3
+github.com/Microsoft/opengcs v0.3.4
 github.com/kr/pty 5cf931ef8f
 github.com/mattn/go-shellwords v1.0.3
 github.com/sirupsen/logrus v1.0.3
@@ -28,9 +27,11 @@ github.com/imdario/mergo 0.2.1
 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0
 
 github.com/containerd/continuity 22694c680ee48fb8f50015b44618517e2bde77e8
+github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8
+github.com/tonistiigi/fsutil 1dedf6e90084bd88c4c518a15e68a37ed1370203
 
 #get libnetwork packages
-github.com/docker/libnetwork 60e002dd61885e1cd909582f00f7eb4da634518a
+github.com/docker/libnetwork 0f08d31bf0e640e0cdc6d5161227f87602d605c5
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -107,7 +108,6 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 github.com/containerd/containerd 06b9cb35161009dcb7123345749fef02f7cea8e0
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 github.com/stevvooe/continuity cd7a8e21e2b6f84799f5dd4b65faf49c8d3ee02d
-github.com/tonistiigi/fsutil 0ac4c11b053b9c5c7c47558f81f96c7100ce50fb
 
 # cluster
 github.com/docker/swarmkit bd7bafb8a61de1f5f23c8215ce7b9ecbcb30ff21

components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go

@@ -57,6 +57,8 @@ func (config *Config) CreateExt4Vhdx(destFile string, sizeGB uint32, cacheFile s
 		return fmt.Errorf("failed to create VHDx %s: %s", destFile, err)
 	}
 
+	defer config.DebugGCS()
+
 	// Attach it to the utility VM, but don't mount it (as there's no filesystem on it)
 	if err := config.HotAddVhd(destFile, "", false, false); err != nil {
 		return fmt.Errorf("opengcs: CreateExt4Vhdx: failed to hot-add %s to utility VM: %s", cacheFile, err)

components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go

@@ -20,6 +20,8 @@ func (config *Config) HotAddVhd(hostPath string, containerPath string, readOnly
 		return fmt.Errorf("cannot hot-add VHD as no utility VM is in configuration")
 	}
 
+	defer config.DebugGCS()
+
 	modification := &hcsshim.ResourceModificationRequestResponse{
 		Resource: "MappedVirtualDisk",
 		Data: hcsshim.MappedVirtualDisk{

components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go

@@ -18,6 +18,8 @@ func (config *Config) HotRemoveVhd(hostPath string) error {
 		return fmt.Errorf("cannot hot-add VHD as no utility VM is in configuration")
 	}
 
+	defer config.DebugGCS()
+
 	modification := &hcsshim.ResourceModificationRequestResponse{
 		Resource: "MappedVirtualDisk",
 		Data: hcsshim.MappedVirtualDisk{

components/engine/vendor/github.com/Microsoft/opengcs/client/process.go

@@ -3,8 +3,12 @@
 package client
 
 import (
+	"bytes"
 	"fmt"
 	"io"
+	"os"
+	"strings"
+	"time"
 
 	"github.com/Microsoft/hcsshim"
 	"github.com/sirupsen/logrus"
@@ -110,3 +114,44 @@ func (config *Config) RunProcess(commandLine string, stdin io.Reader, stdout io.
 	logrus.Debugf("opengcs: runProcess success: %s", commandLine)
 	return process.Process, nil
 }
+
+func debugCommand(s string) string {
+	return fmt.Sprintf(`echo -e 'DEBUG COMMAND: %s\\n--------------\\n';%s;echo -e '\\n\\n';`, s, s)
+}
+
+// DebugGCS extracts logs from the GCS. It's a useful hack for debugging,
+// but not necessarily optimal, but all that is available to us in RS3.
+func (config *Config) DebugGCS() {
+	if logrus.GetLevel() < logrus.DebugLevel || len(os.Getenv("OPENGCS_DEBUG_ENABLE")) == 0 {
+		return
+	}
+
+	var out bytes.Buffer
+	cmd := os.Getenv("OPENGCS_DEBUG_COMMAND")
+	if cmd == "" {
+		cmd = `sh -c "`
+		cmd += debugCommand("ls -l /tmp")
+		cmd += debugCommand("cat /tmp/gcs.log")
+		cmd += debugCommand("ls -l /tmp/gcs")
+		cmd += debugCommand("ls -l /tmp/gcs/*")
+		cmd += debugCommand("cat /tmp/gcs/*/config.json")
+		cmd += debugCommand("ls -lR /var/run/gcsrunc")
+		cmd += debugCommand("cat /var/run/gcsrunc/log.log")
+		cmd += debugCommand("ps -ef")
+		cmd += `"`
+	}
+	proc, err := config.RunProcess(cmd, nil, &out, nil)
+	defer func() {
+		if proc != nil {
+			proc.Kill()
+			proc.Close()
+		}
+	}()
+	if err != nil {
+		logrus.Debugln("benign failure getting gcs logs: ", err)
+	}
+	if proc != nil {
+		proc.WaitTimeout(time.Duration(int(time.Second) * 30))
+	}
+	logrus.Debugf("GCS Debugging:\n%s\n\nEnd GCS Debugging\n", strings.TrimSpace(out.String()))
+}

components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go

@@ -17,6 +17,8 @@ func (config *Config) TarToVhd(targetVHDFile string, reader io.Reader) (int64, e
 		return 0, fmt.Errorf("cannot Tar2Vhd as no utility VM is in configuration")
 	}
 
+	defer config.DebugGCS()
+
 	process, err := config.createUtilsProcess("tar2vhd")
 	if err != nil {
 		return 0, fmt.Errorf("failed to start tar2vhd for %s: %s", targetVHDFile, err)

components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go

@@ -20,6 +20,8 @@ func (config *Config) VhdToTar(vhdFile string, uvmMountPath string, isSandbox bo
 		return nil, fmt.Errorf("cannot VhdToTar as no utility VM is in configuration")
 	}
 
+	defer config.DebugGCS()
+
 	vhdHandle, err := os.Open(vhdFile)
 	if err != nil {
 		return nil, fmt.Errorf("opengcs: VhdToTar: failed to open %s: %s", vhdFile, err)