From 934616e542932a4af3b608db18f5af7db436b138 Mon Sep 17 00:00:00 2001 From: Douglas Curtis Date: Mon, 20 Mar 2017 02:39:40 +0000 Subject: [PATCH 01/34] Replacing os.Lstat with os.Stat to determine directory status in CopyInfoDestinationPath Signed-off-by: Douglas Curtis Commenting out tests for now Signed-off-by: Doug Curtis Added unit test for CopyInfoDestionationPath. Signed-off-by: Doug Curtis Removing integration-cli test case additions Signed-off-by: Doug Curtis Removing extra spaces between archive_unix_test.go test cases Signed-off-by: Doug Curtis Fixed gofmt issues in archive_unix_test.go Signed-off-by: Doug Curtis Upstream-commit: cd7489f2b745578e0d8855aa44213b07b495f86f Component: engine --- .../engine/pkg/archive/archive_unix_test.go | 56 +++++++++++++++++++ components/engine/pkg/archive/changes_test.go | 51 +++++++++-------- components/engine/pkg/archive/copy.go | 2 +- 3 files changed, 84 insertions(+), 25 deletions(-) diff --git a/components/engine/pkg/archive/archive_unix_test.go b/components/engine/pkg/archive/archive_unix_test.go index 2a628f4695..a6d957bfab 100644 --- a/components/engine/pkg/archive/archive_unix_test.go +++ b/components/engine/pkg/archive/archive_unix_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "syscall" "testing" @@ -261,3 +262,58 @@ func TestTarUntarWithXattr(t *testing.T) { } } } + +func TestCopyInfoDestinationPathSymlink(t *testing.T) { + tmpDir, _ := getTestTempDirs(t) + defer removeAllPaths(tmpDir) + + root := strings.TrimRight(tmpDir, "/") + "/" + + type FileTestData struct { + resource FileData + file string + expected CopyInfo + } + + testData := []FileTestData{ + //Create a directory: /tmp/archive-copy-test*/dir1 + //Test will "copy" file1 to dir1 + {resource: FileData{filetype: Dir, path: "dir1", permissions: 0740}, file: "file1", expected: CopyInfo{Path: root + "dir1/file1", Exists: false, IsDir: false}}, + + //Create a symlink directory to dir1: /tmp/archive-copy-test*/dirSymlink -> dir1 + //Test will "copy" file2 to dirSymlink + {resource: FileData{filetype: Symlink, path: "dirSymlink", contents: root + "dir1", permissions: 0600}, file: "file2", expected: CopyInfo{Path: root + "dirSymlink/file2", Exists: false, IsDir: false}}, + + //Create a file in tmp directory: /tmp/archive-copy-test*/file1 + //Test to cover when the full file path already exists. + {resource: FileData{filetype: Regular, path: "file1", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "file1", Exists: true}}, + + //Create a directory: /tmp/archive-copy*/dir2 + //Test to cover when the full directory path already exists + {resource: FileData{filetype: Dir, path: "dir2", permissions: 0740}, file: "", expected: CopyInfo{Path: root + "dir2", Exists: true, IsDir: true}}, + + //Create a symlink to a non-existent target: /tmp/archive-copy*/symlink1 -> noSuchTarget + //Negative test to cover symlinking to a target that does not exit + {resource: FileData{filetype: Symlink, path: "symlink1", contents: "noSuchTarget", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "noSuchTarget", Exists: false}}, + + //Create a file in tmp directory for next test: /tmp/existingfile + {resource: FileData{filetype: Regular, path: "existingfile", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "existingfile", Exists: true}}, + + //Create a symlink to an existing file: /tmp/archive-copy*/symlink2 -> /tmp/existingfile + //Test to cover when the parent directory of a new file is a symlink + {resource: FileData{filetype: Symlink, path: "symlink2", contents: "existingfile", permissions: 0600}, file: "", expected: CopyInfo{Path: root + "existingfile", Exists: true}}, + } + + var dirs []FileData + for _, data := range testData { + dirs = append(dirs, data.resource) + } + provisionSampleDir(t, tmpDir, dirs) + + for _, info := range testData { + p := filepath.Join(tmpDir, info.resource.path, info.file) + ci, err := CopyInfoDestinationPath(p) + assert.NoError(t, err) + assert.Equal(t, info.expected, ci) + } +} diff --git a/components/engine/pkg/archive/changes_test.go b/components/engine/pkg/archive/changes_test.go index 8c14a867ae..b1ab666e1c 100644 --- a/components/engine/pkg/archive/changes_test.go +++ b/components/engine/pkg/archive/changes_test.go @@ -50,32 +50,35 @@ type FileData struct { func createSampleDir(t *testing.T, root string) { files := []FileData{ - {Regular, "file1", "file1\n", 0600}, - {Regular, "file2", "file2\n", 0666}, - {Regular, "file3", "file3\n", 0404}, - {Regular, "file4", "file4\n", 0600}, - {Regular, "file5", "file5\n", 0600}, - {Regular, "file6", "file6\n", 0600}, - {Regular, "file7", "file7\n", 0600}, - {Dir, "dir1", "", 0740}, - {Regular, "dir1/file1-1", "file1-1\n", 01444}, - {Regular, "dir1/file1-2", "file1-2\n", 0666}, - {Dir, "dir2", "", 0700}, - {Regular, "dir2/file2-1", "file2-1\n", 0666}, - {Regular, "dir2/file2-2", "file2-2\n", 0666}, - {Dir, "dir3", "", 0700}, - {Regular, "dir3/file3-1", "file3-1\n", 0666}, - {Regular, "dir3/file3-2", "file3-2\n", 0666}, - {Dir, "dir4", "", 0700}, - {Regular, "dir4/file3-1", "file4-1\n", 0666}, - {Regular, "dir4/file3-2", "file4-2\n", 0666}, - {Symlink, "symlink1", "target1", 0666}, - {Symlink, "symlink2", "target2", 0666}, - {Symlink, "symlink3", root + "/file1", 0666}, - {Symlink, "symlink4", root + "/symlink3", 0666}, - {Symlink, "dirSymlink", root + "/dir1", 0740}, + {filetype: Regular, path: "file1", contents: "file1\n", permissions: 0600}, + {filetype: Regular, path: "file2", contents: "file2\n", permissions: 0666}, + {filetype: Regular, path: "file3", contents: "file3\n", permissions: 0404}, + {filetype: Regular, path: "file4", contents: "file4\n", permissions: 0600}, + {filetype: Regular, path: "file5", contents: "file5\n", permissions: 0600}, + {filetype: Regular, path: "file6", contents: "file6\n", permissions: 0600}, + {filetype: Regular, path: "file7", contents: "file7\n", permissions: 0600}, + {filetype: Dir, path: "dir1", contents: "", permissions: 0740}, + {filetype: Regular, path: "dir1/file1-1", contents: "file1-1\n", permissions: 01444}, + {filetype: Regular, path: "dir1/file1-2", contents: "file1-2\n", permissions: 0666}, + {filetype: Dir, path: "dir2", contents: "", permissions: 0700}, + {filetype: Regular, path: "dir2/file2-1", contents: "file2-1\n", permissions: 0666}, + {filetype: Regular, path: "dir2/file2-2", contents: "file2-2\n", permissions: 0666}, + {filetype: Dir, path: "dir3", contents: "", permissions: 0700}, + {filetype: Regular, path: "dir3/file3-1", contents: "file3-1\n", permissions: 0666}, + {filetype: Regular, path: "dir3/file3-2", contents: "file3-2\n", permissions: 0666}, + {filetype: Dir, path: "dir4", contents: "", permissions: 0700}, + {filetype: Regular, path: "dir4/file3-1", contents: "file4-1\n", permissions: 0666}, + {filetype: Regular, path: "dir4/file3-2", contents: "file4-2\n", permissions: 0666}, + {filetype: Symlink, path: "symlink1", contents: "target1", permissions: 0666}, + {filetype: Symlink, path: "symlink2", contents: "target2", permissions: 0666}, + {filetype: Symlink, path: "symlink3", contents: root + "/file1", permissions: 0666}, + {filetype: Symlink, path: "symlink4", contents: root + "/symlink3", permissions: 0666}, + {filetype: Symlink, path: "dirSymlink", contents: root + "/dir1", permissions: 0740}, } + provisionSampleDir(t, root, files) +} +func provisionSampleDir(t *testing.T, root string, files []FileData) { now := time.Now() for _, info := range files { p := path.Join(root, info.path) diff --git a/components/engine/pkg/archive/copy.go b/components/engine/pkg/archive/copy.go index 3adf8a275c..0fd3e69f8a 100644 --- a/components/engine/pkg/archive/copy.go +++ b/components/engine/pkg/archive/copy.go @@ -218,7 +218,7 @@ func CopyInfoDestinationPath(path string) (info CopyInfo, err error) { // Ensure destination parent dir exists. dstParent, _ := SplitPathDirEntry(path) - parentDirStat, err := os.Lstat(dstParent) + parentDirStat, err := os.Stat(dstParent) if err != nil { return CopyInfo{}, err } From d011c4127f07f7cb621719e073ca824784d94c9c Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 8 Sep 2017 13:35:01 -0700 Subject: [PATCH 02/34] LCOW: Add GCS debugging Signed-off-by: John Howard Upstream-commit: 5a0e2beac330d49c2b7436bf29e87d52dab4f557 Component: engine --- .../engine/libcontainerd/client_windows.go | 5 +++++ .../engine/libcontainerd/container_windows.go | 5 +++++ .../engine/libcontainerd/utils_windows.go | 19 ++++++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/components/engine/libcontainerd/client_windows.go b/components/engine/libcontainerd/client_windows.go index 67ebb5998d..a721b4a8cc 100644 --- a/components/engine/libcontainerd/client_windows.go +++ b/components/engine/libcontainerd/client_windows.go @@ -439,6 +439,11 @@ func (clnt *client) AddProcess(ctx context.Context, containerID, processFriendly if err != nil { return -1, err } + + defer func() { + container.debugGCS() + }() + // Note we always tell HCS to // create stdout as it's required regardless of '-i' or '-t' options, so that // docker can always grab the output through logs. We also tell HCS to always diff --git a/components/engine/libcontainerd/container_windows.go b/components/engine/libcontainerd/container_windows.go index 06f9c82209..5eeb4736f6 100644 --- a/components/engine/libcontainerd/container_windows.go +++ b/components/engine/libcontainerd/container_windows.go @@ -50,6 +50,7 @@ func (ctr *container) start(attachStdio StdioCallback) error { logrus.Debugln("libcontainerd: starting container ", ctr.containerID) if err = ctr.hcsContainer.Start(); err != nil { logrus.Errorf("libcontainerd: failed to start container: %s", err) + ctr.debugGCS() // Before terminating! if err := ctr.terminate(); err != nil { logrus.Errorf("libcontainerd: failed to cleanup after a failed Start. %s", err) } else { @@ -58,6 +59,10 @@ func (ctr *container) start(attachStdio StdioCallback) error { return err } + defer func() { + ctr.debugGCS() + }() + // Note we always tell HCS to // create stdout as it's required regardless of '-i' or '-t' options, so that // docker can always grab the output through logs. We also tell HCS to always diff --git a/components/engine/libcontainerd/utils_windows.go b/components/engine/libcontainerd/utils_windows.go index aa2fe422a6..fc2869b6b4 100644 --- a/components/engine/libcontainerd/utils_windows.go +++ b/components/engine/libcontainerd/utils_windows.go @@ -1,6 +1,10 @@ package libcontainerd -import "strings" +import ( + "strings" + + opengcs "github.com/Microsoft/opengcs/client" +) // setupEnvironmentVariables converts a string array of environment variables // into a map as required by the HCS. Source array is in format [v1=k1] [v2=k2] etc. @@ -19,3 +23,16 @@ func setupEnvironmentVariables(a []string) map[string]string { func (s *LCOWOption) Apply(interface{}) error { return nil } + +// DebugGCS is a dirty hack for debugging for Linux Utility VMs. It simply +// runs a bunch of commands inside the UVM, but seriously aides in advanced debugging. +func (c *container) debugGCS() { + if c == nil || c.isWindows || c.hcsContainer == nil { + return + } + cfg := opengcs.Config{ + Uvm: c.hcsContainer, + UvmTimeoutSeconds: 600, + } + cfg.DebugGCS() +} From 5086fdcfdeb2f9cd24901c4cc211954049f28449 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Fri, 15 Sep 2017 09:53:54 +0200 Subject: [PATCH 03/34] Fix CString memory leaks Make sure to call C.free on C string allocated using C.CString in every exit path. C.CString allocates memory in the C heap using malloc. It is the callers responsibility to free them. See https://golang.org/cmd/cgo/#hdr-Go_references_to_C for details. Signed-off-by: Tobias Klauser Upstream-commit: 593dbfd1448e8dac08488786fde6fe7fb057bdac Component: engine --- components/engine/daemon/graphdriver/driver_solaris.go | 5 ++--- components/engine/daemon/graphdriver/zfs/zfs_solaris.go | 5 ++--- components/engine/pkg/mount/mountinfo_solaris.go | 9 ++++++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/components/engine/daemon/graphdriver/driver_solaris.go b/components/engine/daemon/graphdriver/driver_solaris.go index 121fd9230d..d31aaef8cf 100644 --- a/components/engine/daemon/graphdriver/driver_solaris.go +++ b/components/engine/daemon/graphdriver/driver_solaris.go @@ -81,17 +81,16 @@ func (c *defaultChecker) IsMounted(path string) bool { func Mounted(fsType FsMagic, mountPath string) (bool, error) { cs := C.CString(filepath.Dir(mountPath)) + defer C.free(unsafe.Pointer(cs)) buf := C.getstatfs(cs) + defer C.free(unsafe.Pointer(buf)) // on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ] if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) || (buf.f_basetype[3] != 0) { logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath) - C.free(unsafe.Pointer(buf)) return false, ErrPrerequisites } - C.free(unsafe.Pointer(buf)) - C.free(unsafe.Pointer(cs)) return true, nil } diff --git a/components/engine/daemon/graphdriver/zfs/zfs_solaris.go b/components/engine/daemon/graphdriver/zfs/zfs_solaris.go index d63642252d..ce347f20e0 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs_solaris.go +++ b/components/engine/daemon/graphdriver/zfs/zfs_solaris.go @@ -27,18 +27,17 @@ import ( func checkRootdirFs(rootdir string) error { cs := C.CString(filepath.Dir(rootdir)) + defer C.free(unsafe.Pointer(cs)) buf := C.getstatfs(cs) + defer C.free(unsafe.Pointer(buf)) // on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ] if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) || (buf.f_basetype[3] != 0) { logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) - C.free(unsafe.Pointer(buf)) return graphdriver.ErrPrerequisites } - C.free(unsafe.Pointer(buf)) - C.free(unsafe.Pointer(cs)) return nil } diff --git a/components/engine/pkg/mount/mountinfo_solaris.go b/components/engine/pkg/mount/mountinfo_solaris.go index ad9ab57f8b..069ed8f2de 100644 --- a/components/engine/pkg/mount/mountinfo_solaris.go +++ b/components/engine/pkg/mount/mountinfo_solaris.go @@ -4,16 +4,23 @@ package mount /* #include +#include #include */ import "C" import ( "fmt" + "unsafe" ) func parseMountTable() ([]*Info, error) { - mnttab := C.fopen(C.CString(C.MNTTAB), C.CString("r")) + path := C.CString(C.MNTTAB) + defer C.free(unsafe.Pointer(path)) + mode := C.CString("r") + defer C.free(unsafe.Pointer(mode)) + + mnttab := C.fopen(path, mode) if mnttab == nil { return nil, fmt.Errorf("Failed to open %s", C.MNTTAB) } From e01e198f54dd89f3d0b508cec8ee7af8d1a475e6 Mon Sep 17 00:00:00 2001 From: Simon Ferquel Date: Mon, 22 May 2017 17:21:17 +0200 Subject: [PATCH 04/34] Introduce a typed command system and 2 phase parse/dispatch build This is a work base to introduce more features like build time dockerfile optimisations, dependency analysis and parallel build, as well as a first step to go from a dispatch-inline process to a frontend+backend process. Signed-off-by: Simon Ferquel Upstream-commit: 669c0677980b04bcbf871bb7c2d9f07caccfd42b Component: engine --- .../engine/builder/dockerfile/buildargs.go | 20 + .../engine/builder/dockerfile/builder.go | 217 ++--- .../engine/builder/dockerfile/dispatchers.go | 769 +++++------------- .../builder/dockerfile/dispatchers_test.go | 437 +++++----- .../builder/dockerfile/dispatchers_unix.go | 5 - .../builder/dockerfile/dispatchers_windows.go | 19 - .../engine/builder/dockerfile/evaluator.go | 376 ++++----- .../builder/dockerfile/evaluator_test.go | 157 ++-- .../builder/dockerfile/evaluator_unix.go | 9 - .../builder/dockerfile/evaluator_windows.go | 13 - .../engine/builder/dockerfile/imagecontext.go | 76 -- .../dockerfile/{ => instructions}/bflag.go | 2 +- .../{ => instructions}/bflag_test.go | 2 +- .../dockerfile/instructions/commands.go | 396 +++++++++ .../dockerfile/instructions/errors_unix.go | 9 + .../dockerfile/instructions/errors_windows.go | 27 + .../builder/dockerfile/instructions/parse.go | 635 +++++++++++++++ .../dockerfile/instructions/parse_test.go | 204 +++++ .../dockerfile/{ => instructions}/support.go | 2 +- .../{ => instructions}/support_test.go | 2 +- .../engine/builder/dockerfile/internals.go | 3 - .../integration-cli/docker_api_build_test.go | 76 ++ .../integration-cli/docker_cli_build_test.go | 18 +- 23 files changed, 2095 insertions(+), 1379 deletions(-) delete mode 100644 components/engine/builder/dockerfile/evaluator_unix.go delete mode 100644 components/engine/builder/dockerfile/evaluator_windows.go rename components/engine/builder/dockerfile/{ => instructions}/bflag.go (99%) rename components/engine/builder/dockerfile/{ => instructions}/bflag_test.go (99%) create mode 100644 components/engine/builder/dockerfile/instructions/commands.go create mode 100644 components/engine/builder/dockerfile/instructions/errors_unix.go create mode 100644 components/engine/builder/dockerfile/instructions/errors_windows.go create mode 100644 components/engine/builder/dockerfile/instructions/parse.go create mode 100644 components/engine/builder/dockerfile/instructions/parse_test.go rename components/engine/builder/dockerfile/{ => instructions}/support.go (96%) rename components/engine/builder/dockerfile/{ => instructions}/support_test.go (98%) diff --git a/components/engine/builder/dockerfile/buildargs.go b/components/engine/builder/dockerfile/buildargs.go index e0daf9a77f..c8f34a77a1 100644 --- a/components/engine/builder/dockerfile/buildargs.go +++ b/components/engine/builder/dockerfile/buildargs.go @@ -42,6 +42,26 @@ func newBuildArgs(argsFromOptions map[string]*string) *buildArgs { } } +func (b *buildArgs) Clone() *buildArgs { + result := newBuildArgs(b.argsFromOptions) + for k, v := range b.allowedBuildArgs { + result.allowedBuildArgs[k] = v + } + for k, v := range b.allowedMetaArgs { + result.allowedMetaArgs[k] = v + } + for k := range b.referencedArgs { + result.referencedArgs[k] = struct{}{} + } + return result +} + +func (b *buildArgs) MergeReferencedArgs(other *buildArgs) { + for k := range other.referencedArgs { + b.referencedArgs[k] = struct{}{} + } +} + // WarnOnUnusedBuildArgs checks if there are any leftover build-args that were // passed but not consumed during build. Print a warning, if there are any. func (b *buildArgs) WarnOnUnusedBuildArgs(out io.Writer) { diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index 46a5af7395..27fd4d6208 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -13,7 +13,7 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/dockerfile/command" + "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/fscache" "github.com/docker/docker/builder/remotecontext" @@ -41,6 +41,10 @@ var validCommitCommands = map[string]bool{ "workdir": true, } +const ( + stepFormat = "Step %d/%d : %v" +) + // SessionGetter is object used to get access to a session by uuid type SessionGetter interface { Get(ctx context.Context, uuid string) (session.Caller, error) @@ -176,9 +180,7 @@ type Builder struct { clientCtx context.Context idMappings *idtools.IDMappings - buildStages *buildStages disableCommit bool - buildArgs *buildArgs imageSources *imageSources pathCache pathCache containerManager *containerManager @@ -218,8 +220,6 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder { Output: options.ProgressWriter.Output, docker: options.Backend, idMappings: options.IDMappings, - buildArgs: newBuildArgs(config.BuildArgs), - buildStages: newBuildStages(), imageSources: newImageSources(clientCtx, options), pathCache: options.PathCache, imageProber: newImageProber(options.Backend, config.CacheFrom, options.Platform, config.NoCache), @@ -237,24 +237,27 @@ func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*buil addNodesForLabelOption(dockerfile.AST, b.options.Labels) - if err := checkDispatchDockerfile(dockerfile.AST); err != nil { - buildsFailed.WithValues(metricsDockerfileSyntaxError).Inc() + stages, metaArgs, err := instructions.Parse(dockerfile.AST) + if err != nil { + if instructions.IsUnknownInstruction(err) { + buildsFailed.WithValues(metricsUnknownInstructionError).Inc() + } return nil, validationError{err} } - - dispatchState, err := b.dispatchDockerfileWithCancellation(dockerfile, source) - if err != nil { - return nil, err - } - - if b.options.Target != "" && !dispatchState.isCurrentStage(b.options.Target) { - buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc() - return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target) + if b.options.Target != "" { + targetIx, found := instructions.HasStage(stages, b.options.Target) + if !found { + buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc() + return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target) + } + stages = stages[:targetIx+1] } dockerfile.PrintWarnings(b.Stderr) - b.buildArgs.WarnOnUnusedBuildArgs(b.Stderr) - + dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source) + if err != nil { + return nil, err + } if dispatchState.imageID == "" { buildsFailed.WithValues(metricsDockerfileEmptyError).Inc() return nil, errors.New("No image was generated. Is your Dockerfile empty?") @@ -269,61 +272,91 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error return aux.Emit(types.BuildResult{ID: state.imageID}) } -func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result, source builder.Source) (*dispatchState, error) { - shlex := NewShellLex(dockerfile.EscapeToken) - state := newDispatchState() - total := len(dockerfile.AST.Children) - var err error - for i, n := range dockerfile.AST.Children { - select { - case <-b.clientCtx.Done(): - logrus.Debug("Builder: build cancelled!") - fmt.Fprint(b.Stdout, "Build cancelled") - buildsFailed.WithValues(metricsBuildCanceled).Inc() - return nil, errors.New("Build cancelled") - default: - // Not cancelled yet, keep going... - } +func processMetaArg(meta instructions.ArgCommand, shlex *ShellLex, args *buildArgs) error { + // ShellLex currently only support the concatenated string format + envs := convertMapToEnvList(args.GetAllAllowed()) + if err := meta.Expand(func(word string) (string, error) { + return shlex.ProcessWord(word, envs) + }); err != nil { + return err + } + args.AddArg(meta.Key, meta.Value) + args.AddMetaArg(meta.Key, meta.Value) + return nil +} - // If this is a FROM and we have a previous image then - // emit an aux message for that image since it is the - // end of the previous stage - if n.Value == command.From { - if err := emitImageID(b.Aux, state); err != nil { - return nil, err - } - } +func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int { + fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd) + fmt.Fprintln(out) + return currentCommandIndex + 1 +} - if n.Value == command.From && state.isCurrentStage(b.options.Target) { - break - } +func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) { + dispatchRequest := dispatchRequest{} + buildArgs := newBuildArgs(b.options.BuildArgs) + totalCommands := len(metaArgs) + len(parseResult) + currentCommandIndex := 1 + for _, stage := range parseResult { + totalCommands += len(stage.Commands) + } + shlex := NewShellLex(escapeToken) + for _, meta := range metaArgs { + currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta) - opts := dispatchOptions{ - state: state, - stepMsg: formatStep(i, total), - node: n, - shlex: shlex, - source: source, - } - if state, err = b.dispatch(opts); err != nil { - if b.options.ForceRemove { - b.containerManager.RemoveAll(b.Stdout) - } + err := processMetaArg(meta, shlex, buildArgs) + if err != nil { return nil, err } + } - fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(state.imageID)) - if b.options.Remove { - b.containerManager.RemoveAll(b.Stdout) + stagesResults := newStagesBuildResults() + + for _, stage := range parseResult { + if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil { + return nil, err + } + dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults) + + currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode) + if err := initializeStage(dispatchRequest, &stage); err != nil { + return nil, err + } + dispatchRequest.state.updateRunConfig() + fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID)) + for _, cmd := range stage.Commands { + select { + case <-b.clientCtx.Done(): + logrus.Debug("Builder: build cancelled!") + fmt.Fprint(b.Stdout, "Build cancelled\n") + buildsFailed.WithValues(metricsBuildCanceled).Inc() + return nil, errors.New("Build cancelled") + default: + // Not cancelled yet, keep going... + } + + currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd) + + if err := dispatch(dispatchRequest, cmd); err != nil { + return nil, err + } + + dispatchRequest.state.updateRunConfig() + fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID)) + + } + if err := emitImageID(b.Aux, dispatchRequest.state); err != nil { + return nil, err + } + buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs) + if err := commitStage(dispatchRequest.state, stagesResults); err != nil { + return nil, err } } - - // Emit a final aux message for the final image - if err := emitImageID(b.Aux, state); err != nil { - return nil, err + if b.options.Remove { + b.containerManager.RemoveAll(b.Stdout) } - - return state, nil + buildArgs.WarnOnUnusedBuildArgs(b.Stdout) + return dispatchRequest.state, nil } func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) { @@ -380,39 +413,33 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con b.Stderr = ioutil.Discard b.disableCommit = true - if err := checkDispatchDockerfile(dockerfile.AST); err != nil { - return nil, validationError{err} + commands := []instructions.Command{} + for _, n := range dockerfile.AST.Children { + cmd, err := instructions.ParseCommand(n) + if err != nil { + return nil, validationError{err} + } + commands = append(commands, cmd) } - dispatchState := newDispatchState() - dispatchState.runConfig = config - return dispatchFromDockerfile(b, dockerfile, dispatchState, nil) + + dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, newBuildArgs(b.options.BuildArgs), newStagesBuildResults()) + dispatchRequest.state.runConfig = config + dispatchRequest.state.imageID = config.Image + for _, cmd := range commands { + err := dispatch(dispatchRequest, cmd) + if err != nil { + return nil, validationError{err} + } + dispatchRequest.state.updateRunConfig() + } + + return dispatchRequest.state.runConfig, nil } -func checkDispatchDockerfile(dockerfile *parser.Node) error { - for _, n := range dockerfile.Children { - if err := checkDispatch(n); err != nil { - return errors.Wrapf(err, "Dockerfile parse error line %d", n.StartLine) - } +func convertMapToEnvList(m map[string]string) []string { + result := []string{} + for k, v := range m { + result = append(result, k+"="+v) } - return nil -} - -func dispatchFromDockerfile(b *Builder, result *parser.Result, dispatchState *dispatchState, source builder.Source) (*container.Config, error) { - shlex := NewShellLex(result.EscapeToken) - ast := result.AST - total := len(ast.Children) - - for i, n := range ast.Children { - opts := dispatchOptions{ - state: dispatchState, - stepMsg: formatStep(i, total), - node: n, - shlex: shlex, - source: source, - } - if _, err := b.dispatch(opts); err != nil { - return nil, err - } - } - return dispatchState.runConfig, nil + return result } diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index d8a835802a..76bcb0b2d1 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -10,17 +10,15 @@ package dockerfile import ( "bytes" "fmt" - "regexp" "runtime" "sort" - "strconv" "strings" - "time" "github.com/docker/docker/api" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/builder" + "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/image" "github.com/docker/docker/pkg/jsonmessage" @@ -36,32 +34,14 @@ import ( // Sets the environment variable foo to bar, also makes interpolation // in the dockerfile available from the next statement on via ${foo}. // -func env(req dispatchRequest) error { - if len(req.args) == 0 { - return errAtLeastOneArgument("ENV") - } - - if len(req.args)%2 != 0 { - // should never get here, but just in case - return errTooManyArguments("ENV") - } - - if err := req.flags.Parse(); err != nil { - return err - } - - runConfig := req.state.runConfig +func dispatchEnv(d dispatchRequest, c *instructions.EnvCommand) error { + runConfig := d.state.runConfig commitMessage := bytes.NewBufferString("ENV") + for _, e := range c.Env { + name := e.Key + newVar := e.String() - for j := 0; j < len(req.args); j += 2 { - if len(req.args[j]) == 0 { - return errBlankCommandNames("ENV") - } - name := req.args[j] - value := req.args[j+1] - newVar := name + "=" + value commitMessage.WriteString(" " + newVar) - gotOne := false for i, envVar := range runConfig.Env { envParts := strings.SplitN(envVar, "=", 2) @@ -76,64 +56,32 @@ func env(req dispatchRequest) error { runConfig.Env = append(runConfig.Env, newVar) } } - - return req.builder.commit(req.state, commitMessage.String()) + return d.builder.commit(d.state, commitMessage.String()) } // MAINTAINER some text // // Sets the maintainer metadata. -func maintainer(req dispatchRequest) error { - if len(req.args) != 1 { - return errExactlyOneArgument("MAINTAINER") - } +func dispatchMaintainer(d dispatchRequest, c *instructions.MaintainerCommand) error { - if err := req.flags.Parse(); err != nil { - return err - } - - maintainer := req.args[0] - req.state.maintainer = maintainer - return req.builder.commit(req.state, "MAINTAINER "+maintainer) + d.state.maintainer = c.Maintainer + return d.builder.commit(d.state, "MAINTAINER "+c.Maintainer) } // LABEL some json data describing the image // // Sets the Label variable foo to bar, // -func label(req dispatchRequest) error { - if len(req.args) == 0 { - return errAtLeastOneArgument("LABEL") +func dispatchLabel(d dispatchRequest, c *instructions.LabelCommand) error { + if d.state.runConfig.Labels == nil { + d.state.runConfig.Labels = make(map[string]string) } - if len(req.args)%2 != 0 { - // should never get here, but just in case - return errTooManyArguments("LABEL") - } - - if err := req.flags.Parse(); err != nil { - return err - } - commitStr := "LABEL" - runConfig := req.state.runConfig - - if runConfig.Labels == nil { - runConfig.Labels = map[string]string{} + for _, v := range c.Labels { + d.state.runConfig.Labels[v.Key] = v.Value + commitStr += " " + v.String() } - - for j := 0; j < len(req.args); j++ { - name := req.args[j] - if name == "" { - return errBlankCommandNames("LABEL") - } - - value := req.args[j+1] - commitStr += " " + name + "=" + value - - runConfig.Labels[name] = value - j++ - } - return req.builder.commit(req.state, commitStr) + return d.builder.commit(d.state, commitStr) } // ADD foo /path @@ -141,257 +89,172 @@ func label(req dispatchRequest) error { // Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling // exist here. If you do not wish to have this automatic handling, use COPY. // -func add(req dispatchRequest) error { - if len(req.args) < 2 { - return errAtLeastTwoArguments("ADD") - } - - flChown := req.flags.AddString("chown", "") - if err := req.flags.Parse(); err != nil { - return err - } - - downloader := newRemoteSourceDownloader(req.builder.Output, req.builder.Stdout) - copier := copierFromDispatchRequest(req, downloader, nil) +func dispatchAdd(d dispatchRequest, c *instructions.AddCommand) error { + downloader := newRemoteSourceDownloader(d.builder.Output, d.builder.Stdout) + copier := copierFromDispatchRequest(d, downloader, nil) defer copier.Cleanup() - copyInstruction, err := copier.createCopyInstruction(req.args, "ADD") + + copyInstruction, err := copier.createCopyInstruction(c.SourcesAndDest, "ADD") if err != nil { return err } - copyInstruction.chownStr = flChown.Value + copyInstruction.chownStr = c.Chown copyInstruction.allowLocalDecompression = true - return req.builder.performCopy(req.state, copyInstruction) + return d.builder.performCopy(d.state, copyInstruction) } // COPY foo /path // // Same as 'ADD' but without the tar and remote url handling. // -func dispatchCopy(req dispatchRequest) error { - if len(req.args) < 2 { - return errAtLeastTwoArguments("COPY") +func dispatchCopy(d dispatchRequest, c *instructions.CopyCommand) error { + var im *imageMount + var err error + if c.From != "" { + im, err = d.getImageMount(c.From) + if err != nil { + return errors.Wrapf(err, "invalid from flag value %s", c.From) + } } - - flFrom := req.flags.AddString("from", "") - flChown := req.flags.AddString("chown", "") - if err := req.flags.Parse(); err != nil { - return err - } - - im, err := req.builder.getImageMount(flFrom) - if err != nil { - return errors.Wrapf(err, "invalid from flag value %s", flFrom.Value) - } - - copier := copierFromDispatchRequest(req, errOnSourceDownload, im) + copier := copierFromDispatchRequest(d, errOnSourceDownload, im) defer copier.Cleanup() - copyInstruction, err := copier.createCopyInstruction(req.args, "COPY") + copyInstruction, err := copier.createCopyInstruction(c.SourcesAndDest, "COPY") if err != nil { return err } - copyInstruction.chownStr = flChown.Value + copyInstruction.chownStr = c.Chown - return req.builder.performCopy(req.state, copyInstruction) + return d.builder.performCopy(d.state, copyInstruction) } -func (b *Builder) getImageMount(fromFlag *Flag) (*imageMount, error) { - if !fromFlag.IsUsed() { +func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error) { + if imageRefOrID == "" { // TODO: this could return the source in the default case as well? return nil, nil } var localOnly bool - imageRefOrID := fromFlag.Value - stage, err := b.buildStages.get(fromFlag.Value) + stage, err := d.stages.get(imageRefOrID) if err != nil { return nil, err } if stage != nil { - imageRefOrID = stage.ImageID() + imageRefOrID = stage.Image localOnly = true } - return b.imageSources.Get(imageRefOrID, localOnly) + return d.builder.imageSources.Get(imageRefOrID, localOnly) } // FROM imagename[:tag | @digest] [AS build-stage-name] // -func from(req dispatchRequest) error { - stageName, err := parseBuildStageName(req.args) +func initializeStage(d dispatchRequest, cmd *instructions.Stage) error { + d.builder.imageProber.Reset() + image, err := d.getFromImage(d.shlex, cmd.BaseName) if err != nil { return err } - - if err := req.flags.Parse(); err != nil { - return err + state := d.state + state.beginStage(cmd.Name, image) + if len(state.runConfig.OnBuild) > 0 { + triggers := state.runConfig.OnBuild + state.runConfig.OnBuild = nil + return dispatchTriggeredOnBuild(d, triggers) } - - req.builder.imageProber.Reset() - image, err := req.builder.getFromImage(req.shlex, req.args[0]) - if err != nil { - return err - } - if err := req.builder.buildStages.add(stageName, image); err != nil { - return err - } - req.state.beginStage(stageName, image) - req.builder.buildArgs.ResetAllowed() - if image.ImageID() == "" { - // Typically this means they used "FROM scratch" - return nil - } - - return processOnBuild(req) + return nil } -func parseBuildStageName(args []string) (string, error) { - stageName := "" - switch { - case len(args) == 3 && strings.EqualFold(args[1], "as"): - stageName = strings.ToLower(args[2]) - if ok, _ := regexp.MatchString("^[a-z][a-z0-9-_\\.]*$", stageName); !ok { - return "", errors.Errorf("invalid name for build stage: %q, name can't start with a number or contain symbols", stageName) - } - case len(args) != 1: - return "", errors.New("FROM requires either one or three arguments") +func dispatchTriggeredOnBuild(d dispatchRequest, triggers []string) error { + fmt.Fprintf(d.builder.Stdout, "# Executing %d build trigger", len(triggers)) + if len(triggers) > 1 { + fmt.Fprint(d.builder.Stdout, "s") } - - return stageName, nil -} - -// scratchImage is used as a token for the empty base image. -var scratchImage builder.Image = &image.Image{} - -func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, error) { - substitutionArgs := []string{} - for key, value := range b.buildArgs.GetAllMeta() { - substitutionArgs = append(substitutionArgs, key+"="+value) - } - - name, err := shlex.ProcessWord(name, substitutionArgs) - if err != nil { - return nil, err - } - - var localOnly bool - if stage, ok := b.buildStages.getByName(name); ok { - name = stage.ImageID() - localOnly = true - } - - // Windows cannot support a container with no base image unless it is LCOW. - if name == api.NoBaseImageSpecifier { - if runtime.GOOS == "windows" { - if b.platform == "windows" || (b.platform != "windows" && !system.LCOWSupported()) { - return nil, errors.New("Windows does not support FROM scratch") - } - } - return scratchImage, nil - } - imageMount, err := b.imageSources.Get(name, localOnly) - if err != nil { - return nil, err - } - return imageMount.Image(), nil -} - -func processOnBuild(req dispatchRequest) error { - dispatchState := req.state - // Process ONBUILD triggers if they exist - if nTriggers := len(dispatchState.runConfig.OnBuild); nTriggers != 0 { - word := "trigger" - if nTriggers > 1 { - word = "triggers" - } - fmt.Fprintf(req.builder.Stderr, "# Executing %d build %s...\n", nTriggers, word) - } - - // Copy the ONBUILD triggers, and remove them from the config, since the config will be committed. - onBuildTriggers := dispatchState.runConfig.OnBuild - dispatchState.runConfig.OnBuild = []string{} - - // Reset stdin settings as all build actions run without stdin - dispatchState.runConfig.OpenStdin = false - dispatchState.runConfig.StdinOnce = false - - // parse the ONBUILD triggers by invoking the parser - for _, step := range onBuildTriggers { - dockerfile, err := parser.Parse(strings.NewReader(step)) + fmt.Fprintln(d.builder.Stdout) + for _, trigger := range triggers { + d.state.updateRunConfig() + ast, err := parser.Parse(strings.NewReader(trigger)) if err != nil { return err } - - for _, n := range dockerfile.AST.Children { - if err := checkDispatch(n); err != nil { - return err - } - - upperCasedCmd := strings.ToUpper(n.Value) - switch upperCasedCmd { - case "ONBUILD": - return errors.New("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed") - case "MAINTAINER", "FROM": - return errors.Errorf("%s isn't allowed as an ONBUILD trigger", upperCasedCmd) - } + if len(ast.AST.Children) != 1 { + return errors.New("onbuild trigger should be a single expression") } - - if _, err := dispatchFromDockerfile(req.builder, dockerfile, dispatchState, req.source); err != nil { + cmd, err := instructions.ParseCommand(ast.AST.Children[0]) + if err != nil { + if instructions.IsUnknownInstruction(err) { + buildsFailed.WithValues(metricsUnknownInstructionError).Inc() + } + return err + } + err = dispatch(d, cmd) + if err != nil { return err } } return nil } -// ONBUILD RUN echo yo -// -// ONBUILD triggers run when the image is used in a FROM statement. -// -// ONBUILD handling has a lot of special-case functionality, the heading in -// evaluator.go and comments around dispatch() in the same file explain the -// special cases. search for 'OnBuild' in internals.go for additional special -// cases. -// -func onbuild(req dispatchRequest) error { - if len(req.args) == 0 { - return errAtLeastOneArgument("ONBUILD") +// scratchImage is used as a token for the empty base image. It uses buildStage +// as a convenient implementation of builder.Image, but is not actually a +// buildStage. +var scratchImage builder.Image = &image.Image{} + +func (d *dispatchRequest) getExpandedImageName(shlex *ShellLex, name string) (string, error) { + substitutionArgs := []string{} + for key, value := range d.state.buildArgs.GetAllMeta() { + substitutionArgs = append(substitutionArgs, key+"="+value) } - if err := req.flags.Parse(); err != nil { - return err + name, err := shlex.ProcessWord(name, substitutionArgs) + if err != nil { + return "", err + } + return name, nil +} +func (d *dispatchRequest) getImageOrStage(name string) (builder.Image, error) { + var localOnly bool + if im, ok := d.stages.getByName(name); ok { + name = im.Image + localOnly = true } - triggerInstruction := strings.ToUpper(strings.TrimSpace(req.args[0])) - switch triggerInstruction { - case "ONBUILD": - return errors.New("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed") - case "MAINTAINER", "FROM": - return fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction) + // Windows cannot support a container with no base image unless it is LCOW. + if name == api.NoBaseImageSpecifier { + if runtime.GOOS == "windows" { + if d.builder.platform == "windows" || (d.builder.platform != "windows" && !system.LCOWSupported()) { + return nil, errors.New("Windows does not support FROM scratch") + } + } + return scratchImage, nil } + imageMount, err := d.builder.imageSources.Get(name, localOnly) + if err != nil { + return nil, err + } + return imageMount.Image(), nil +} +func (d *dispatchRequest) getFromImage(shlex *ShellLex, name string) (builder.Image, error) { + name, err := d.getExpandedImageName(shlex, name) + if err != nil { + return nil, err + } + return d.getImageOrStage(name) +} - runConfig := req.state.runConfig - original := regexp.MustCompile(`(?i)^\s*ONBUILD\s*`).ReplaceAllString(req.original, "") - runConfig.OnBuild = append(runConfig.OnBuild, original) - return req.builder.commit(req.state, "ONBUILD "+original) +func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error { + + d.state.runConfig.OnBuild = append(d.state.runConfig.OnBuild, c.Expression) + return d.builder.commit(d.state, "ONBUILD "+c.Expression) } // WORKDIR /tmp // // Set the working directory for future RUN/CMD/etc statements. // -func workdir(req dispatchRequest) error { - if len(req.args) != 1 { - return errExactlyOneArgument("WORKDIR") - } - - err := req.flags.Parse() - if err != nil { - return err - } - - runConfig := req.state.runConfig - // This is from the Dockerfile and will not necessarily be in platform - // specific semantics, hence ensure it is converted. - runConfig.WorkingDir, err = normalizeWorkdir(req.builder.platform, runConfig.WorkingDir, req.args[0]) +func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error { + runConfig := d.state.runConfig + var err error + runConfig.WorkingDir, err = normalizeWorkdir(d.builder.platform, runConfig.WorkingDir, c.Path) if err != nil { return err } @@ -400,23 +263,31 @@ func workdir(req dispatchRequest) error { // This avoids having an unnecessary expensive mount/unmount calls // (on Windows in particular) during each container create. // Prior to 1.13, the mkdir was deferred and not executed at this step. - if req.builder.disableCommit { + if d.builder.disableCommit { // Don't call back into the daemon if we're going through docker commit --change "WORKDIR /foo". // We've already updated the runConfig and that's enough. return nil } comment := "WORKDIR " + runConfig.WorkingDir - runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, req.builder.platform)) - containerID, err := req.builder.probeAndCreate(req.state, runConfigWithCommentCmd) + runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, d.builder.platform)) + containerID, err := d.builder.probeAndCreate(d.state, runConfigWithCommentCmd) if err != nil || containerID == "" { return err } - if err := req.builder.docker.ContainerCreateWorkdir(containerID); err != nil { + if err := d.builder.docker.ContainerCreateWorkdir(containerID); err != nil { return err } - return req.builder.commitContainer(req.state, containerID, runConfigWithCommentCmd) + return d.builder.commitContainer(d.state, containerID, runConfigWithCommentCmd) +} + +func resolveCmdLine(cmd instructions.ShellDependantCmdLine, runConfig *container.Config, platform string) []string { + result := cmd.CmdLine + if cmd.PrependShell && result != nil { + result = append(getShell(runConfig, platform), result...) + } + return result } // RUN some command yo @@ -429,32 +300,21 @@ func workdir(req dispatchRequest) error { // RUN echo hi # cmd /S /C echo hi (Windows) // RUN [ "echo", "hi" ] # echo hi // -func run(req dispatchRequest) error { - if !req.state.hasFromImage() { - return errors.New("Please provide a source image with `from` prior to run") - } +func dispatchRun(d dispatchRequest, c *instructions.RunCommand) error { - if err := req.flags.Parse(); err != nil { - return err - } - - stateRunConfig := req.state.runConfig - args := handleJSONArgs(req.args, req.attributes) - if !req.attributes["json"] { - args = append(getShell(stateRunConfig, req.builder.platform), args...) - } - cmdFromArgs := strslice.StrSlice(args) - buildArgs := req.builder.buildArgs.FilterAllowed(stateRunConfig.Env) + stateRunConfig := d.state.runConfig + cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, d.builder.platform) + buildArgs := d.state.buildArgs.FilterAllowed(stateRunConfig.Env) saveCmd := cmdFromArgs if len(buildArgs) > 0 { - saveCmd = prependEnvOnCmd(req.builder.buildArgs, buildArgs, cmdFromArgs) + saveCmd = prependEnvOnCmd(d.state.buildArgs, buildArgs, cmdFromArgs) } runConfigForCacheProbe := copyRunConfig(stateRunConfig, withCmd(saveCmd), withEntrypointOverride(saveCmd, nil)) - hit, err := req.builder.probeCache(req.state, runConfigForCacheProbe) + hit, err := d.builder.probeCache(d.state, runConfigForCacheProbe) if err != nil || hit { return err } @@ -468,11 +328,11 @@ func run(req dispatchRequest) error { runConfig.ArgsEscaped = true logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd) - cID, err := req.builder.create(runConfig) + cID, err := d.builder.create(runConfig) if err != nil { return err } - if err := req.builder.containerManager.Run(req.builder.clientCtx, cID, req.builder.Stdout, req.builder.Stderr); err != nil { + if err := d.builder.containerManager.Run(d.builder.clientCtx, cID, d.builder.Stdout, d.builder.Stderr); err != nil { if err, ok := err.(*statusCodeError); ok { // TODO: change error type, because jsonmessage.JSONError assumes HTTP return &jsonmessage.JSONError{ @@ -485,7 +345,7 @@ func run(req dispatchRequest) error { return err } - return req.builder.commitContainer(req.state, cID, runConfigForCacheProbe) + return d.builder.commitContainer(d.state, cID, runConfigForCacheProbe) } // Derive the command to use for probeCache() and to commit in this container. @@ -518,139 +378,39 @@ func prependEnvOnCmd(buildArgs *buildArgs, buildArgVars []string, cmd strslice.S // Set the default command to run in the container (which may be empty). // Argument handling is the same as RUN. // -func cmd(req dispatchRequest) error { - if err := req.flags.Parse(); err != nil { - return err - } - - runConfig := req.state.runConfig - cmdSlice := handleJSONArgs(req.args, req.attributes) - if !req.attributes["json"] { - cmdSlice = append(getShell(runConfig, req.builder.platform), cmdSlice...) - } - - runConfig.Cmd = strslice.StrSlice(cmdSlice) +func dispatchCmd(d dispatchRequest, c *instructions.CmdCommand) error { + runConfig := d.state.runConfig + cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.builder.platform) + runConfig.Cmd = cmd // set config as already being escaped, this prevents double escaping on windows runConfig.ArgsEscaped = true - if err := req.builder.commit(req.state, fmt.Sprintf("CMD %q", cmdSlice)); err != nil { + if err := d.builder.commit(d.state, fmt.Sprintf("CMD %q", cmd)); err != nil { return err } - if len(req.args) != 0 { - req.state.cmdSet = true + if len(c.ShellDependantCmdLine.CmdLine) != 0 { + d.state.cmdSet = true } return nil } -// parseOptInterval(flag) is the duration of flag.Value, or 0 if -// empty. An error is reported if the value is given and less than minimum duration. -func parseOptInterval(f *Flag) (time.Duration, error) { - s := f.Value - if s == "" { - return 0, nil - } - d, err := time.ParseDuration(s) - if err != nil { - return 0, err - } - if d < container.MinimumDuration { - return 0, fmt.Errorf("Interval %#v cannot be less than %s", f.name, container.MinimumDuration) - } - return d, nil -} - // HEALTHCHECK foo // // Set the default healthcheck command to run in the container (which may be empty). // Argument handling is the same as RUN. // -func healthcheck(req dispatchRequest) error { - if len(req.args) == 0 { - return errAtLeastOneArgument("HEALTHCHECK") +func dispatchHealthcheck(d dispatchRequest, c *instructions.HealthCheckCommand) error { + runConfig := d.state.runConfig + if runConfig.Healthcheck != nil { + oldCmd := runConfig.Healthcheck.Test + if len(oldCmd) > 0 && oldCmd[0] != "NONE" { + fmt.Fprintf(d.builder.Stdout, "Note: overriding previous HEALTHCHECK: %v\n", oldCmd) + } } - runConfig := req.state.runConfig - typ := strings.ToUpper(req.args[0]) - args := req.args[1:] - if typ == "NONE" { - if len(args) != 0 { - return errors.New("HEALTHCHECK NONE takes no arguments") - } - test := strslice.StrSlice{typ} - runConfig.Healthcheck = &container.HealthConfig{ - Test: test, - } - } else { - if runConfig.Healthcheck != nil { - oldCmd := runConfig.Healthcheck.Test - if len(oldCmd) > 0 && oldCmd[0] != "NONE" { - fmt.Fprintf(req.builder.Stdout, "Note: overriding previous HEALTHCHECK: %v\n", oldCmd) - } - } - - healthcheck := container.HealthConfig{} - - flInterval := req.flags.AddString("interval", "") - flTimeout := req.flags.AddString("timeout", "") - flStartPeriod := req.flags.AddString("start-period", "") - flRetries := req.flags.AddString("retries", "") - - if err := req.flags.Parse(); err != nil { - return err - } - - switch typ { - case "CMD": - cmdSlice := handleJSONArgs(args, req.attributes) - if len(cmdSlice) == 0 { - return errors.New("Missing command after HEALTHCHECK CMD") - } - - if !req.attributes["json"] { - typ = "CMD-SHELL" - } - - healthcheck.Test = strslice.StrSlice(append([]string{typ}, cmdSlice...)) - default: - return fmt.Errorf("Unknown type %#v in HEALTHCHECK (try CMD)", typ) - } - - interval, err := parseOptInterval(flInterval) - if err != nil { - return err - } - healthcheck.Interval = interval - - timeout, err := parseOptInterval(flTimeout) - if err != nil { - return err - } - healthcheck.Timeout = timeout - - startPeriod, err := parseOptInterval(flStartPeriod) - if err != nil { - return err - } - healthcheck.StartPeriod = startPeriod - - if flRetries.Value != "" { - retries, err := strconv.ParseInt(flRetries.Value, 10, 32) - if err != nil { - return err - } - if retries < 1 { - return fmt.Errorf("--retries must be at least 1 (not %d)", retries) - } - healthcheck.Retries = int(retries) - } else { - healthcheck.Retries = 0 - } - - runConfig.Healthcheck = &healthcheck - } - - return req.builder.commit(req.state, fmt.Sprintf("HEALTHCHECK %q", runConfig.Healthcheck)) + runConfig.Healthcheck = c.Health + return d.builder.commit(d.state, fmt.Sprintf("HEALTHCHECK %q", runConfig.Healthcheck)) } // ENTRYPOINT /usr/sbin/nginx @@ -661,33 +421,15 @@ func healthcheck(req dispatchRequest) error { // Handles command processing similar to CMD and RUN, only req.runConfig.Entrypoint // is initialized at newBuilder time instead of through argument parsing. // -func entrypoint(req dispatchRequest) error { - if err := req.flags.Parse(); err != nil { - return err - } - - runConfig := req.state.runConfig - parsed := handleJSONArgs(req.args, req.attributes) - - switch { - case req.attributes["json"]: - // ENTRYPOINT ["echo", "hi"] - runConfig.Entrypoint = strslice.StrSlice(parsed) - case len(parsed) == 0: - // ENTRYPOINT [] - runConfig.Entrypoint = nil - default: - // ENTRYPOINT echo hi - runConfig.Entrypoint = strslice.StrSlice(append(getShell(runConfig, req.builder.platform), parsed[0])) - } - - // when setting the entrypoint if a CMD was not explicitly set then - // set the command to nil - if !req.state.cmdSet { +func dispatchEntrypoint(d dispatchRequest, c *instructions.EntrypointCommand) error { + runConfig := d.state.runConfig + cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.builder.platform) + runConfig.Entrypoint = cmd + if !d.state.cmdSet { runConfig.Cmd = nil } - return req.builder.commit(req.state, fmt.Sprintf("ENTRYPOINT %q", runConfig.Entrypoint)) + return d.builder.commit(d.state, fmt.Sprintf("ENTRYPOINT %q", runConfig.Entrypoint)) } // EXPOSE 6667/tcp 7000/tcp @@ -695,41 +437,33 @@ func entrypoint(req dispatchRequest) error { // Expose ports for links and port mappings. This all ends up in // req.runConfig.ExposedPorts for runconfig. // -func expose(req dispatchRequest) error { - portsTab := req.args - - if len(req.args) == 0 { - return errAtLeastOneArgument("EXPOSE") +func dispatchExpose(d dispatchRequest, c *instructions.ExposeCommand, envs []string) error { + // custom multi word expansion + // expose $FOO with FOO="80 443" is expanded as EXPOSE [80,443]. This is the only command supporting word to words expansion + // so the word processing has been de-generalized + ports := []string{} + for _, p := range c.Ports { + ps, err := d.shlex.ProcessWords(p, envs) + if err != nil { + return err + } + ports = append(ports, ps...) } + c.Ports = ports - if err := req.flags.Parse(); err != nil { - return err - } - - runConfig := req.state.runConfig - if runConfig.ExposedPorts == nil { - runConfig.ExposedPorts = make(nat.PortSet) - } - - ports, _, err := nat.ParsePortSpecs(portsTab) + ps, _, err := nat.ParsePortSpecs(ports) if err != nil { return err } - // instead of using ports directly, we build a list of ports and sort it so - // the order is consistent. This prevents cache burst where map ordering - // changes between builds - portList := make([]string, len(ports)) - var i int - for port := range ports { - if _, exists := runConfig.ExposedPorts[port]; !exists { - runConfig.ExposedPorts[port] = struct{}{} - } - portList[i] = string(port) - i++ + if d.state.runConfig.ExposedPorts == nil { + d.state.runConfig.ExposedPorts = make(nat.PortSet) } - sort.Strings(portList) - return req.builder.commit(req.state, "EXPOSE "+strings.Join(portList, " ")) + for p := range ps { + d.state.runConfig.ExposedPorts[p] = struct{}{} + } + + return d.builder.commit(d.state, "EXPOSE "+strings.Join(c.Ports, " ")) } // USER foo @@ -737,62 +471,39 @@ func expose(req dispatchRequest) error { // Set the user to 'foo' for future commands and when running the // ENTRYPOINT/CMD at container run time. // -func user(req dispatchRequest) error { - if len(req.args) != 1 { - return errExactlyOneArgument("USER") - } - - if err := req.flags.Parse(); err != nil { - return err - } - - req.state.runConfig.User = req.args[0] - return req.builder.commit(req.state, fmt.Sprintf("USER %v", req.args)) +func dispatchUser(d dispatchRequest, c *instructions.UserCommand) error { + d.state.runConfig.User = c.User + return d.builder.commit(d.state, fmt.Sprintf("USER %v", c.User)) } // VOLUME /foo // // Expose the volume /foo for use. Will also accept the JSON array form. // -func volume(req dispatchRequest) error { - if len(req.args) == 0 { - return errAtLeastOneArgument("VOLUME") +func dispatchVolume(d dispatchRequest, c *instructions.VolumeCommand) error { + if d.state.runConfig.Volumes == nil { + d.state.runConfig.Volumes = map[string]struct{}{} } - - if err := req.flags.Parse(); err != nil { - return err - } - - runConfig := req.state.runConfig - if runConfig.Volumes == nil { - runConfig.Volumes = map[string]struct{}{} - } - for _, v := range req.args { - v = strings.TrimSpace(v) + for _, v := range c.Volumes { if v == "" { return errors.New("VOLUME specified can not be an empty string") } - runConfig.Volumes[v] = struct{}{} + d.state.runConfig.Volumes[v] = struct{}{} } - return req.builder.commit(req.state, fmt.Sprintf("VOLUME %v", req.args)) + return d.builder.commit(d.state, fmt.Sprintf("VOLUME %v", c.Volumes)) } // STOPSIGNAL signal // // Set the signal that will be used to kill the container. -func stopSignal(req dispatchRequest) error { - if len(req.args) != 1 { - return errExactlyOneArgument("STOPSIGNAL") - } +func dispatchStopSignal(d dispatchRequest, c *instructions.StopSignalCommand) error { - sig := req.args[0] - _, err := signal.ParseSignal(sig) + _, err := signal.ParseSignal(c.Signal) if err != nil { return validationError{err} } - - req.state.runConfig.StopSignal = sig - return req.builder.commit(req.state, fmt.Sprintf("STOPSIGNAL %v", req.args)) + d.state.runConfig.StopSignal = c.Signal + return d.builder.commit(d.state, fmt.Sprintf("STOPSIGNAL %v", c.Signal)) } // ARG name[=value] @@ -800,89 +511,21 @@ func stopSignal(req dispatchRequest) error { // Adds the variable foo to the trusted list of variables that can be passed // to builder using the --build-arg flag for expansion/substitution or passing to 'run'. // Dockerfile author may optionally set a default value of this variable. -func arg(req dispatchRequest) error { - if len(req.args) != 1 { - return errExactlyOneArgument("ARG") +func dispatchArg(d dispatchRequest, c *instructions.ArgCommand) error { + + commitStr := "ARG " + c.Key + if c.Value != nil { + commitStr += "=" + *c.Value } - var ( - name string - newValue string - hasDefault bool - ) - - arg := req.args[0] - // 'arg' can just be a name or name-value pair. Note that this is different - // from 'env' that handles the split of name and value at the parser level. - // The reason for doing it differently for 'arg' is that we support just - // defining an arg and not assign it a value (while 'env' always expects a - // name-value pair). If possible, it will be good to harmonize the two. - if strings.Contains(arg, "=") { - parts := strings.SplitN(arg, "=", 2) - if len(parts[0]) == 0 { - return errBlankCommandNames("ARG") - } - - name = parts[0] - newValue = parts[1] - hasDefault = true - } else { - name = arg - hasDefault = false - } - - var value *string - if hasDefault { - value = &newValue - } - req.builder.buildArgs.AddArg(name, value) - - // Arg before FROM doesn't add a layer - if !req.state.hasFromImage() { - req.builder.buildArgs.AddMetaArg(name, value) - return nil - } - return req.builder.commit(req.state, "ARG "+arg) + d.state.buildArgs.AddArg(c.Key, c.Value) + return d.builder.commit(d.state, commitStr) } // SHELL powershell -command // // Set the non-default shell to use. -func shell(req dispatchRequest) error { - if err := req.flags.Parse(); err != nil { - return err - } - shellSlice := handleJSONArgs(req.args, req.attributes) - switch { - case len(shellSlice) == 0: - // SHELL [] - return errAtLeastOneArgument("SHELL") - case req.attributes["json"]: - // SHELL ["powershell", "-command"] - req.state.runConfig.Shell = strslice.StrSlice(shellSlice) - default: - // SHELL powershell -command - not JSON - return errNotJSON("SHELL", req.original) - } - return req.builder.commit(req.state, fmt.Sprintf("SHELL %v", shellSlice)) -} - -func errAtLeastOneArgument(command string) error { - return fmt.Errorf("%s requires at least one argument", command) -} - -func errExactlyOneArgument(command string) error { - return fmt.Errorf("%s requires exactly one argument", command) -} - -func errAtLeastTwoArguments(command string) error { - return fmt.Errorf("%s requires at least two arguments", command) -} - -func errBlankCommandNames(command string) error { - return fmt.Errorf("%s names can not be blank", command) -} - -func errTooManyArguments(command string) error { - return fmt.Errorf("Bad input to %s, too many arguments", command) +func dispatchShell(d dispatchRequest, c *instructions.ShellCommand) error { + d.state.runConfig.Shell = c.Shell + return d.builder.commit(d.state, fmt.Sprintf("SHELL %v", d.state.runConfig.Shell)) } diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index fb1df83b83..f7f1b1e310 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -1,60 +1,29 @@ package dockerfile import ( - "fmt" - "runtime" - "testing" - "bytes" "context" + "runtime" + "testing" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/dockerfile/parser" - "github.com/docker/docker/internal/testutil" + "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/pkg/system" "github.com/docker/go-connections/nat" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type commandWithFunction struct { - name string - function func(args []string) error -} - -func withArgs(f dispatcher) func([]string) error { - return func(args []string) error { - return f(dispatchRequest{args: args}) - } -} - -func withBuilderAndArgs(builder *Builder, f dispatcher) func([]string) error { - return func(args []string) error { - return f(defaultDispatchReq(builder, args...)) - } -} - -func defaultDispatchReq(builder *Builder, args ...string) dispatchRequest { - return dispatchRequest{ - builder: builder, - args: args, - flags: NewBFlags(), - shlex: NewShellLex(parser.DefaultEscapeToken), - state: &dispatchState{runConfig: &container.Config{}}, - } -} - func newBuilderWithMockBackend() *Builder { mockBackend := &MockBackend{} ctx := context.Background() b := &Builder{ options: &types.ImageBuildOptions{}, docker: mockBackend, - buildArgs: newBuildArgs(make(map[string]*string)), Stdout: new(bytes.Buffer), clientCtx: ctx, disableCommit: true, @@ -62,137 +31,84 @@ func newBuilderWithMockBackend() *Builder { Options: &types.ImageBuildOptions{}, Backend: mockBackend, }), - buildStages: newBuildStages(), imageProber: newImageProber(mockBackend, nil, runtime.GOOS, false), containerManager: newContainerManager(mockBackend), } return b } -func TestCommandsExactlyOneArgument(t *testing.T) { - commands := []commandWithFunction{ - {"MAINTAINER", withArgs(maintainer)}, - {"WORKDIR", withArgs(workdir)}, - {"USER", withArgs(user)}, - {"STOPSIGNAL", withArgs(stopSignal)}, - } - - for _, command := range commands { - err := command.function([]string{}) - assert.EqualError(t, err, errExactlyOneArgument(command.name).Error()) - } -} - -func TestCommandsAtLeastOneArgument(t *testing.T) { - commands := []commandWithFunction{ - {"ENV", withArgs(env)}, - {"LABEL", withArgs(label)}, - {"ONBUILD", withArgs(onbuild)}, - {"HEALTHCHECK", withArgs(healthcheck)}, - {"EXPOSE", withArgs(expose)}, - {"VOLUME", withArgs(volume)}, - } - - for _, command := range commands { - err := command.function([]string{}) - assert.EqualError(t, err, errAtLeastOneArgument(command.name).Error()) - } -} - -func TestCommandsAtLeastTwoArguments(t *testing.T) { - commands := []commandWithFunction{ - {"ADD", withArgs(add)}, - {"COPY", withArgs(dispatchCopy)}} - - for _, command := range commands { - err := command.function([]string{"arg1"}) - assert.EqualError(t, err, errAtLeastTwoArguments(command.name).Error()) - } -} - -func TestCommandsTooManyArguments(t *testing.T) { - commands := []commandWithFunction{ - {"ENV", withArgs(env)}, - {"LABEL", withArgs(label)}} - - for _, command := range commands { - err := command.function([]string{"arg1", "arg2", "arg3"}) - assert.EqualError(t, err, errTooManyArguments(command.name).Error()) - } -} - -func TestCommandsBlankNames(t *testing.T) { - builder := newBuilderWithMockBackend() - commands := []commandWithFunction{ - {"ENV", withBuilderAndArgs(builder, env)}, - {"LABEL", withBuilderAndArgs(builder, label)}, - } - - for _, command := range commands { - err := command.function([]string{"", ""}) - assert.EqualError(t, err, errBlankCommandNames(command.name).Error()) - } -} - func TestEnv2Variables(t *testing.T) { b := newBuilderWithMockBackend() - - args := []string{"var1", "val1", "var2", "val2"} - req := defaultDispatchReq(b, args...) - err := env(req) + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + envCommand := &instructions.EnvCommand{ + Env: instructions.KeyValuePairs{ + instructions.KeyValuePair{Key: "var1", Value: "val1"}, + instructions.KeyValuePair{Key: "var2", Value: "val2"}, + }, + } + err := dispatch(sb, envCommand) require.NoError(t, err) expected := []string{ - fmt.Sprintf("%s=%s", args[0], args[1]), - fmt.Sprintf("%s=%s", args[2], args[3]), + "var1=val1", + "var2=val2", } - assert.Equal(t, expected, req.state.runConfig.Env) + assert.Equal(t, expected, sb.state.runConfig.Env) } func TestEnvValueWithExistingRunConfigEnv(t *testing.T) { b := newBuilderWithMockBackend() - - args := []string{"var1", "val1"} - req := defaultDispatchReq(b, args...) - req.state.runConfig.Env = []string{"var1=old", "var2=fromenv"} - err := env(req) + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + sb.state.runConfig.Env = []string{"var1=old", "var2=fromenv"} + envCommand := &instructions.EnvCommand{ + Env: instructions.KeyValuePairs{ + instructions.KeyValuePair{Key: "var1", Value: "val1"}, + }, + } + err := dispatch(sb, envCommand) require.NoError(t, err) - expected := []string{ - fmt.Sprintf("%s=%s", args[0], args[1]), + "var1=val1", "var2=fromenv", } - assert.Equal(t, expected, req.state.runConfig.Env) + assert.Equal(t, expected, sb.state.runConfig.Env) } func TestMaintainer(t *testing.T) { maintainerEntry := "Some Maintainer " - b := newBuilderWithMockBackend() - req := defaultDispatchReq(b, maintainerEntry) - err := maintainer(req) + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + cmd := &instructions.MaintainerCommand{Maintainer: maintainerEntry} + err := dispatch(sb, cmd) require.NoError(t, err) - assert.Equal(t, maintainerEntry, req.state.maintainer) + assert.Equal(t, maintainerEntry, sb.state.maintainer) } func TestLabel(t *testing.T) { labelName := "label" labelValue := "value" - labelEntry := []string{labelName, labelValue} b := newBuilderWithMockBackend() - req := defaultDispatchReq(b, labelEntry...) - err := label(req) + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + cmd := &instructions.LabelCommand{ + Labels: instructions.KeyValuePairs{ + instructions.KeyValuePair{Key: labelName, Value: labelValue}, + }, + } + err := dispatch(sb, cmd) require.NoError(t, err) - require.Contains(t, req.state.runConfig.Labels, labelName) - assert.Equal(t, req.state.runConfig.Labels[labelName], labelValue) + require.Contains(t, sb.state.runConfig.Labels, labelName) + assert.Equal(t, sb.state.runConfig.Labels[labelName], labelValue) } func TestFromScratch(t *testing.T) { b := newBuilderWithMockBackend() - req := defaultDispatchReq(b, "scratch") - err := from(req) + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + cmd := &instructions.Stage{ + BaseName: "scratch", + } + err := initializeStage(sb, cmd) if runtime.GOOS == "windows" && !system.LCOWSupported() { assert.EqualError(t, err, "Windows does not support FROM scratch") @@ -200,14 +116,14 @@ func TestFromScratch(t *testing.T) { } require.NoError(t, err) - assert.True(t, req.state.hasFromImage()) - assert.Equal(t, "", req.state.imageID) + assert.True(t, sb.state.hasFromImage()) + assert.Equal(t, "", sb.state.imageID) // Windows does not set the default path. TODO @jhowardmsft LCOW support. This will need revisiting as we get further into the implementation expected := "PATH=" + system.DefaultPathEnv(runtime.GOOS) if runtime.GOOS == "windows" { expected = "" } - assert.Equal(t, []string{expected}, req.state.runConfig.Env) + assert.Equal(t, []string{expected}, sb.state.runConfig.Env) } func TestFromWithArg(t *testing.T) { @@ -219,16 +135,27 @@ func TestFromWithArg(t *testing.T) { } b := newBuilderWithMockBackend() b.docker.(*MockBackend).getImageFunc = getImage + args := newBuildArgs(make(map[string]*string)) - require.NoError(t, arg(defaultDispatchReq(b, "THETAG="+tag))) - req := defaultDispatchReq(b, "alpine${THETAG}") - err := from(req) + val := "sometag" + metaArg := instructions.ArgCommand{ + Key: "THETAG", + Value: &val, + } + cmd := &instructions.Stage{ + BaseName: "alpine:${THETAG}", + } + err := processMetaArg(metaArg, NewShellLex('\\'), args) + sb := newDispatchRequest(b, '\\', nil, args, newStagesBuildResults()) require.NoError(t, err) - assert.Equal(t, expected, req.state.imageID) - assert.Equal(t, expected, req.state.baseImage.ImageID()) - assert.Len(t, b.buildArgs.GetAllAllowed(), 0) - assert.Len(t, b.buildArgs.GetAllMeta(), 1) + err = initializeStage(sb, cmd) + require.NoError(t, err) + + assert.Equal(t, expected, sb.state.imageID) + assert.Equal(t, expected, sb.state.baseImage.ImageID()) + assert.Len(t, sb.state.buildArgs.GetAllAllowed(), 0) + assert.Len(t, sb.state.buildArgs.GetAllMeta(), 1) } func TestFromWithUndefinedArg(t *testing.T) { @@ -240,74 +167,74 @@ func TestFromWithUndefinedArg(t *testing.T) { } b := newBuilderWithMockBackend() b.docker.(*MockBackend).getImageFunc = getImage + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + b.options.BuildArgs = map[string]*string{"THETAG": &tag} - req := defaultDispatchReq(b, "alpine${THETAG}") - err := from(req) + cmd := &instructions.Stage{ + BaseName: "alpine${THETAG}", + } + err := initializeStage(sb, cmd) require.NoError(t, err) - assert.Equal(t, expected, req.state.imageID) + assert.Equal(t, expected, sb.state.imageID) } -func TestFromMultiStageWithScratchNamedStage(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows does not support scratch") - } +func TestFromMultiStageWithNamedStage(t *testing.T) { b := newBuilderWithMockBackend() - req := defaultDispatchReq(b, "scratch", "AS", "base") - - require.NoError(t, from(req)) - assert.True(t, req.state.hasFromImage()) - - req.args = []string{"base"} - require.NoError(t, from(req)) - assert.True(t, req.state.hasFromImage()) -} - -func TestOnbuildIllegalTriggers(t *testing.T) { - triggers := []struct{ command, expectedError string }{ - {"ONBUILD", "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed"}, - {"MAINTAINER", "MAINTAINER isn't allowed as an ONBUILD trigger"}, - {"FROM", "FROM isn't allowed as an ONBUILD trigger"}} - - for _, trigger := range triggers { - b := newBuilderWithMockBackend() - - err := onbuild(defaultDispatchReq(b, trigger.command)) - testutil.ErrorContains(t, err, trigger.expectedError) - } + firstFrom := &instructions.Stage{BaseName: "someimg", Name: "base"} + secondFrom := &instructions.Stage{BaseName: "base"} + previousResults := newStagesBuildResults() + firstSB := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), previousResults) + secondSB := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), previousResults) + err := initializeStage(firstSB, firstFrom) + require.NoError(t, err) + assert.True(t, firstSB.state.hasFromImage()) + previousResults.indexed["base"] = firstSB.state.runConfig + previousResults.flat = append(previousResults.flat, firstSB.state.runConfig) + err = initializeStage(secondSB, secondFrom) + require.NoError(t, err) + assert.True(t, secondSB.state.hasFromImage()) } func TestOnbuild(t *testing.T) { b := newBuilderWithMockBackend() - - req := defaultDispatchReq(b, "ADD", ".", "/app/src") - req.original = "ONBUILD ADD . /app/src" - req.state.runConfig = &container.Config{} - - err := onbuild(req) + sb := newDispatchRequest(b, '\\', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + cmd := &instructions.OnbuildCommand{ + Expression: "ADD . /app/src", + } + err := dispatch(sb, cmd) require.NoError(t, err) - assert.Equal(t, "ADD . /app/src", req.state.runConfig.OnBuild[0]) + assert.Equal(t, "ADD . /app/src", sb.state.runConfig.OnBuild[0]) } func TestWorkdir(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) workingDir := "/app" if runtime.GOOS == "windows" { - workingDir = "C:\app" + workingDir = "C:\\app" + } + cmd := &instructions.WorkdirCommand{ + Path: workingDir, } - req := defaultDispatchReq(b, workingDir) - err := workdir(req) + err := dispatch(sb, cmd) require.NoError(t, err) - assert.Equal(t, workingDir, req.state.runConfig.WorkingDir) + assert.Equal(t, workingDir, sb.state.runConfig.WorkingDir) } func TestCmd(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) command := "./executable" - req := defaultDispatchReq(b, command) - err := cmd(req) + cmd := &instructions.CmdCommand{ + ShellDependantCmdLine: instructions.ShellDependantCmdLine{ + CmdLine: strslice.StrSlice{command}, + PrependShell: true, + }, + } + err := dispatch(sb, cmd) require.NoError(t, err) var expectedCommand strslice.StrSlice @@ -317,42 +244,56 @@ func TestCmd(t *testing.T) { expectedCommand = strslice.StrSlice(append([]string{"/bin/sh"}, "-c", command)) } - assert.Equal(t, expectedCommand, req.state.runConfig.Cmd) - assert.True(t, req.state.cmdSet) + assert.Equal(t, expectedCommand, sb.state.runConfig.Cmd) + assert.True(t, sb.state.cmdSet) } func TestHealthcheckNone(t *testing.T) { b := newBuilderWithMockBackend() - - req := defaultDispatchReq(b, "NONE") - err := healthcheck(req) + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + cmd := &instructions.HealthCheckCommand{ + Health: &container.HealthConfig{ + Test: []string{"NONE"}, + }, + } + err := dispatch(sb, cmd) require.NoError(t, err) - require.NotNil(t, req.state.runConfig.Healthcheck) - assert.Equal(t, []string{"NONE"}, req.state.runConfig.Healthcheck.Test) + require.NotNil(t, sb.state.runConfig.Healthcheck) + assert.Equal(t, []string{"NONE"}, sb.state.runConfig.Healthcheck.Test) } func TestHealthcheckCmd(t *testing.T) { - b := newBuilderWithMockBackend() - args := []string{"CMD", "curl", "-f", "http://localhost/", "||", "exit", "1"} - req := defaultDispatchReq(b, args...) - err := healthcheck(req) + b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + expectedTest := []string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"} + cmd := &instructions.HealthCheckCommand{ + Health: &container.HealthConfig{ + Test: expectedTest, + }, + } + err := dispatch(sb, cmd) require.NoError(t, err) - require.NotNil(t, req.state.runConfig.Healthcheck) - expectedTest := []string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"} - assert.Equal(t, expectedTest, req.state.runConfig.Healthcheck.Test) + require.NotNil(t, sb.state.runConfig.Healthcheck) + assert.Equal(t, expectedTest, sb.state.runConfig.Healthcheck.Test) } func TestEntrypoint(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) entrypointCmd := "/usr/sbin/nginx" - req := defaultDispatchReq(b, entrypointCmd) - err := entrypoint(req) + cmd := &instructions.EntrypointCommand{ + ShellDependantCmdLine: instructions.ShellDependantCmdLine{ + CmdLine: strslice.StrSlice{entrypointCmd}, + PrependShell: true, + }, + } + err := dispatch(sb, cmd) require.NoError(t, err) - require.NotNil(t, req.state.runConfig.Entrypoint) + require.NotNil(t, sb.state.runConfig.Entrypoint) var expectedEntrypoint strslice.StrSlice if runtime.GOOS == "windows" { @@ -360,99 +301,99 @@ func TestEntrypoint(t *testing.T) { } else { expectedEntrypoint = strslice.StrSlice(append([]string{"/bin/sh"}, "-c", entrypointCmd)) } - assert.Equal(t, expectedEntrypoint, req.state.runConfig.Entrypoint) + assert.Equal(t, expectedEntrypoint, sb.state.runConfig.Entrypoint) } func TestExpose(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) exposedPort := "80" - req := defaultDispatchReq(b, exposedPort) - err := expose(req) + cmd := &instructions.ExposeCommand{ + Ports: []string{exposedPort}, + } + err := dispatch(sb, cmd) require.NoError(t, err) - require.NotNil(t, req.state.runConfig.ExposedPorts) - require.Len(t, req.state.runConfig.ExposedPorts, 1) + require.NotNil(t, sb.state.runConfig.ExposedPorts) + require.Len(t, sb.state.runConfig.ExposedPorts, 1) portsMapping, err := nat.ParsePortSpec(exposedPort) require.NoError(t, err) - assert.Contains(t, req.state.runConfig.ExposedPorts, portsMapping[0].Port) + assert.Contains(t, sb.state.runConfig.ExposedPorts, portsMapping[0].Port) } func TestUser(t *testing.T) { b := newBuilderWithMockBackend() - userCommand := "foo" + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) - req := defaultDispatchReq(b, userCommand) - err := user(req) + cmd := &instructions.UserCommand{ + User: "test", + } + err := dispatch(sb, cmd) require.NoError(t, err) - assert.Equal(t, userCommand, req.state.runConfig.User) + assert.Equal(t, "test", sb.state.runConfig.User) } func TestVolume(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) exposedVolume := "/foo" - req := defaultDispatchReq(b, exposedVolume) - err := volume(req) + cmd := &instructions.VolumeCommand{ + Volumes: []string{exposedVolume}, + } + err := dispatch(sb, cmd) require.NoError(t, err) - - require.NotNil(t, req.state.runConfig.Volumes) - assert.Len(t, req.state.runConfig.Volumes, 1) - assert.Contains(t, req.state.runConfig.Volumes, exposedVolume) + require.NotNil(t, sb.state.runConfig.Volumes) + assert.Len(t, sb.state.runConfig.Volumes, 1) + assert.Contains(t, sb.state.runConfig.Volumes, exposedVolume) } func TestStopSignal(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not support stopsignal") + return + } b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) signal := "SIGKILL" - req := defaultDispatchReq(b, signal) - err := stopSignal(req) + cmd := &instructions.StopSignalCommand{ + Signal: signal, + } + err := dispatch(sb, cmd) require.NoError(t, err) - assert.Equal(t, signal, req.state.runConfig.StopSignal) + assert.Equal(t, signal, sb.state.runConfig.StopSignal) } func TestArg(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) argName := "foo" argVal := "bar" - argDef := fmt.Sprintf("%s=%s", argName, argVal) - - err := arg(defaultDispatchReq(b, argDef)) + cmd := &instructions.ArgCommand{Key: argName, Value: &argVal} + err := dispatch(sb, cmd) require.NoError(t, err) expected := map[string]string{argName: argVal} - assert.Equal(t, expected, b.buildArgs.GetAllAllowed()) + assert.Equal(t, expected, sb.state.buildArgs.GetAllAllowed()) } func TestShell(t *testing.T) { b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) shellCmd := "powershell" - req := defaultDispatchReq(b, shellCmd) - req.attributes = map[string]bool{"json": true} + cmd := &instructions.ShellCommand{Shell: strslice.StrSlice{shellCmd}} - err := shell(req) + err := dispatch(sb, cmd) require.NoError(t, err) expectedShell := strslice.StrSlice([]string{shellCmd}) - assert.Equal(t, expectedShell, req.state.runConfig.Shell) -} - -func TestParseOptInterval(t *testing.T) { - flInterval := &Flag{ - name: "interval", - flagType: stringType, - Value: "50ns", - } - _, err := parseOptInterval(flInterval) - testutil.ErrorContains(t, err, "cannot be less than 1ms") - - flInterval.Value = "1ms" - _, err = parseOptInterval(flInterval) - require.NoError(t, err) + assert.Equal(t, expectedShell, sb.state.runConfig.Shell) } func TestPrependEnvOnCmd(t *testing.T) { @@ -469,8 +410,10 @@ func TestPrependEnvOnCmd(t *testing.T) { func TestRunWithBuildArgs(t *testing.T) { b := newBuilderWithMockBackend() - b.buildArgs.argsFromOptions["HTTP_PROXY"] = strPtr("FOO") + args := newBuildArgs(make(map[string]*string)) + args.argsFromOptions["HTTP_PROXY"] = strPtr("FOO") b.disableCommit = false + sb := newDispatchRequest(b, '`', nil, args, newStagesBuildResults()) runConfig := &container.Config{} origCmd := strslice.StrSlice([]string{"cmd", "in", "from", "image"}) @@ -512,14 +455,18 @@ func TestRunWithBuildArgs(t *testing.T) { assert.Equal(t, strslice.StrSlice(nil), cfg.Config.Entrypoint) return "", nil } - - req := defaultDispatchReq(b, "abcdef") - require.NoError(t, from(req)) - b.buildArgs.AddArg("one", strPtr("two")) - - req.args = []string{"echo foo"} - require.NoError(t, run(req)) + from := &instructions.Stage{BaseName: "abcdef"} + err := initializeStage(sb, from) + require.NoError(t, err) + sb.state.buildArgs.AddArg("one", strPtr("two")) + run := &instructions.RunCommand{ + ShellDependantCmdLine: instructions.ShellDependantCmdLine{ + CmdLine: strslice.StrSlice{"echo foo"}, + PrependShell: true, + }, + } + require.NoError(t, dispatch(sb, run)) // Check that runConfig.Cmd has not been modified by run - assert.Equal(t, origCmd, req.state.runConfig.Cmd) + assert.Equal(t, origCmd, sb.state.runConfig.Cmd) } diff --git a/components/engine/builder/dockerfile/dispatchers_unix.go b/components/engine/builder/dockerfile/dispatchers_unix.go index c815ec57e7..6f0d581b94 100644 --- a/components/engine/builder/dockerfile/dispatchers_unix.go +++ b/components/engine/builder/dockerfile/dispatchers_unix.go @@ -4,7 +4,6 @@ package dockerfile import ( "errors" - "fmt" "os" "path/filepath" ) @@ -23,10 +22,6 @@ func normalizeWorkdir(_ string, current string, requested string) (string, error return requested, nil } -func errNotJSON(command, _ string) error { - return fmt.Errorf("%s requires the arguments to be in JSON form", command) -} - // equalEnvKeys compare two strings and returns true if they are equal. On // Windows this comparison is case insensitive. func equalEnvKeys(from, to string) bool { diff --git a/components/engine/builder/dockerfile/dispatchers_windows.go b/components/engine/builder/dockerfile/dispatchers_windows.go index 1c77b29918..8f6eaac180 100644 --- a/components/engine/builder/dockerfile/dispatchers_windows.go +++ b/components/engine/builder/dockerfile/dispatchers_windows.go @@ -94,25 +94,6 @@ func normalizeWorkdirWindows(current string, requested string) (string, error) { return (strings.ToUpper(string(requested[0])) + requested[1:]), nil } -func errNotJSON(command, original string) error { - // For Windows users, give a hint if it looks like it might contain - // a path which hasn't been escaped such as ["c:\windows\system32\prog.exe", "-param"], - // as JSON must be escaped. Unfortunate... - // - // Specifically looking for quote-driveletter-colon-backslash, there's no - // double backslash and a [] pair. No, this is not perfect, but it doesn't - // have to be. It's simply a hint to make life a little easier. - extra := "" - original = filepath.FromSlash(strings.ToLower(strings.Replace(strings.ToLower(original), strings.ToLower(command)+" ", "", -1))) - if len(regexp.MustCompile(`"[a-z]:\\.*`).FindStringSubmatch(original)) > 0 && - !strings.Contains(original, `\\`) && - strings.Contains(original, "[") && - strings.Contains(original, "]") { - extra = fmt.Sprintf(`. It looks like '%s' includes a file path without an escaped back-slash. JSON requires back-slashes to be escaped such as ["c:\\path\\to\\file.exe", "/parameter"]`, original) - } - return fmt.Errorf("%s requires the arguments to be in JSON form%s", command, extra) -} - // equalEnvKeys compare two strings and returns true if they are equal. On // Windows this comparison is case insensitive. func equalEnvKeys(from, to string) bool { diff --git a/components/engine/builder/dockerfile/evaluator.go b/components/engine/builder/dockerfile/evaluator.go index a3ca201e75..cd4f513fad 100644 --- a/components/engine/builder/dockerfile/evaluator.go +++ b/components/engine/builder/dockerfile/evaluator.go @@ -20,169 +20,79 @@ package dockerfile import ( - "bytes" - "fmt" + "reflect" "runtime" + "strconv" "strings" "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/dockerfile/command" - "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig/opts" "github.com/pkg/errors" ) -// Environment variable interpolation will happen on these statements only. -var replaceEnvAllowed = map[string]bool{ - command.Env: true, - command.Label: true, - command.Add: true, - command.Copy: true, - command.Workdir: true, - command.Expose: true, - command.Volume: true, - command.User: true, - command.StopSignal: true, - command.Arg: true, -} - -// Certain commands are allowed to have their args split into more -// words after env var replacements. Meaning: -// ENV foo="123 456" -// EXPOSE $foo -// should result in the same thing as: -// EXPOSE 123 456 -// and not treat "123 456" as a single word. -// Note that: EXPOSE "$foo" and EXPOSE $foo are not the same thing. -// Quotes will cause it to still be treated as single word. -var allowWordExpansion = map[string]bool{ - command.Expose: true, -} - -type dispatchRequest struct { - builder *Builder // TODO: replace this with a smaller interface - args []string - attributes map[string]bool - flags *BFlags - original string - shlex *ShellLex - state *dispatchState - source builder.Source -} - -func newDispatchRequestFromOptions(options dispatchOptions, builder *Builder, args []string) dispatchRequest { - return dispatchRequest{ - builder: builder, - args: args, - attributes: options.node.Attributes, - original: options.node.Original, - flags: NewBFlagsWithArgs(options.node.Flags), - shlex: options.shlex, - state: options.state, - source: options.source, - } -} - -type dispatcher func(dispatchRequest) error - -var evaluateTable map[string]dispatcher - -func init() { - evaluateTable = map[string]dispatcher{ - command.Add: add, - command.Arg: arg, - command.Cmd: cmd, - command.Copy: dispatchCopy, // copy() is a go builtin - command.Entrypoint: entrypoint, - command.Env: env, - command.Expose: expose, - command.From: from, - command.Healthcheck: healthcheck, - command.Label: label, - command.Maintainer: maintainer, - command.Onbuild: onbuild, - command.Run: run, - command.Shell: shell, - command.StopSignal: stopSignal, - command.User: user, - command.Volume: volume, - command.Workdir: workdir, - } -} - -func formatStep(stepN int, stepTotal int) string { - return fmt.Sprintf("%d/%d", stepN+1, stepTotal) -} - -// This method is the entrypoint to all statement handling routines. -// -// Almost all nodes will have this structure: -// Child[Node, Node, Node] where Child is from parser.Node.Children and each -// node comes from parser.Node.Next. This forms a "line" with a statement and -// arguments and we process them in this normalized form by hitting -// evaluateTable with the leaf nodes of the command and the Builder object. -// -// ONBUILD is a special case; in this case the parser will emit: -// Child[Node, Child[Node, Node...]] where the first node is the literal -// "onbuild" and the child entrypoint is the command of the ONBUILD statement, -// such as `RUN` in ONBUILD RUN foo. There is special case logic in here to -// deal with that, at least until it becomes more of a general concern with new -// features. -func (b *Builder) dispatch(options dispatchOptions) (*dispatchState, error) { - node := options.node - cmd := node.Value - upperCasedCmd := strings.ToUpper(cmd) - - // To ensure the user is given a decent error message if the platform - // on which the daemon is running does not support a builder command. - if err := platformSupports(strings.ToLower(cmd)); err != nil { - buildsFailed.WithValues(metricsCommandNotSupportedError).Inc() - return nil, validationError{err} - } - - msg := bytes.NewBufferString(fmt.Sprintf("Step %s : %s%s", - options.stepMsg, upperCasedCmd, formatFlags(node.Flags))) - - args := []string{} - ast := node - if cmd == command.Onbuild { - var err error - ast, args, err = handleOnBuildNode(node, msg) +func dispatch(d dispatchRequest, cmd instructions.Command) error { + if c, ok := cmd.(instructions.PlatformSpecific); ok { + err := c.CheckPlatform(d.builder.platform) if err != nil { - return nil, validationError{err} + return validationError{err} + } + } + runConfigEnv := d.state.runConfig.Env + envs := append(runConfigEnv, d.state.buildArgs.FilterAllowed(runConfigEnv)...) + + if ex, ok := cmd.(instructions.SupportsSingleWordExpansion); ok { + err := ex.Expand(func(word string) (string, error) { + return d.shlex.ProcessWord(word, envs) + }) + if err != nil { + return validationError{err} } } - runConfigEnv := options.state.runConfig.Env - envs := append(runConfigEnv, b.buildArgs.FilterAllowed(runConfigEnv)...) - processFunc := createProcessWordFunc(options.shlex, cmd, envs) - words, err := getDispatchArgsFromNode(ast, processFunc, msg) - if err != nil { - buildsFailed.WithValues(metricsErrorProcessingCommandsError).Inc() - return nil, validationError{err} + if d.builder.options.ForceRemove { + defer d.builder.containerManager.RemoveAll(d.builder.Stdout) } - args = append(args, words...) - fmt.Fprintln(b.Stdout, msg.String()) - - f, ok := evaluateTable[cmd] - if !ok { - buildsFailed.WithValues(metricsUnknownInstructionError).Inc() - return nil, validationError{errors.Errorf("unknown instruction: %s", upperCasedCmd)} + switch c := cmd.(type) { + case *instructions.EnvCommand: + return dispatchEnv(d, c) + case *instructions.MaintainerCommand: + return dispatchMaintainer(d, c) + case *instructions.LabelCommand: + return dispatchLabel(d, c) + case *instructions.AddCommand: + return dispatchAdd(d, c) + case *instructions.CopyCommand: + return dispatchCopy(d, c) + case *instructions.OnbuildCommand: + return dispatchOnbuild(d, c) + case *instructions.WorkdirCommand: + return dispatchWorkdir(d, c) + case *instructions.RunCommand: + return dispatchRun(d, c) + case *instructions.CmdCommand: + return dispatchCmd(d, c) + case *instructions.HealthCheckCommand: + return dispatchHealthcheck(d, c) + case *instructions.EntrypointCommand: + return dispatchEntrypoint(d, c) + case *instructions.ExposeCommand: + return dispatchExpose(d, c, envs) + case *instructions.UserCommand: + return dispatchUser(d, c) + case *instructions.VolumeCommand: + return dispatchVolume(d, c) + case *instructions.StopSignalCommand: + return dispatchStopSignal(d, c) + case *instructions.ArgCommand: + return dispatchArg(d, c) + case *instructions.ShellCommand: + return dispatchShell(d, c) } - options.state.updateRunConfig() - err = f(newDispatchRequestFromOptions(options, b, args)) - return options.state, err -} - -type dispatchOptions struct { - state *dispatchState - stepMsg string - node *parser.Node - shlex *ShellLex - source builder.Source + return errors.Errorf("unsupported command type: %v", reflect.TypeOf(cmd)) } // dispatchState is a data object which is modified by dispatchers @@ -193,10 +103,95 @@ type dispatchState struct { imageID string baseImage builder.Image stageName string + buildArgs *buildArgs } -func newDispatchState() *dispatchState { - return &dispatchState{runConfig: &container.Config{}} +func newDispatchState(baseArgs *buildArgs) *dispatchState { + args := baseArgs.Clone() + args.ResetAllowed() + return &dispatchState{runConfig: &container.Config{}, buildArgs: args} +} + +type stagesBuildResults struct { + flat []*container.Config + indexed map[string]*container.Config +} + +func newStagesBuildResults() *stagesBuildResults { + return &stagesBuildResults{ + indexed: make(map[string]*container.Config), + } +} + +func (r *stagesBuildResults) getByName(name string) (*container.Config, bool) { + c, ok := r.indexed[strings.ToLower(name)] + return c, ok +} + +func (r *stagesBuildResults) validateIndex(i int) error { + if i == len(r.flat) { + return errors.New("refers to current build stage") + } + if i < 0 || i > len(r.flat) { + return errors.New("index out of bounds") + } + return nil +} + +func (r *stagesBuildResults) get(nameOrIndex string) (*container.Config, error) { + if c, ok := r.getByName(nameOrIndex); ok { + return c, nil + } + ix, err := strconv.ParseInt(nameOrIndex, 10, 0) + if err != nil { + return nil, nil + } + if err := r.validateIndex(int(ix)); err != nil { + return nil, err + } + return r.flat[ix], nil +} + +func (r *stagesBuildResults) checkStageNameAvailable(name string) error { + if name != "" { + if _, ok := r.getByName(name); ok { + return errors.Errorf("%s stage name already used", name) + } + } + return nil +} + +func (r *stagesBuildResults) commitStage(name string, config *container.Config) error { + if name != "" { + if _, ok := r.getByName(name); ok { + return errors.Errorf("%s stage name already used", name) + } + r.indexed[strings.ToLower(name)] = config + } + r.flat = append(r.flat, config) + return nil +} + +func commitStage(state *dispatchState, stages *stagesBuildResults) error { + return stages.commitStage(state.stageName, state.runConfig) +} + +type dispatchRequest struct { + state *dispatchState + shlex *ShellLex + builder *Builder + source builder.Source + stages *stagesBuildResults +} + +func newDispatchRequest(builder *Builder, escapeToken rune, source builder.Source, buildArgs *buildArgs, stages *stagesBuildResults) dispatchRequest { + return dispatchRequest{ + state: newDispatchState(buildArgs), + shlex: NewShellLex(escapeToken), + builder: builder, + source: source, + stages: stages, + } } func (s *dispatchState) updateRunConfig() { @@ -220,12 +215,14 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) { s.imageID = image.ImageID() if image.RunConfig() != nil { - s.runConfig = image.RunConfig() + s.runConfig = copyRunConfig(image.RunConfig()) // copy avoids referencing the same instance when 2 stages have the same base } else { s.runConfig = &container.Config{} } s.baseImage = image s.setDefaultPath() + s.runConfig.OpenStdin = false + s.runConfig.StdinOnce = false } // Add the default PATH to runConfig.ENV if one exists for the platform and there @@ -244,84 +241,3 @@ func (s *dispatchState) setDefaultPath() { s.runConfig.Env = append(s.runConfig.Env, "PATH="+system.DefaultPathEnv(platform)) } } - -func handleOnBuildNode(ast *parser.Node, msg *bytes.Buffer) (*parser.Node, []string, error) { - if ast.Next == nil { - return nil, nil, validationError{errors.New("ONBUILD requires at least one argument")} - } - ast = ast.Next.Children[0] - msg.WriteString(" " + ast.Value + formatFlags(ast.Flags)) - return ast, []string{ast.Value}, nil -} - -func formatFlags(flags []string) string { - if len(flags) > 0 { - return " " + strings.Join(flags, " ") - } - return "" -} - -func getDispatchArgsFromNode(ast *parser.Node, processFunc processWordFunc, msg *bytes.Buffer) ([]string, error) { - args := []string{} - for i := 0; ast.Next != nil; i++ { - ast = ast.Next - words, err := processFunc(ast.Value) - if err != nil { - return nil, err - } - args = append(args, words...) - msg.WriteString(" " + ast.Value) - } - return args, nil -} - -type processWordFunc func(string) ([]string, error) - -func createProcessWordFunc(shlex *ShellLex, cmd string, envs []string) processWordFunc { - switch { - case !replaceEnvAllowed[cmd]: - return func(word string) ([]string, error) { - return []string{word}, nil - } - case allowWordExpansion[cmd]: - return func(word string) ([]string, error) { - return shlex.ProcessWords(word, envs) - } - default: - return func(word string) ([]string, error) { - word, err := shlex.ProcessWord(word, envs) - return []string{word}, err - } - } -} - -// checkDispatch does a simple check for syntax errors of the Dockerfile. -// Because some of the instructions can only be validated through runtime, -// arg, env, etc., this syntax check will not be complete and could not replace -// the runtime check. Instead, this function is only a helper that allows -// user to find out the obvious error in Dockerfile earlier on. -func checkDispatch(ast *parser.Node) error { - cmd := ast.Value - upperCasedCmd := strings.ToUpper(cmd) - - // To ensure the user is given a decent error message if the platform - // on which the daemon is running does not support a builder command. - if err := platformSupports(strings.ToLower(cmd)); err != nil { - return err - } - - // The instruction itself is ONBUILD, we will make sure it follows with at - // least one argument - if upperCasedCmd == "ONBUILD" { - if ast.Next == nil { - buildsFailed.WithValues(metricsMissingOnbuildArgumentsError).Inc() - return errors.New("ONBUILD requires at least one argument") - } - } - - if _, ok := evaluateTable[cmd]; ok { - return nil - } - buildsFailed.WithValues(metricsUnknownInstructionError).Inc() - return errors.Errorf("unknown instruction: %s", upperCasedCmd) -} diff --git a/components/engine/builder/dockerfile/evaluator_test.go b/components/engine/builder/dockerfile/evaluator_test.go index b64e21e625..fc5512d0a3 100644 --- a/components/engine/builder/dockerfile/evaluator_test.go +++ b/components/engine/builder/dockerfile/evaluator_test.go @@ -1,13 +1,9 @@ package dockerfile import ( - "io/ioutil" - "strings" "testing" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/remotecontext" "github.com/docker/docker/internal/testutil" "github.com/docker/docker/pkg/archive" @@ -15,8 +11,9 @@ import ( ) type dispatchTestCase struct { - name, dockerfile, expectedError string - files map[string]string + name, expectedError string + cmd instructions.Command + files map[string]string } func init() { @@ -24,108 +21,73 @@ func init() { } func initDispatchTestCases() []dispatchTestCase { - dispatchTestCases := []dispatchTestCase{{ - name: "copyEmptyWhitespace", - dockerfile: `COPY - quux \ - bar`, - expectedError: "COPY requires at least two arguments", - }, + dispatchTestCases := []dispatchTestCase{ { - name: "ONBUILD forbidden FROM", - dockerfile: "ONBUILD FROM scratch", - expectedError: "FROM isn't allowed as an ONBUILD trigger", - files: nil, - }, - { - name: "ONBUILD forbidden MAINTAINER", - dockerfile: "ONBUILD MAINTAINER docker.io", - expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger", - files: nil, - }, - { - name: "ARG two arguments", - dockerfile: "ARG foo bar", - expectedError: "ARG requires exactly one argument", - files: nil, - }, - { - name: "MAINTAINER unknown flag", - dockerfile: "MAINTAINER --boo joe@example.com", - expectedError: "Unknown flag: boo", - files: nil, - }, - { - name: "ADD multiple files to file", - dockerfile: "ADD file1.txt file2.txt test", + name: "ADD multiple files to file", + cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{ + "file1.txt", + "file2.txt", + "test", + }}, expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /", files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"}, }, { - name: "JSON ADD multiple files to file", - dockerfile: `ADD ["file1.txt", "file2.txt", "test"]`, + name: "Wildcard ADD multiple files to file", + cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{ + "file*.txt", + "test", + }}, expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /", files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"}, }, { - name: "Wildcard ADD multiple files to file", - dockerfile: "ADD file*.txt test", - expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /", - files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"}, - }, - { - name: "Wildcard JSON ADD multiple files to file", - dockerfile: `ADD ["file*.txt", "test"]`, - expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /", - files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"}, - }, - { - name: "COPY multiple files to file", - dockerfile: "COPY file1.txt file2.txt test", + name: "COPY multiple files to file", + cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{ + "file1.txt", + "file2.txt", + "test", + }}, expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /", files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"}, }, { - name: "JSON COPY multiple files to file", - dockerfile: `COPY ["file1.txt", "file2.txt", "test"]`, - expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /", - files: map[string]string{"file1.txt": "test1", "file2.txt": "test2"}, - }, - { - name: "ADD multiple files to file with whitespace", - dockerfile: `ADD [ "test file1.txt", "test file2.txt", "test" ]`, + name: "ADD multiple files to file with whitespace", + cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{ + "test file1.txt", + "test file2.txt", + "test", + }}, expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /", files: map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"}, }, { - name: "COPY multiple files to file with whitespace", - dockerfile: `COPY [ "test file1.txt", "test file2.txt", "test" ]`, + name: "COPY multiple files to file with whitespace", + cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{ + "test file1.txt", + "test file2.txt", + "test", + }}, expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /", files: map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"}, }, { - name: "COPY wildcard no files", - dockerfile: `COPY file*.txt /tmp/`, + name: "COPY wildcard no files", + cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{ + "file*.txt", + "/tmp/", + }}, expectedError: "COPY failed: no source files were specified", files: nil, }, { - name: "COPY url", - dockerfile: `COPY https://index.docker.io/robots.txt /`, + name: "COPY url", + cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{ + "https://index.docker.io/robots.txt", + "/", + }}, expectedError: "source can't be a URL for COPY", files: nil, - }, - { - name: "Chaining ONBUILD", - dockerfile: `ONBUILD ONBUILD RUN touch foobar`, - expectedError: "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed", - files: nil, - }, - { - name: "Invalid instruction", - dockerfile: `foo bar`, - expectedError: "unknown instruction: FOO", - files: nil, }} return dispatchTestCases @@ -171,33 +133,8 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) { } }() - r := strings.NewReader(testCase.dockerfile) - result, err := parser.Parse(r) - - if err != nil { - t.Fatalf("Error when parsing Dockerfile: %s", err) - } - - options := &types.ImageBuildOptions{ - BuildArgs: make(map[string]*string), - } - - b := &Builder{ - options: options, - Stdout: ioutil.Discard, - buildArgs: newBuildArgs(options.BuildArgs), - } - - shlex := NewShellLex(parser.DefaultEscapeToken) - n := result.AST - state := &dispatchState{runConfig: &container.Config{}} - opts := dispatchOptions{ - state: state, - stepMsg: formatStep(0, len(n.Children)), - node: n.Children[0], - shlex: shlex, - source: context, - } - _, err = b.dispatch(opts) + b := newBuilderWithMockBackend() + sb := newDispatchRequest(b, '`', context, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + err = dispatch(sb, testCase.cmd) testutil.ErrorContains(t, err, testCase.expectedError) } diff --git a/components/engine/builder/dockerfile/evaluator_unix.go b/components/engine/builder/dockerfile/evaluator_unix.go deleted file mode 100644 index 28fd5b156b..0000000000 --- a/components/engine/builder/dockerfile/evaluator_unix.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !windows - -package dockerfile - -// platformSupports is a short-term function to give users a quality error -// message if a Dockerfile uses a command not supported on the platform. -func platformSupports(command string) error { - return nil -} diff --git a/components/engine/builder/dockerfile/evaluator_windows.go b/components/engine/builder/dockerfile/evaluator_windows.go deleted file mode 100644 index 72483a2ec8..0000000000 --- a/components/engine/builder/dockerfile/evaluator_windows.go +++ /dev/null @@ -1,13 +0,0 @@ -package dockerfile - -import "fmt" - -// platformSupports is gives users a quality error message if a Dockerfile uses -// a command not supported on the platform. -func platformSupports(command string) error { - switch command { - case "stopsignal": - return fmt.Errorf("The daemon on this platform does not support the command '%s'", command) - } - return nil -} diff --git a/components/engine/builder/dockerfile/imagecontext.go b/components/engine/builder/dockerfile/imagecontext.go index fedad6fdf3..084255d173 100644 --- a/components/engine/builder/dockerfile/imagecontext.go +++ b/components/engine/builder/dockerfile/imagecontext.go @@ -1,9 +1,6 @@ package dockerfile import ( - "strconv" - "strings" - "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/builder/remotecontext" @@ -13,79 +10,6 @@ import ( "golang.org/x/net/context" ) -type buildStage struct { - id string -} - -func newBuildStage(imageID string) *buildStage { - return &buildStage{id: imageID} -} - -func (b *buildStage) ImageID() string { - return b.id -} - -func (b *buildStage) update(imageID string) { - b.id = imageID -} - -// buildStages tracks each stage of a build so they can be retrieved by index -// or by name. -type buildStages struct { - sequence []*buildStage - byName map[string]*buildStage -} - -func newBuildStages() *buildStages { - return &buildStages{byName: make(map[string]*buildStage)} -} - -func (s *buildStages) getByName(name string) (*buildStage, bool) { - stage, ok := s.byName[strings.ToLower(name)] - return stage, ok -} - -func (s *buildStages) get(indexOrName string) (*buildStage, error) { - index, err := strconv.Atoi(indexOrName) - if err == nil { - if err := s.validateIndex(index); err != nil { - return nil, err - } - return s.sequence[index], nil - } - if im, ok := s.byName[strings.ToLower(indexOrName)]; ok { - return im, nil - } - return nil, nil -} - -func (s *buildStages) validateIndex(i int) error { - if i < 0 || i >= len(s.sequence)-1 { - if i == len(s.sequence)-1 { - return errors.New("refers to current build stage") - } - return errors.New("index out of bounds") - } - return nil -} - -func (s *buildStages) add(name string, image builder.Image) error { - stage := newBuildStage(image.ImageID()) - name = strings.ToLower(name) - if len(name) > 0 { - if _, ok := s.byName[name]; ok { - return errors.Errorf("duplicate name %s", name) - } - s.byName[name] = stage - } - s.sequence = append(s.sequence, stage) - return nil -} - -func (s *buildStages) update(imageID string) { - s.sequence[len(s.sequence)-1].update(imageID) -} - type getAndMountFunc func(string, bool) (builder.Image, builder.ReleaseableLayer, error) // imageSources mounts images and provides a cache for mounted images. It tracks diff --git a/components/engine/builder/dockerfile/bflag.go b/components/engine/builder/dockerfile/instructions/bflag.go similarity index 99% rename from components/engine/builder/dockerfile/bflag.go rename to components/engine/builder/dockerfile/instructions/bflag.go index d849661620..7a81e3c136 100644 --- a/components/engine/builder/dockerfile/bflag.go +++ b/components/engine/builder/dockerfile/instructions/bflag.go @@ -1,4 +1,4 @@ -package dockerfile +package instructions import ( "fmt" diff --git a/components/engine/builder/dockerfile/bflag_test.go b/components/engine/builder/dockerfile/instructions/bflag_test.go similarity index 99% rename from components/engine/builder/dockerfile/bflag_test.go rename to components/engine/builder/dockerfile/instructions/bflag_test.go index 4ea10fff4b..b194ba785f 100644 --- a/components/engine/builder/dockerfile/bflag_test.go +++ b/components/engine/builder/dockerfile/instructions/bflag_test.go @@ -1,4 +1,4 @@ -package dockerfile +package instructions import ( "testing" diff --git a/components/engine/builder/dockerfile/instructions/commands.go b/components/engine/builder/dockerfile/instructions/commands.go new file mode 100644 index 0000000000..8d5d6b09ae --- /dev/null +++ b/components/engine/builder/dockerfile/instructions/commands.go @@ -0,0 +1,396 @@ +package instructions + +import ( + "errors" + + "strings" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/strslice" +) + +// KeyValuePair represent an arbitrary named value (usefull in slice insted of map[string] string to preserve ordering) +type KeyValuePair struct { + Key string + Value string +} + +func (kvp *KeyValuePair) String() string { + return kvp.Key + "=" + kvp.Value +} + +// Command is implemented by every command present in a dockerfile +type Command interface { + Name() string +} + +// KeyValuePairs is a slice of KeyValuePair +type KeyValuePairs []KeyValuePair + +// withNameAndCode is the base of every command in a Dockerfile (String() returns its source code) +type withNameAndCode struct { + code string + name string +} + +func (c *withNameAndCode) String() string { + return c.code +} + +// Name of the command +func (c *withNameAndCode) Name() string { + return c.name +} + +func newWithNameAndCode(req parseRequest) withNameAndCode { + return withNameAndCode{code: strings.TrimSpace(req.original), name: req.command} +} + +// SingleWordExpander is a provider for variable expansion where 1 word => 1 output +type SingleWordExpander func(word string) (string, error) + +// SupportsSingleWordExpansion interface marks a command as supporting variable expansion +type SupportsSingleWordExpansion interface { + Expand(expander SingleWordExpander) error +} + +// PlatformSpecific adds platform checks to a command +type PlatformSpecific interface { + CheckPlatform(platform string) error +} + +func expandKvp(kvp KeyValuePair, expander SingleWordExpander) (KeyValuePair, error) { + key, err := expander(kvp.Key) + if err != nil { + return KeyValuePair{}, err + } + value, err := expander(kvp.Value) + if err != nil { + return KeyValuePair{}, err + } + return KeyValuePair{Key: key, Value: value}, nil +} +func expandKvpsInPlace(kvps KeyValuePairs, expander SingleWordExpander) error { + for i, kvp := range kvps { + newKvp, err := expandKvp(kvp, expander) + if err != nil { + return err + } + kvps[i] = newKvp + } + return nil +} + +func expandSliceInPlace(values []string, expander SingleWordExpander) error { + for i, v := range values { + newValue, err := expander(v) + if err != nil { + return err + } + values[i] = newValue + } + return nil +} + +// EnvCommand : ENV key1 value1 [keyN valueN...] +type EnvCommand struct { + withNameAndCode + Env KeyValuePairs // kvp slice instead of map to preserve ordering +} + +// Expand variables +func (c *EnvCommand) Expand(expander SingleWordExpander) error { + return expandKvpsInPlace(c.Env, expander) +} + +// MaintainerCommand : MAINTAINER maintainer_name +type MaintainerCommand struct { + withNameAndCode + Maintainer string +} + +// LabelCommand : LABEL some json data describing the image +// +// Sets the Label variable foo to bar, +// +type LabelCommand struct { + withNameAndCode + Labels KeyValuePairs // kvp slice instead of map to preserve ordering +} + +// Expand variables +func (c *LabelCommand) Expand(expander SingleWordExpander) error { + return expandKvpsInPlace(c.Labels, expander) +} + +// SourcesAndDest represent a list of source files and a destination +type SourcesAndDest []string + +// Sources list the source paths +func (s SourcesAndDest) Sources() []string { + res := make([]string, len(s)-1) + copy(res, s[:len(s)-1]) + return res +} + +// Dest path of the operation +func (s SourcesAndDest) Dest() string { + return s[len(s)-1] +} + +// AddCommand : ADD foo /path +// +// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling +// exist here. If you do not wish to have this automatic handling, use COPY. +// +type AddCommand struct { + withNameAndCode + SourcesAndDest + Chown string +} + +// Expand variables +func (c *AddCommand) Expand(expander SingleWordExpander) error { + return expandSliceInPlace(c.SourcesAndDest, expander) +} + +// CopyCommand : COPY foo /path +// +// Same as 'ADD' but without the tar and remote url handling. +// +type CopyCommand struct { + withNameAndCode + SourcesAndDest + From string + Chown string +} + +// Expand variables +func (c *CopyCommand) Expand(expander SingleWordExpander) error { + return expandSliceInPlace(c.SourcesAndDest, expander) +} + +// OnbuildCommand : ONBUILD +type OnbuildCommand struct { + withNameAndCode + Expression string +} + +// WorkdirCommand : WORKDIR /tmp +// +// Set the working directory for future RUN/CMD/etc statements. +// +type WorkdirCommand struct { + withNameAndCode + Path string +} + +// Expand variables +func (c *WorkdirCommand) Expand(expander SingleWordExpander) error { + p, err := expander(c.Path) + if err != nil { + return err + } + c.Path = p + return nil +} + +// ShellDependantCmdLine represents a cmdline optionaly prepended with the shell +type ShellDependantCmdLine struct { + CmdLine strslice.StrSlice + PrependShell bool +} + +// RunCommand : RUN some command yo +// +// run a command and commit the image. Args are automatically prepended with +// the current SHELL which defaults to 'sh -c' under linux or 'cmd /S /C' under +// Windows, in the event there is only one argument The difference in processing: +// +// RUN echo hi # sh -c echo hi (Linux) +// RUN echo hi # cmd /S /C echo hi (Windows) +// RUN [ "echo", "hi" ] # echo hi +// +type RunCommand struct { + withNameAndCode + ShellDependantCmdLine +} + +// CmdCommand : CMD foo +// +// Set the default command to run in the container (which may be empty). +// Argument handling is the same as RUN. +// +type CmdCommand struct { + withNameAndCode + ShellDependantCmdLine +} + +// HealthCheckCommand : HEALTHCHECK foo +// +// Set the default healthcheck command to run in the container (which may be empty). +// Argument handling is the same as RUN. +// +type HealthCheckCommand struct { + withNameAndCode + Health *container.HealthConfig +} + +// EntrypointCommand : ENTRYPOINT /usr/sbin/nginx +// +// Set the entrypoint to /usr/sbin/nginx. Will accept the CMD as the arguments +// to /usr/sbin/nginx. Uses the default shell if not in JSON format. +// +// Handles command processing similar to CMD and RUN, only req.runConfig.Entrypoint +// is initialized at newBuilder time instead of through argument parsing. +// +type EntrypointCommand struct { + withNameAndCode + ShellDependantCmdLine +} + +// ExposeCommand : EXPOSE 6667/tcp 7000/tcp +// +// Expose ports for links and port mappings. This all ends up in +// req.runConfig.ExposedPorts for runconfig. +// +type ExposeCommand struct { + withNameAndCode + Ports []string +} + +// UserCommand : USER foo +// +// Set the user to 'foo' for future commands and when running the +// ENTRYPOINT/CMD at container run time. +// +type UserCommand struct { + withNameAndCode + User string +} + +// Expand variables +func (c *UserCommand) Expand(expander SingleWordExpander) error { + p, err := expander(c.User) + if err != nil { + return err + } + c.User = p + return nil +} + +// VolumeCommand : VOLUME /foo +// +// Expose the volume /foo for use. Will also accept the JSON array form. +// +type VolumeCommand struct { + withNameAndCode + Volumes []string +} + +// Expand variables +func (c *VolumeCommand) Expand(expander SingleWordExpander) error { + return expandSliceInPlace(c.Volumes, expander) +} + +// StopSignalCommand : STOPSIGNAL signal +// +// Set the signal that will be used to kill the container. +type StopSignalCommand struct { + withNameAndCode + Signal string +} + +// Expand variables +func (c *StopSignalCommand) Expand(expander SingleWordExpander) error { + p, err := expander(c.Signal) + if err != nil { + return err + } + c.Signal = p + return nil +} + +// CheckPlatform checks that the command is supported in the target platform +func (c *StopSignalCommand) CheckPlatform(platform string) error { + if platform == "windows" { + return errors.New("The daemon on this platform does not support the command stopsignal") + } + return nil +} + +// ArgCommand : ARG name[=value] +// +// Adds the variable foo to the trusted list of variables that can be passed +// to builder using the --build-arg flag for expansion/substitution or passing to 'run'. +// Dockerfile author may optionally set a default value of this variable. +type ArgCommand struct { + withNameAndCode + Key string + Value *string +} + +// Expand variables +func (c *ArgCommand) Expand(expander SingleWordExpander) error { + p, err := expander(c.Key) + if err != nil { + return err + } + c.Key = p + if c.Value != nil { + p, err = expander(*c.Value) + if err != nil { + return err + } + c.Value = &p + } + return nil +} + +// ShellCommand : SHELL powershell -command +// +// Set the non-default shell to use. +type ShellCommand struct { + withNameAndCode + Shell strslice.StrSlice +} + +// Stage represents a single stage in a multi-stage build +type Stage struct { + Name string + Commands []Command + BaseName string + SourceCode string +} + +// AddCommand to the stage +func (s *Stage) AddCommand(cmd Command) { + // todo: validate cmd type + s.Commands = append(s.Commands, cmd) +} + +// IsCurrentStage check if the stage name is the current stage +func IsCurrentStage(s []Stage, name string) bool { + if len(s) == 0 { + return false + } + return s[len(s)-1].Name == name +} + +// CurrentStage return the last stage in a slice +func CurrentStage(s []Stage) (*Stage, error) { + if len(s) == 0 { + return nil, errors.New("No build stage in current context") + } + return &s[len(s)-1], nil +} + +// HasStage looks for the presence of a given stage name +func HasStage(s []Stage, name string) (int, bool) { + for i, stage := range s { + if stage.Name == name { + return i, true + } + } + return -1, false +} diff --git a/components/engine/builder/dockerfile/instructions/errors_unix.go b/components/engine/builder/dockerfile/instructions/errors_unix.go new file mode 100644 index 0000000000..0b03b34cd1 --- /dev/null +++ b/components/engine/builder/dockerfile/instructions/errors_unix.go @@ -0,0 +1,9 @@ +// +build !windows + +package instructions + +import "fmt" + +func errNotJSON(command, _ string) error { + return fmt.Errorf("%s requires the arguments to be in JSON form", command) +} diff --git a/components/engine/builder/dockerfile/instructions/errors_windows.go b/components/engine/builder/dockerfile/instructions/errors_windows.go new file mode 100644 index 0000000000..a4843c5b6a --- /dev/null +++ b/components/engine/builder/dockerfile/instructions/errors_windows.go @@ -0,0 +1,27 @@ +package instructions + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" +) + +func errNotJSON(command, original string) error { + // For Windows users, give a hint if it looks like it might contain + // a path which hasn't been escaped such as ["c:\windows\system32\prog.exe", "-param"], + // as JSON must be escaped. Unfortunate... + // + // Specifically looking for quote-driveletter-colon-backslash, there's no + // double backslash and a [] pair. No, this is not perfect, but it doesn't + // have to be. It's simply a hint to make life a little easier. + extra := "" + original = filepath.FromSlash(strings.ToLower(strings.Replace(strings.ToLower(original), strings.ToLower(command)+" ", "", -1))) + if len(regexp.MustCompile(`"[a-z]:\\.*`).FindStringSubmatch(original)) > 0 && + !strings.Contains(original, `\\`) && + strings.Contains(original, "[") && + strings.Contains(original, "]") { + extra = fmt.Sprintf(`. It looks like '%s' includes a file path without an escaped back-slash. JSON requires back-slashes to be escaped such as ["c:\\path\\to\\file.exe", "/parameter"]`, original) + } + return fmt.Errorf("%s requires the arguments to be in JSON form%s", command, extra) +} diff --git a/components/engine/builder/dockerfile/instructions/parse.go b/components/engine/builder/dockerfile/instructions/parse.go new file mode 100644 index 0000000000..e52ac47a11 --- /dev/null +++ b/components/engine/builder/dockerfile/instructions/parse.go @@ -0,0 +1,635 @@ +package instructions + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/builder/dockerfile/command" + "github.com/docker/docker/builder/dockerfile/parser" + "github.com/pkg/errors" +) + +type parseRequest struct { + command string + args []string + attributes map[string]bool + flags *BFlags + original string +} + +func nodeArgs(node *parser.Node) []string { + result := []string{} + for ; node.Next != nil; node = node.Next { + arg := node.Next + if len(arg.Children) == 0 { + result = append(result, arg.Value) + } else if len(arg.Children) == 1 { + //sub command + result = append(result, arg.Children[0].Value) + result = append(result, nodeArgs(arg.Children[0])...) + } + } + return result +} + +func newParseRequestFromNode(node *parser.Node) parseRequest { + return parseRequest{ + command: node.Value, + args: nodeArgs(node), + attributes: node.Attributes, + original: node.Original, + flags: NewBFlagsWithArgs(node.Flags), + } +} + +// ParseInstruction converts an AST to a typed instruction (either a command or a build stage beginning when encountering a `FROM` statement) +func ParseInstruction(node *parser.Node) (interface{}, error) { + req := newParseRequestFromNode(node) + switch node.Value { + case command.Env: + return parseEnv(req) + case command.Maintainer: + return parseMaintainer(req) + case command.Label: + return parseLabel(req) + case command.Add: + return parseAdd(req) + case command.Copy: + return parseCopy(req) + case command.From: + return parseFrom(req) + case command.Onbuild: + return parseOnBuild(req) + case command.Workdir: + return parseWorkdir(req) + case command.Run: + return parseRun(req) + case command.Cmd: + return parseCmd(req) + case command.Healthcheck: + return parseHealthcheck(req) + case command.Entrypoint: + return parseEntrypoint(req) + case command.Expose: + return parseExpose(req) + case command.User: + return parseUser(req) + case command.Volume: + return parseVolume(req) + case command.StopSignal: + return parseStopSignal(req) + case command.Arg: + return parseArg(req) + case command.Shell: + return parseShell(req) + } + + return nil, &UnknownInstruction{Instruction: node.Value, Line: node.StartLine} +} + +// ParseCommand converts an AST to a typed Command +func ParseCommand(node *parser.Node) (Command, error) { + s, err := ParseInstruction(node) + if err != nil { + return nil, err + } + if c, ok := s.(Command); ok { + return c, nil + } + return nil, errors.Errorf("%T is not a command type", s) +} + +// UnknownInstruction represents an error occuring when a command is unresolvable +type UnknownInstruction struct { + Line int + Instruction string +} + +func (e *UnknownInstruction) Error() string { + return fmt.Sprintf("unknown instruction: %s", strings.ToUpper(e.Instruction)) +} + +// IsUnknownInstruction checks if the error is an UnknownInstruction or a parseError containing an UnknownInstruction +func IsUnknownInstruction(err error) bool { + _, ok := err.(*UnknownInstruction) + if !ok { + var pe *parseError + if pe, ok = err.(*parseError); ok { + _, ok = pe.inner.(*UnknownInstruction) + } + } + return ok +} + +type parseError struct { + inner error + node *parser.Node +} + +func (e *parseError) Error() string { + return fmt.Sprintf("Dockerfile parse error line %d: %v", e.node.StartLine, e.inner.Error()) +} + +// Parse a docker file into a collection of buildable stages +func Parse(ast *parser.Node) (stages []Stage, metaArgs []ArgCommand, err error) { + for _, n := range ast.Children { + cmd, err := ParseInstruction(n) + if err != nil { + return nil, nil, &parseError{inner: err, node: n} + } + if len(stages) == 0 { + // meta arg case + if a, isArg := cmd.(*ArgCommand); isArg { + metaArgs = append(metaArgs, *a) + continue + } + } + switch c := cmd.(type) { + case *Stage: + stages = append(stages, *c) + case Command: + stage, err := CurrentStage(stages) + if err != nil { + return nil, nil, err + } + stage.AddCommand(c) + default: + return nil, nil, errors.Errorf("%T is not a command type", cmd) + } + + } + return stages, metaArgs, nil +} + +func parseKvps(args []string, cmdName string) (KeyValuePairs, error) { + if len(args) == 0 { + return nil, errAtLeastOneArgument(cmdName) + } + if len(args)%2 != 0 { + // should never get here, but just in case + return nil, errTooManyArguments(cmdName) + } + var res KeyValuePairs + for j := 0; j < len(args); j += 2 { + if len(args[j]) == 0 { + return nil, errBlankCommandNames(cmdName) + } + name := args[j] + value := args[j+1] + res = append(res, KeyValuePair{Key: name, Value: value}) + } + return res, nil +} + +func parseEnv(req parseRequest) (*EnvCommand, error) { + + if err := req.flags.Parse(); err != nil { + return nil, err + } + envs, err := parseKvps(req.args, "ENV") + if err != nil { + return nil, err + } + return &EnvCommand{ + Env: envs, + withNameAndCode: newWithNameAndCode(req), + }, nil +} + +func parseMaintainer(req parseRequest) (*MaintainerCommand, error) { + if len(req.args) != 1 { + return nil, errExactlyOneArgument("MAINTAINER") + } + + if err := req.flags.Parse(); err != nil { + return nil, err + } + return &MaintainerCommand{ + Maintainer: req.args[0], + withNameAndCode: newWithNameAndCode(req), + }, nil +} + +func parseLabel(req parseRequest) (*LabelCommand, error) { + + if err := req.flags.Parse(); err != nil { + return nil, err + } + + labels, err := parseKvps(req.args, "LABEL") + if err != nil { + return nil, err + } + + return &LabelCommand{ + Labels: labels, + withNameAndCode: newWithNameAndCode(req), + }, nil +} + +func parseAdd(req parseRequest) (*AddCommand, error) { + if len(req.args) < 2 { + return nil, errAtLeastTwoArguments("ADD") + } + flChown := req.flags.AddString("chown", "") + if err := req.flags.Parse(); err != nil { + return nil, err + } + return &AddCommand{ + SourcesAndDest: SourcesAndDest(req.args), + withNameAndCode: newWithNameAndCode(req), + Chown: flChown.Value, + }, nil +} + +func parseCopy(req parseRequest) (*CopyCommand, error) { + if len(req.args) < 2 { + return nil, errAtLeastTwoArguments("COPY") + } + flChown := req.flags.AddString("chown", "") + flFrom := req.flags.AddString("from", "") + if err := req.flags.Parse(); err != nil { + return nil, err + } + return &CopyCommand{ + SourcesAndDest: SourcesAndDest(req.args), + From: flFrom.Value, + withNameAndCode: newWithNameAndCode(req), + Chown: flChown.Value, + }, nil +} + +func parseFrom(req parseRequest) (*Stage, error) { + stageName, err := parseBuildStageName(req.args) + if err != nil { + return nil, err + } + + if err := req.flags.Parse(); err != nil { + return nil, err + } + code := strings.TrimSpace(req.original) + + return &Stage{ + BaseName: req.args[0], + Name: stageName, + SourceCode: code, + Commands: []Command{}, + }, nil + +} + +func parseBuildStageName(args []string) (string, error) { + stageName := "" + switch { + case len(args) == 3 && strings.EqualFold(args[1], "as"): + stageName = strings.ToLower(args[2]) + if ok, _ := regexp.MatchString("^[a-z][a-z0-9-_\\.]*$", stageName); !ok { + return "", errors.Errorf("invalid name for build stage: %q, name can't start with a number or contain symbols", stageName) + } + case len(args) != 1: + return "", errors.New("FROM requires either one or three arguments") + } + + return stageName, nil +} + +func parseOnBuild(req parseRequest) (*OnbuildCommand, error) { + if len(req.args) == 0 { + return nil, errAtLeastOneArgument("ONBUILD") + } + if err := req.flags.Parse(); err != nil { + return nil, err + } + + triggerInstruction := strings.ToUpper(strings.TrimSpace(req.args[0])) + switch strings.ToUpper(triggerInstruction) { + case "ONBUILD": + return nil, errors.New("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed") + case "MAINTAINER", "FROM": + return nil, fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction) + } + + original := regexp.MustCompile(`(?i)^\s*ONBUILD\s*`).ReplaceAllString(req.original, "") + return &OnbuildCommand{ + Expression: original, + withNameAndCode: newWithNameAndCode(req), + }, nil + +} + +func parseWorkdir(req parseRequest) (*WorkdirCommand, error) { + if len(req.args) != 1 { + return nil, errExactlyOneArgument("WORKDIR") + } + + err := req.flags.Parse() + if err != nil { + return nil, err + } + return &WorkdirCommand{ + Path: req.args[0], + withNameAndCode: newWithNameAndCode(req), + }, nil + +} + +func parseShellDependentCommand(req parseRequest, emptyAsNil bool) ShellDependantCmdLine { + args := handleJSONArgs(req.args, req.attributes) + cmd := strslice.StrSlice(args) + if emptyAsNil && len(cmd) == 0 { + cmd = nil + } + return ShellDependantCmdLine{ + CmdLine: cmd, + PrependShell: !req.attributes["json"], + } +} + +func parseRun(req parseRequest) (*RunCommand, error) { + + if err := req.flags.Parse(); err != nil { + return nil, err + } + return &RunCommand{ + ShellDependantCmdLine: parseShellDependentCommand(req, false), + withNameAndCode: newWithNameAndCode(req), + }, nil + +} + +func parseCmd(req parseRequest) (*CmdCommand, error) { + if err := req.flags.Parse(); err != nil { + return nil, err + } + return &CmdCommand{ + ShellDependantCmdLine: parseShellDependentCommand(req, false), + withNameAndCode: newWithNameAndCode(req), + }, nil + +} + +func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) { + if err := req.flags.Parse(); err != nil { + return nil, err + } + + cmd := &EntrypointCommand{ + ShellDependantCmdLine: parseShellDependentCommand(req, true), + withNameAndCode: newWithNameAndCode(req), + } + + return cmd, nil +} + +// parseOptInterval(flag) is the duration of flag.Value, or 0 if +// empty. An error is reported if the value is given and less than minimum duration. +func parseOptInterval(f *Flag) (time.Duration, error) { + s := f.Value + if s == "" { + return 0, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, err + } + if d < container.MinimumDuration { + return 0, fmt.Errorf("Interval %#v cannot be less than %s", f.name, container.MinimumDuration) + } + return d, nil +} +func parseHealthcheck(req parseRequest) (*HealthCheckCommand, error) { + if len(req.args) == 0 { + return nil, errAtLeastOneArgument("HEALTHCHECK") + } + cmd := &HealthCheckCommand{ + withNameAndCode: newWithNameAndCode(req), + } + + typ := strings.ToUpper(req.args[0]) + args := req.args[1:] + if typ == "NONE" { + if len(args) != 0 { + return nil, errors.New("HEALTHCHECK NONE takes no arguments") + } + test := strslice.StrSlice{typ} + cmd.Health = &container.HealthConfig{ + Test: test, + } + } else { + + healthcheck := container.HealthConfig{} + + flInterval := req.flags.AddString("interval", "") + flTimeout := req.flags.AddString("timeout", "") + flStartPeriod := req.flags.AddString("start-period", "") + flRetries := req.flags.AddString("retries", "") + + if err := req.flags.Parse(); err != nil { + return nil, err + } + + switch typ { + case "CMD": + cmdSlice := handleJSONArgs(args, req.attributes) + if len(cmdSlice) == 0 { + return nil, errors.New("Missing command after HEALTHCHECK CMD") + } + + if !req.attributes["json"] { + typ = "CMD-SHELL" + } + + healthcheck.Test = strslice.StrSlice(append([]string{typ}, cmdSlice...)) + default: + return nil, fmt.Errorf("Unknown type %#v in HEALTHCHECK (try CMD)", typ) + } + + interval, err := parseOptInterval(flInterval) + if err != nil { + return nil, err + } + healthcheck.Interval = interval + + timeout, err := parseOptInterval(flTimeout) + if err != nil { + return nil, err + } + healthcheck.Timeout = timeout + + startPeriod, err := parseOptInterval(flStartPeriod) + if err != nil { + return nil, err + } + healthcheck.StartPeriod = startPeriod + + if flRetries.Value != "" { + retries, err := strconv.ParseInt(flRetries.Value, 10, 32) + if err != nil { + return nil, err + } + if retries < 1 { + return nil, fmt.Errorf("--retries must be at least 1 (not %d)", retries) + } + healthcheck.Retries = int(retries) + } else { + healthcheck.Retries = 0 + } + + cmd.Health = &healthcheck + } + return cmd, nil +} + +func parseExpose(req parseRequest) (*ExposeCommand, error) { + portsTab := req.args + + if len(req.args) == 0 { + return nil, errAtLeastOneArgument("EXPOSE") + } + + if err := req.flags.Parse(); err != nil { + return nil, err + } + + sort.Strings(portsTab) + return &ExposeCommand{ + Ports: portsTab, + withNameAndCode: newWithNameAndCode(req), + }, nil +} + +func parseUser(req parseRequest) (*UserCommand, error) { + if len(req.args) != 1 { + return nil, errExactlyOneArgument("USER") + } + + if err := req.flags.Parse(); err != nil { + return nil, err + } + return &UserCommand{ + User: req.args[0], + withNameAndCode: newWithNameAndCode(req), + }, nil +} + +func parseVolume(req parseRequest) (*VolumeCommand, error) { + if len(req.args) == 0 { + return nil, errAtLeastOneArgument("VOLUME") + } + + if err := req.flags.Parse(); err != nil { + return nil, err + } + + cmd := &VolumeCommand{ + withNameAndCode: newWithNameAndCode(req), + } + + for _, v := range req.args { + v = strings.TrimSpace(v) + if v == "" { + return nil, errors.New("VOLUME specified can not be an empty string") + } + cmd.Volumes = append(cmd.Volumes, v) + } + return cmd, nil + +} + +func parseStopSignal(req parseRequest) (*StopSignalCommand, error) { + if len(req.args) != 1 { + return nil, errExactlyOneArgument("STOPSIGNAL") + } + sig := req.args[0] + + cmd := &StopSignalCommand{ + Signal: sig, + withNameAndCode: newWithNameAndCode(req), + } + return cmd, nil + +} + +func parseArg(req parseRequest) (*ArgCommand, error) { + if len(req.args) != 1 { + return nil, errExactlyOneArgument("ARG") + } + + var ( + name string + newValue *string + ) + + arg := req.args[0] + // 'arg' can just be a name or name-value pair. Note that this is different + // from 'env' that handles the split of name and value at the parser level. + // The reason for doing it differently for 'arg' is that we support just + // defining an arg and not assign it a value (while 'env' always expects a + // name-value pair). If possible, it will be good to harmonize the two. + if strings.Contains(arg, "=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts[0]) == 0 { + return nil, errBlankCommandNames("ARG") + } + + name = parts[0] + newValue = &parts[1] + } else { + name = arg + } + + return &ArgCommand{ + Key: name, + Value: newValue, + withNameAndCode: newWithNameAndCode(req), + }, nil +} + +func parseShell(req parseRequest) (*ShellCommand, error) { + if err := req.flags.Parse(); err != nil { + return nil, err + } + shellSlice := handleJSONArgs(req.args, req.attributes) + switch { + case len(shellSlice) == 0: + // SHELL [] + return nil, errAtLeastOneArgument("SHELL") + case req.attributes["json"]: + // SHELL ["powershell", "-command"] + + return &ShellCommand{ + Shell: strslice.StrSlice(shellSlice), + withNameAndCode: newWithNameAndCode(req), + }, nil + default: + // SHELL powershell -command - not JSON + return nil, errNotJSON("SHELL", req.original) + } +} + +func errAtLeastOneArgument(command string) error { + return errors.Errorf("%s requires at least one argument", command) +} + +func errExactlyOneArgument(command string) error { + return errors.Errorf("%s requires exactly one argument", command) +} + +func errAtLeastTwoArguments(command string) error { + return errors.Errorf("%s requires at least two arguments", command) +} + +func errBlankCommandNames(command string) error { + return errors.Errorf("%s names can not be blank", command) +} + +func errTooManyArguments(command string) error { + return errors.Errorf("Bad input to %s, too many arguments", command) +} diff --git a/components/engine/builder/dockerfile/instructions/parse_test.go b/components/engine/builder/dockerfile/instructions/parse_test.go new file mode 100644 index 0000000000..bf41b1a266 --- /dev/null +++ b/components/engine/builder/dockerfile/instructions/parse_test.go @@ -0,0 +1,204 @@ +package instructions + +import ( + "strings" + "testing" + + "github.com/docker/docker/builder/dockerfile/command" + "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCommandsExactlyOneArgument(t *testing.T) { + commands := []string{ + "MAINTAINER", + "WORKDIR", + "USER", + "STOPSIGNAL", + } + + for _, command := range commands { + ast, err := parser.Parse(strings.NewReader(command)) + require.NoError(t, err) + _, err = ParseInstruction(ast.AST.Children[0]) + assert.EqualError(t, err, errExactlyOneArgument(command).Error()) + } +} + +func TestCommandsAtLeastOneArgument(t *testing.T) { + commands := []string{ + "ENV", + "LABEL", + "ONBUILD", + "HEALTHCHECK", + "EXPOSE", + "VOLUME", + } + + for _, command := range commands { + ast, err := parser.Parse(strings.NewReader(command)) + require.NoError(t, err) + _, err = ParseInstruction(ast.AST.Children[0]) + assert.EqualError(t, err, errAtLeastOneArgument(command).Error()) + } +} + +func TestCommandsAtLeastTwoArgument(t *testing.T) { + commands := []string{ + "ADD", + "COPY", + } + + for _, command := range commands { + ast, err := parser.Parse(strings.NewReader(command + " arg1")) + require.NoError(t, err) + _, err = ParseInstruction(ast.AST.Children[0]) + assert.EqualError(t, err, errAtLeastTwoArguments(command).Error()) + } +} + +func TestCommandsTooManyArguments(t *testing.T) { + commands := []string{ + "ENV", + "LABEL", + } + + for _, command := range commands { + node := &parser.Node{ + Original: command + "arg1 arg2 arg3", + Value: strings.ToLower(command), + Next: &parser.Node{ + Value: "arg1", + Next: &parser.Node{ + Value: "arg2", + Next: &parser.Node{ + Value: "arg3", + }, + }, + }, + } + _, err := ParseInstruction(node) + assert.EqualError(t, err, errTooManyArguments(command).Error()) + } +} + +func TestCommandsBlankNames(t *testing.T) { + commands := []string{ + "ENV", + "LABEL", + } + + for _, command := range commands { + node := &parser.Node{ + Original: command + " =arg2", + Value: strings.ToLower(command), + Next: &parser.Node{ + Value: "", + Next: &parser.Node{ + Value: "arg2", + }, + }, + } + _, err := ParseInstruction(node) + assert.EqualError(t, err, errBlankCommandNames(command).Error()) + } +} + +func TestHealthCheckCmd(t *testing.T) { + node := &parser.Node{ + Value: command.Healthcheck, + Next: &parser.Node{ + Value: "CMD", + Next: &parser.Node{ + Value: "hello", + Next: &parser.Node{ + Value: "world", + }, + }, + }, + } + cmd, err := ParseInstruction(node) + assert.NoError(t, err) + hc, ok := cmd.(*HealthCheckCommand) + assert.True(t, ok) + expected := []string{"CMD-SHELL", "hello world"} + assert.Equal(t, expected, hc.Health.Test) +} + +func TestParseOptInterval(t *testing.T) { + flInterval := &Flag{ + name: "interval", + flagType: stringType, + Value: "50ns", + } + _, err := parseOptInterval(flInterval) + testutil.ErrorContains(t, err, "cannot be less than 1ms") + + flInterval.Value = "1ms" + _, err = parseOptInterval(flInterval) + require.NoError(t, err) +} + +func TestErrorCases(t *testing.T) { + cases := []struct { + name string + dockerfile string + expectedError string + }{ + { + name: "copyEmptyWhitespace", + dockerfile: `COPY + quux \ + bar`, + expectedError: "COPY requires at least two arguments", + }, + { + name: "ONBUILD forbidden FROM", + dockerfile: "ONBUILD FROM scratch", + expectedError: "FROM isn't allowed as an ONBUILD trigger", + }, + { + name: "ONBUILD forbidden MAINTAINER", + dockerfile: "ONBUILD MAINTAINER docker.io", + expectedError: "MAINTAINER isn't allowed as an ONBUILD trigger", + }, + { + name: "ARG two arguments", + dockerfile: "ARG foo bar", + expectedError: "ARG requires exactly one argument", + }, + { + name: "MAINTAINER unknown flag", + dockerfile: "MAINTAINER --boo joe@example.com", + expectedError: "Unknown flag: boo", + }, + { + name: "Chaining ONBUILD", + dockerfile: `ONBUILD ONBUILD RUN touch foobar`, + expectedError: "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed", + }, + { + name: "Invalid instruction", + dockerfile: `foo bar`, + expectedError: "unknown instruction: FOO", + }, + } + for _, c := range cases { + r := strings.NewReader(c.dockerfile) + ast, err := parser.Parse(r) + + if err != nil { + t.Fatalf("Error when parsing Dockerfile: %s", err) + } + n := ast.AST.Children[0] + _, err = ParseInstruction(n) + if err != nil { + testutil.ErrorContains(t, err, c.expectedError) + return + } + t.Fatalf("No error when executing test %s", c.name) + } + +} diff --git a/components/engine/builder/dockerfile/support.go b/components/engine/builder/dockerfile/instructions/support.go similarity index 96% rename from components/engine/builder/dockerfile/support.go rename to components/engine/builder/dockerfile/instructions/support.go index e87588910b..beefe775ce 100644 --- a/components/engine/builder/dockerfile/support.go +++ b/components/engine/builder/dockerfile/instructions/support.go @@ -1,4 +1,4 @@ -package dockerfile +package instructions import "strings" diff --git a/components/engine/builder/dockerfile/support_test.go b/components/engine/builder/dockerfile/instructions/support_test.go similarity index 98% rename from components/engine/builder/dockerfile/support_test.go rename to components/engine/builder/dockerfile/instructions/support_test.go index 7cc6fe9dcb..2b888dca0d 100644 --- a/components/engine/builder/dockerfile/support_test.go +++ b/components/engine/builder/dockerfile/instructions/support_test.go @@ -1,4 +1,4 @@ -package dockerfile +package instructions import "testing" diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index 04ed6dc337..4c82e61588 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -124,7 +124,6 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta } dispatchState.imageID = imageID - b.buildStages.update(imageID) return nil } @@ -164,7 +163,6 @@ func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runC state.imageID = exportedImage.ImageID() b.imageSources.Add(newImageMount(exportedImage, newLayer)) - b.buildStages.update(state.imageID) return nil } @@ -460,7 +458,6 @@ func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container. fmt.Fprint(b.Stdout, " ---> Using cache\n") dispatchState.imageID = cachedID - b.buildStages.update(dispatchState.imageID) return true, nil } diff --git a/components/engine/integration-cli/docker_api_build_test.go b/components/engine/integration-cli/docker_api_build_test.go index 59b451024e..de78da465e 100644 --- a/components/engine/integration-cli/docker_api_build_test.go +++ b/components/engine/integration-cli/docker_api_build_test.go @@ -438,6 +438,82 @@ func (s *DockerSuite) TestBuildChownOnCopy(c *check.C) { assert.Contains(c, string(out), "Successfully built") } +func (s *DockerSuite) TestBuildCopyCacheOnFileChange(c *check.C) { + + dockerfile := `FROM busybox +COPY file /file` + + ctx1 := fakecontext.New(c, "", + fakecontext.WithDockerfile(dockerfile), + fakecontext.WithFile("file", "foo")) + ctx2 := fakecontext.New(c, "", + fakecontext.WithDockerfile(dockerfile), + fakecontext.WithFile("file", "bar")) + + var build = func(ctx *fakecontext.Fake) string { + res, body, err := request.Post("/build", + request.RawContent(ctx.AsTarReader(c)), + request.ContentType("application/x-tar")) + + require.NoError(c, err) + assert.Equal(c, http.StatusOK, res.StatusCode) + + out, err := request.ReadBody(body) + + ids := getImageIDsFromBuild(c, out) + return ids[len(ids)-1] + } + + id1 := build(ctx1) + id2 := build(ctx1) + id3 := build(ctx2) + + if id1 != id2 { + c.Fatal("didn't use the cache") + } + if id1 == id3 { + c.Fatal("COPY With different source file should not share same cache") + } +} + +func (s *DockerSuite) TestBuildAddCacheOnFileChange(c *check.C) { + + dockerfile := `FROM busybox +ADD file /file` + + ctx1 := fakecontext.New(c, "", + fakecontext.WithDockerfile(dockerfile), + fakecontext.WithFile("file", "foo")) + ctx2 := fakecontext.New(c, "", + fakecontext.WithDockerfile(dockerfile), + fakecontext.WithFile("file", "bar")) + + var build = func(ctx *fakecontext.Fake) string { + res, body, err := request.Post("/build", + request.RawContent(ctx.AsTarReader(c)), + request.ContentType("application/x-tar")) + + require.NoError(c, err) + assert.Equal(c, http.StatusOK, res.StatusCode) + + out, err := request.ReadBody(body) + + ids := getImageIDsFromBuild(c, out) + return ids[len(ids)-1] + } + + id1 := build(ctx1) + id2 := build(ctx1) + id3 := build(ctx2) + + if id1 != id2 { + c.Fatal("didn't use the cache") + } + if id1 == id3 { + c.Fatal("COPY With different source file should not share same cache") + } +} + func (s *DockerSuite) TestBuildWithSession(c *check.C) { testRequires(c, ExperimentalDaemon) diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 59213e5404..f6ab1923b9 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -1173,12 +1173,13 @@ func (s *DockerSuite) TestBuildForceRm(c *check.C) { containerCountBefore := getContainerCount(c) name := "testbuildforcerm" - buildImage(name, cli.WithFlags("--force-rm"), build.WithBuildContext(c, - build.WithFile("Dockerfile", `FROM `+minimalBaseImage()+` + r := buildImage(name, cli.WithFlags("--force-rm"), build.WithBuildContext(c, + build.WithFile("Dockerfile", `FROM busybox RUN true - RUN thiswillfail`))).Assert(c, icmd.Expected{ - ExitCode: 1, - }) + RUN thiswillfail`))) + if r.ExitCode != 1 && r.ExitCode != 127 { // different on Linux / Windows + c.Fatalf("Wrong exit code") + } containerCountAfter := getContainerCount(c) if containerCountBefore != containerCountAfter { @@ -4542,7 +4543,6 @@ func (s *DockerSuite) TestBuildBuildTimeArgOverrideEnvDefinedBeforeArg(c *check. } func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) { - testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldvarstest" wdVar := "WDIR" @@ -4559,6 +4559,10 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) { userVal := "testUser" volVar := "VOL" volVal := "/testVol/" + if DaemonIsWindows() { + volVal = "C:\\testVol" + wdVal = "C:\\tmp" + } buildImageSuccessfully(c, imgName, cli.WithFlags( @@ -4594,7 +4598,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) { ) res := inspectField(c, imgName, "Config.WorkingDir") - c.Check(res, check.Equals, filepath.ToSlash(wdVal)) + c.Check(filepath.ToSlash(res), check.Equals, filepath.ToSlash(wdVal)) var resArr []string inspectFieldAndUnmarshall(c, imgName, "Config.Env", &resArr) From b07dcb72fbe5bfbecc240c3b070ced606f1c5f28 Mon Sep 17 00:00:00 2001 From: Shukui Yang Date: Tue, 19 Sep 2017 01:25:39 +0000 Subject: [PATCH 05/34] Close pipe if mountFrom failed. Signed-off-by: Shukui Yang Upstream-commit: 9f38923901352459bb621d0b3587a6517e67eeb3 Component: engine --- components/engine/daemon/graphdriver/overlay2/mount.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/daemon/graphdriver/overlay2/mount.go b/components/engine/daemon/graphdriver/overlay2/mount.go index 77bff06621..e59178c669 100644 --- a/components/engine/daemon/graphdriver/overlay2/mount.go +++ b/components/engine/daemon/graphdriver/overlay2/mount.go @@ -49,18 +49,19 @@ func mountFrom(dir, device, target, mType string, flags uintptr, label string) e output := bytes.NewBuffer(nil) cmd.Stdout = output cmd.Stderr = output - if err := cmd.Start(); err != nil { + w.Close() return fmt.Errorf("mountfrom error on re-exec cmd: %v", err) } //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { + w.Close() return fmt.Errorf("mountfrom json encode to pipe failed: %v", err) } w.Close() if err := cmd.Wait(); err != nil { - return fmt.Errorf("mountfrom re-exec error: %v: output: %s", err, output) + return fmt.Errorf("mountfrom re-exec error: %v: output: %v", err, output) } return nil } From f7daf26c0feb70eeaddcb20a0cef5eff9da294e2 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 30 Aug 2017 13:10:15 -0400 Subject: [PATCH 06/34] Set selinux label on local volumes from mounts API When using a volume via the `Binds` API, a shared selinux label is automatically set. The `Mounts` API is not setting this, which makes volumes specified via the mounts API useless when selinux is enabled. This fix adopts the same selinux label for volumes on the mounts API as on binds. Note in the case of both the `Binds` API and the `Mounts` API, the selinux label is only applied when the volume driver is the `local` driver. Signed-off-by: Brian Goff Upstream-commit: 5bbf5cc671ec8007bf8e0416799fff01d6a79b7e Component: engine --- components/engine/daemon/volumes.go | 3 + .../docker_api_containers_test.go | 97 ++++++++++++------- components/engine/volume/local/local.go | 5 + 3 files changed, 71 insertions(+), 34 deletions(-) diff --git a/components/engine/daemon/volumes.go b/components/engine/daemon/volumes.go index 03bfda6f5d..d5d31b260c 100644 --- a/components/engine/daemon/volumes.go +++ b/components/engine/daemon/volumes.go @@ -207,6 +207,9 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo }); ok { mp.Source = cv.CachedPath() } + if mp.Driver == volume.DefaultDriverName { + setBindModeIfNull(mp) + } } binds[mp.Destination] = true diff --git a/components/engine/integration-cli/docker_api_containers_test.go b/components/engine/integration-cli/docker_api_containers_test.go index 173d5f80b0..cd3566f4b4 100644 --- a/components/engine/integration-cli/docker_api_containers_test.go +++ b/components/engine/integration-cli/docker_api_containers_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strconv" "strings" "time" @@ -1901,33 +1902,37 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) { } type testCase struct { - cfg mounttypes.Mount + spec mounttypes.Mount expected types.MountPoint } + var selinuxSharedLabel string + if runtime.GOOS == "linux" { + selinuxSharedLabel = "z" + } + cases := []testCase{ // use literal strings here for `Type` instead of the defined constants in the volume package to keep this honest // Validation of the actual `Mount` struct is done in another test is not needed here - {mounttypes.Mount{Type: "volume", Target: destPath}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}}, - {mounttypes.Mount{Type: "volume", Target: destPath + slash}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}}, - {mounttypes.Mount{Type: "volume", Target: destPath, Source: "test1"}, types.MountPoint{Type: "volume", Name: "test1", RW: true, Destination: destPath}}, - {mounttypes.Mount{Type: "volume", Target: destPath, ReadOnly: true, Source: "test2"}, types.MountPoint{Type: "volume", Name: "test2", RW: false, Destination: destPath}}, { - mounttypes.Mount{ - Type: "volume", - Target: destPath, - Source: "test3", - VolumeOptions: &mounttypes.VolumeOptions{ - DriverConfig: &mounttypes.Driver{Name: volume.DefaultDriverName}, - }, - }, - types.MountPoint{ - Driver: volume.DefaultDriverName, - Type: "volume", - Name: "test3", - RW: true, - Destination: destPath, - }, + spec: mounttypes.Mount{Type: "volume", Target: destPath}, + expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath + slash}, + expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath, Source: "test1"}, + expected: types.MountPoint{Type: "volume", Name: "test1", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath, ReadOnly: true, Source: "test2"}, + expected: types.MountPoint{Type: "volume", Name: "test2", RW: false, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath, Source: "test3", VolumeOptions: &mounttypes.VolumeOptions{DriverConfig: &mounttypes.Driver{Name: volume.DefaultDriverName}}}, + expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", Name: "test3", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, }, } @@ -1938,19 +1943,22 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) { defer os.RemoveAll(tmpDir1) cases = append(cases, []testCase{ { - mounttypes.Mount{ + spec: mounttypes.Mount{ Type: "bind", Source: tmpDir1, Target: destPath, }, - types.MountPoint{ + expected: types.MountPoint{ Type: "bind", RW: true, Destination: destPath, Source: tmpDir1, }, }, - {mounttypes.Mount{Type: "bind", Source: tmpDir1, Target: destPath, ReadOnly: true}, types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir1}}, + { + spec: mounttypes.Mount{Type: "bind", Source: tmpDir1, Target: destPath, ReadOnly: true}, + expected: types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir1}, + }, }...) // for modes only supported on Linux @@ -1963,19 +1971,40 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) { c.Assert(mount.ForceMount("", tmpDir3, "none", "shared"), checker.IsNil) cases = append(cases, []testCase{ - {mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath}, types.MountPoint{Type: "bind", RW: true, Destination: destPath, Source: tmpDir3}}, - {mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true}, types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3}}, - {mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true, BindOptions: &mounttypes.BindOptions{Propagation: "shared"}}, types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3, Propagation: "shared"}}, + { + spec: mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath}, + expected: types.MountPoint{Type: "bind", RW: true, Destination: destPath, Source: tmpDir3}, + }, + { + spec: mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true}, + expected: types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3}, + }, + { + spec: mounttypes.Mount{Type: "bind", Source: tmpDir3, Target: destPath, ReadOnly: true, BindOptions: &mounttypes.BindOptions{Propagation: "shared"}}, + expected: types.MountPoint{Type: "bind", RW: false, Destination: destPath, Source: tmpDir3, Propagation: "shared"}, + }, }...) } } if testEnv.DaemonPlatform() != "windows" { // Windows does not support volume populate cases = append(cases, []testCase{ - {mounttypes.Mount{Type: "volume", Target: destPath, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}}, - {mounttypes.Mount{Type: "volume", Target: destPath + slash, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath}}, - {mounttypes.Mount{Type: "volume", Target: destPath, Source: "test4", VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Type: "volume", Name: "test4", RW: true, Destination: destPath}}, - {mounttypes.Mount{Type: "volume", Target: destPath, Source: "test5", ReadOnly: true, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, types.MountPoint{Type: "volume", Name: "test5", RW: false, Destination: destPath}}, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, + expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath + slash, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, + expected: types.MountPoint{Driver: volume.DefaultDriverName, Type: "volume", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath, Source: "test4", VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, + expected: types.MountPoint{Type: "volume", Name: "test4", RW: true, Destination: destPath, Mode: selinuxSharedLabel}, + }, + { + spec: mounttypes.Mount{Type: "volume", Target: destPath, Source: "test5", ReadOnly: true, VolumeOptions: &mounttypes.VolumeOptions{NoCopy: true}}, + expected: types.MountPoint{Type: "volume", Name: "test5", RW: false, Destination: destPath, Mode: selinuxSharedLabel}, + }, }...) } @@ -1990,11 +2019,11 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) { ctx := context.Background() apiclient := testEnv.APIClient() for i, x := range cases { - c.Logf("case %d - config: %v", i, x.cfg) + c.Logf("case %d - config: %v", i, x.spec) container, err := apiclient.ContainerCreate( ctx, &containertypes.Config{Image: testImg}, - &containertypes.HostConfig{Mounts: []mounttypes.Mount{x.cfg}}, + &containertypes.HostConfig{Mounts: []mounttypes.Mount{x.spec}}, &networktypes.NetworkingConfig{}, "") require.NoError(c, err) @@ -2035,12 +2064,12 @@ func (s *DockerSuite) TestContainersAPICreateMountsCreate(c *check.C) { switch { // Named volumes still exist after the container is removed - case x.cfg.Type == "volume" && len(x.cfg.Source) > 0: + case x.spec.Type == "volume" && len(x.spec.Source) > 0: _, err := apiclient.VolumeInspect(ctx, mountPoint.Name) require.NoError(c, err) // Bind mounts are never removed with the container - case x.cfg.Type == "bind": + case x.spec.Type == "bind": // anonymous volumes are removed default: diff --git a/components/engine/volume/local/local.go b/components/engine/volume/local/local.go index c85122d63a..b37c45e61e 100644 --- a/components/engine/volume/local/local.go +++ b/components/engine/volume/local/local.go @@ -334,6 +334,11 @@ func (v *localVolume) Path() string { return v.path } +// CachedPath returns the data location +func (v *localVolume) CachedPath() string { + return v.path +} + // Mount implements the localVolume interface, returning the data location. // If there are any provided mount options, the resources will be mounted at this point func (v *localVolume) Mount(id string) (string, error) { From cb0c1a12c4b2d386b008d2206b31fff5e888358a Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 14 Jul 2017 16:45:32 -0400 Subject: [PATCH 07/34] Decouple plugin manager from libcontainerd package libcontainerd has a bunch of platform dependent code and huge interfaces that are a pain implement. To make the plugin manager a bit easier to work with, extract the plugin executor into an interface and move the containerd implementation to a separate package. Signed-off-by: Brian Goff Upstream-commit: c85e8622a4813d7b72d74517faa03ab5de4c4550 Component: engine --- components/engine/daemon/daemon.go | 7 +- .../integration-cli/fixtures/plugin/plugin.go | 149 ++++++++++++++++ .../fixtures/plugin/plugin_linux.go | 162 ------------------ .../fixtures/plugin/plugin_unsuported.go | 19 -- .../plugin/executor/containerd/containerd.go | 77 +++++++++ components/engine/plugin/manager.go | 112 ++++++------ components/engine/plugin/manager_linux.go | 25 +-- 7 files changed, 298 insertions(+), 253 deletions(-) delete mode 100644 components/engine/integration-cli/fixtures/plugin/plugin_linux.go delete mode 100644 components/engine/integration-cli/fixtures/plugin/plugin_unsuported.go create mode 100644 components/engine/plugin/executor/containerd/containerd.go diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 7208f3c5c4..19d22bf702 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -48,6 +48,7 @@ import ( "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/truncindex" "github.com/docker/docker/plugin" + pluginexec "github.com/docker/docker/plugin/executor/containerd" refstore "github.com/docker/docker/reference" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" @@ -646,12 +647,16 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe } registerMetricsPluginCallback(d.PluginStore, metricsSockPath) + createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) { + return pluginexec.New(containerdRemote, m) + } + // Plugin system initialization should happen before restore. Do not change order. d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{ Root: filepath.Join(config.Root, "plugins"), ExecRoot: getPluginExecRoot(config.Root), Store: d.PluginStore, - Executor: containerdRemote, + CreateExecutor: createPluginExec, RegistryService: registryService, LiveRestoreEnabled: config.LiveRestoreEnabled, LogPluginEvent: d.LogPluginEvent, // todo: make private diff --git a/components/engine/integration-cli/fixtures/plugin/plugin.go b/components/engine/integration-cli/fixtures/plugin/plugin.go index 4ab15c23de..0b13134563 100644 --- a/components/engine/integration-cli/fixtures/plugin/plugin.go +++ b/components/engine/integration-cli/fixtures/plugin/plugin.go @@ -1,9 +1,19 @@ package plugin import ( + "encoding/json" "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "time" "github.com/docker/docker/api/types" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/plugin" + "github.com/docker/docker/registry" + "github.com/pkg/errors" "golang.org/x/net/context" ) @@ -32,3 +42,142 @@ func WithBinary(bin string) CreateOpt { type CreateClient interface { PluginCreate(context.Context, io.Reader, types.PluginCreateOptions) error } + +// Create creates a new plugin with the specified name +func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error { + tmpDir, err := ioutil.TempDir("", "create-test-plugin") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + tar, err := makePluginBundle(tmpDir, opts...) + if err != nil { + return err + } + defer tar.Close() + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + return c.PluginCreate(ctx, tar, types.PluginCreateOptions{RepoName: name}) +} + +// CreateInRegistry makes a plugin (locally) and pushes it to a registry. +// This does not use a dockerd instance to create or push the plugin. +// If you just want to create a plugin in some daemon, use `Create`. +// +// This can be useful when testing plugins on swarm where you don't really want +// the plugin to exist on any of the daemons (immediately) and there needs to be +// some way to distribute the plugin. +func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error { + tmpDir, err := ioutil.TempDir("", "create-test-plugin-local") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + inPath := filepath.Join(tmpDir, "plugin") + if err := os.MkdirAll(inPath, 0755); err != nil { + return errors.Wrap(err, "error creating plugin root") + } + + tar, err := makePluginBundle(inPath, opts...) + if err != nil { + return err + } + defer tar.Close() + + dummyExec := func(m *plugin.Manager) (plugin.Executor, error) { + return nil, nil + } + + regService, err := registry.NewService(registry.ServiceOptions{V2Only: true}) + if err != nil { + return err + } + + managerConfig := plugin.ManagerConfig{ + Store: plugin.NewStore(), + RegistryService: regService, + Root: filepath.Join(tmpDir, "root"), + ExecRoot: "/run/docker", // manager init fails if not set + CreateExecutor: dummyExec, + LogPluginEvent: func(id, name, action string) {}, // panics when not set + } + manager, err := plugin.NewManager(managerConfig) + if err != nil { + return errors.Wrap(err, "error creating plugin manager") + } + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := manager.CreateFromContext(ctx, tar, &types.PluginCreateOptions{RepoName: repo}); err != nil { + return err + } + + if auth == nil { + auth = &types.AuthConfig{} + } + err = manager.Push(ctx, repo, nil, auth, ioutil.Discard) + return errors.Wrap(err, "error pushing plugin") +} + +func makePluginBundle(inPath string, opts ...CreateOpt) (io.ReadCloser, error) { + p := &types.PluginConfig{ + Interface: types.PluginConfigInterface{ + Socket: "basic.sock", + Types: []types.PluginInterfaceType{{Capability: "docker.dummy/1.0"}}, + }, + Entrypoint: []string{"/basic"}, + } + cfg := &Config{ + PluginConfig: p, + } + for _, o := range opts { + o(cfg) + } + if cfg.binPath == "" { + binPath, err := ensureBasicPluginBin() + if err != nil { + return nil, err + } + cfg.binPath = binPath + } + + configJSON, err := json.Marshal(p) + if err != nil { + return nil, err + } + if err := ioutil.WriteFile(filepath.Join(inPath, "config.json"), configJSON, 0644); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(inPath, "rootfs", filepath.Dir(p.Entrypoint[0])), 0755); err != nil { + return nil, errors.Wrap(err, "error creating plugin rootfs dir") + } + if err := archive.NewDefaultArchiver().CopyFileWithTar(cfg.binPath, filepath.Join(inPath, "rootfs", p.Entrypoint[0])); err != nil { + return nil, errors.Wrap(err, "error copying plugin binary to rootfs path") + } + tar, err := archive.Tar(inPath, archive.Uncompressed) + return tar, errors.Wrap(err, "error making plugin archive") +} + +func ensureBasicPluginBin() (string, error) { + name := "docker-basic-plugin" + p, err := exec.LookPath(name) + if err == nil { + return p, nil + } + + goBin, err := exec.LookPath("go") + if err != nil { + return "", err + } + installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name) + cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("fixtures", "plugin", "basic")) + cmd.Env = append(cmd.Env, "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + return "", errors.Wrapf(err, "error building basic plugin bin: %s", string(out)) + } + return installPath, nil +} diff --git a/components/engine/integration-cli/fixtures/plugin/plugin_linux.go b/components/engine/integration-cli/fixtures/plugin/plugin_linux.go deleted file mode 100644 index 5da79fcb77..0000000000 --- a/components/engine/integration-cli/fixtures/plugin/plugin_linux.go +++ /dev/null @@ -1,162 +0,0 @@ -package plugin - -import ( - "encoding/json" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/libcontainerd" - "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/plugin" - "github.com/docker/docker/registry" - "github.com/pkg/errors" - "golang.org/x/net/context" -) - -// Create creates a new plugin with the specified name -func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error { - tmpDir, err := ioutil.TempDir("", "create-test-plugin") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - - tar, err := makePluginBundle(tmpDir, opts...) - if err != nil { - return err - } - defer tar.Close() - - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - return c.PluginCreate(ctx, tar, types.PluginCreateOptions{RepoName: name}) -} - -// TODO(@cpuguy83): we really shouldn't have to do this... -// The manager panics on init when `Executor` is not set. -type dummyExecutor struct{} - -func (dummyExecutor) Client(libcontainerd.Backend) (libcontainerd.Client, error) { return nil, nil } -func (dummyExecutor) Cleanup() {} -func (dummyExecutor) UpdateOptions(...libcontainerd.RemoteOption) error { return nil } - -// CreateInRegistry makes a plugin (locally) and pushes it to a registry. -// This does not use a dockerd instance to create or push the plugin. -// If you just want to create a plugin in some daemon, use `Create`. -// -// This can be useful when testing plugins on swarm where you don't really want -// the plugin to exist on any of the daemons (immediately) and there needs to be -// some way to distribute the plugin. -func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error { - tmpDir, err := ioutil.TempDir("", "create-test-plugin-local") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - - inPath := filepath.Join(tmpDir, "plugin") - if err := os.MkdirAll(inPath, 0755); err != nil { - return errors.Wrap(err, "error creating plugin root") - } - - tar, err := makePluginBundle(inPath, opts...) - if err != nil { - return err - } - defer tar.Close() - - regService, err := registry.NewService(registry.ServiceOptions{V2Only: true}) - if err != nil { - return err - } - - managerConfig := plugin.ManagerConfig{ - Store: plugin.NewStore(), - RegistryService: regService, - Root: filepath.Join(tmpDir, "root"), - ExecRoot: "/run/docker", // manager init fails if not set - Executor: dummyExecutor{}, - LogPluginEvent: func(id, name, action string) {}, // panics when not set - } - manager, err := plugin.NewManager(managerConfig) - if err != nil { - return errors.Wrap(err, "error creating plugin manager") - } - - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - if err := manager.CreateFromContext(ctx, tar, &types.PluginCreateOptions{RepoName: repo}); err != nil { - return err - } - - if auth == nil { - auth = &types.AuthConfig{} - } - err = manager.Push(ctx, repo, nil, auth, ioutil.Discard) - return errors.Wrap(err, "error pushing plugin") -} - -func makePluginBundle(inPath string, opts ...CreateOpt) (io.ReadCloser, error) { - p := &types.PluginConfig{ - Interface: types.PluginConfigInterface{ - Socket: "basic.sock", - Types: []types.PluginInterfaceType{{Capability: "docker.dummy/1.0"}}, - }, - Entrypoint: []string{"/basic"}, - } - cfg := &Config{ - PluginConfig: p, - } - for _, o := range opts { - o(cfg) - } - if cfg.binPath == "" { - binPath, err := ensureBasicPluginBin() - if err != nil { - return nil, err - } - cfg.binPath = binPath - } - - configJSON, err := json.Marshal(p) - if err != nil { - return nil, err - } - if err := ioutil.WriteFile(filepath.Join(inPath, "config.json"), configJSON, 0644); err != nil { - return nil, err - } - if err := os.MkdirAll(filepath.Join(inPath, "rootfs", filepath.Dir(p.Entrypoint[0])), 0755); err != nil { - return nil, errors.Wrap(err, "error creating plugin rootfs dir") - } - if err := archive.NewDefaultArchiver().CopyFileWithTar(cfg.binPath, filepath.Join(inPath, "rootfs", p.Entrypoint[0])); err != nil { - return nil, errors.Wrap(err, "error copying plugin binary to rootfs path") - } - tar, err := archive.Tar(inPath, archive.Uncompressed) - return tar, errors.Wrap(err, "error making plugin archive") -} - -func ensureBasicPluginBin() (string, error) { - name := "docker-basic-plugin" - p, err := exec.LookPath(name) - if err == nil { - return p, nil - } - - goBin, err := exec.LookPath("go") - if err != nil { - return "", err - } - installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name) - cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("fixtures", "plugin", "basic")) - cmd.Env = append(cmd.Env, "CGO_ENABLED=0") - if out, err := cmd.CombinedOutput(); err != nil { - return "", errors.Wrapf(err, "error building basic plugin bin: %s", string(out)) - } - return installPath, nil -} diff --git a/components/engine/integration-cli/fixtures/plugin/plugin_unsuported.go b/components/engine/integration-cli/fixtures/plugin/plugin_unsuported.go deleted file mode 100644 index 7c272a317f..0000000000 --- a/components/engine/integration-cli/fixtures/plugin/plugin_unsuported.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build !linux - -package plugin - -import ( - "github.com/docker/docker/api/types" - "github.com/pkg/errors" - "golang.org/x/net/context" -) - -// Create is not supported on this platform -func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error { - return errors.New("not supported on this platform") -} - -// CreateInRegistry is not supported on this platform -func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error { - return errors.New("not supported on this platform") -} diff --git a/components/engine/plugin/executor/containerd/containerd.go b/components/engine/plugin/executor/containerd/containerd.go new file mode 100644 index 0000000000..74cf530cf1 --- /dev/null +++ b/components/engine/plugin/executor/containerd/containerd.go @@ -0,0 +1,77 @@ +package containerd + +import ( + "io" + + "github.com/docker/docker/libcontainerd" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +// ExitHandler represents an object that is called when the exit event is received from containerd +type ExitHandler interface { + HandleExitEvent(id string) error +} + +// New creates a new containerd plugin executor +func New(remote libcontainerd.Remote, exitHandler ExitHandler) (*Executor, error) { + e := &Executor{exitHandler: exitHandler} + client, err := remote.Client(e) + if err != nil { + return nil, errors.Wrap(err, "error creating containerd exec client") + } + e.client = client + return e, nil +} + +// Executor is the containerd client implementation of a plugin executor +type Executor struct { + client libcontainerd.Client + exitHandler ExitHandler +} + +// Create creates a new container +func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error { + return e.client.Create(id, "", "", spec, attachStreamsFunc(stdout, stderr)) +} + +// Restore restores a container +func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) error { + return e.client.Restore(id, attachStreamsFunc(stdout, stderr)) +} + +// IsRunning returns if the container with the given id is running +func (e *Executor) IsRunning(id string) (bool, error) { + pids, err := e.client.GetPidsForContainer(id) + return len(pids) > 0, err +} + +// Signal sends the specified signal to the container +func (e *Executor) Signal(id string, signal int) error { + return e.client.Signal(id, signal) +} + +// StateChanged handles state changes from containerd +// All events are ignored except the exit event, which is sent of to the stored handler +func (e *Executor) StateChanged(id string, event libcontainerd.StateInfo) error { + switch event.State { + case libcontainerd.StateExit: + return e.exitHandler.HandleExitEvent(id) + } + return nil +} + +func attachStreamsFunc(stdout, stderr io.WriteCloser) func(libcontainerd.IOPipe) error { + return func(iop libcontainerd.IOPipe) error { + iop.Stdin.Close() + go func() { + io.Copy(stdout, iop.Stdout) + stdout.Close() + }() + go func() { + io.Copy(stderr, iop.Stderr) + stderr.Close() + }() + return nil + } +} diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index 2281dfdd6c..0c03192d76 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -17,7 +17,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/image" "github.com/docker/docker/layer" - "github.com/docker/docker/libcontainerd" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/mount" @@ -26,6 +25,7 @@ import ( "github.com/docker/docker/plugin/v2" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -35,6 +35,14 @@ const rootFSFileName = "rootfs" var validFullID = regexp.MustCompile(`^([a-f0-9]{64})$`) +// Executor is the interface that the plugin manager uses to interact with for starting/stopping plugins +type Executor interface { + Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error + Restore(id string, stdout, stderr io.WriteCloser) error + IsRunning(id string) (bool, error) + Signal(id string, signal int) error +} + func (pm *Manager) restorePlugin(p *v2.Plugin) error { if p.IsEnabled() { return pm.restore(p) @@ -47,24 +55,27 @@ type eventLogger func(id, name, action string) // ManagerConfig defines configuration needed to start new manager. type ManagerConfig struct { Store *Store // remove - Executor libcontainerd.Remote RegistryService registry.Service LiveRestoreEnabled bool // TODO: remove LogPluginEvent eventLogger Root string ExecRoot string + CreateExecutor ExecutorCreator AuthzMiddleware *authorization.Middleware } +// ExecutorCreator is used in the manager config to pass in an `Executor` +type ExecutorCreator func(*Manager) (Executor, error) + // Manager controls the plugin subsystem. type Manager struct { - config ManagerConfig - mu sync.RWMutex // protects cMap - muGC sync.RWMutex // protects blobstore deletions - cMap map[*v2.Plugin]*controller - containerdClient libcontainerd.Client - blobStore *basicBlobStore - publisher *pubsub.Publisher + config ManagerConfig + mu sync.RWMutex // protects cMap + muGC sync.RWMutex // protects blobstore deletions + cMap map[*v2.Plugin]*controller + blobStore *basicBlobStore + publisher *pubsub.Publisher + executor Executor } // controller represents the manager's control on a plugin. @@ -111,10 +122,11 @@ func NewManager(config ManagerConfig) (*Manager, error) { } var err error - manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct + manager.executor, err = config.CreateExecutor(manager) if err != nil { - return nil, errors.Wrap(err, "failed to create containerd client") + return nil, err } + manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs")) if err != nil { return nil, err @@ -133,42 +145,37 @@ func (pm *Manager) tmpDir() string { return filepath.Join(pm.config.Root, "tmp") } -// StateChanged updates plugin internals using libcontainerd events. -func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error { - logrus.Debugf("plugin state changed %s %#v", id, e) +// HandleExitEvent is called when the executor receives the exit event +// In the future we may change this, but for now all we care about is the exit event. +func (pm *Manager) HandleExitEvent(id string) error { + p, err := pm.config.Store.GetV2Plugin(id) + if err != nil { + return err + } - switch e.State { - case libcontainerd.StateExit: - p, err := pm.config.Store.GetV2Plugin(id) - if err != nil { - return err + os.RemoveAll(filepath.Join(pm.config.ExecRoot, id)) + + if p.PropagatedMount != "" { + if err := mount.Unmount(p.PropagatedMount); err != nil { + logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err) } - - os.RemoveAll(filepath.Join(pm.config.ExecRoot, id)) - - if p.PropagatedMount != "" { - if err := mount.Unmount(p.PropagatedMount); err != nil { - logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err) - } - propRoot := filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount") - if err := mount.Unmount(propRoot); err != nil { - logrus.Warn("Could not unmount %s: %v", propRoot, err) - } - } - - pm.mu.RLock() - c := pm.cMap[p] - if c.exitChan != nil { - close(c.exitChan) - } - restart := c.restart - pm.mu.RUnlock() - - if restart { - pm.enable(p, c, true) + propRoot := filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount") + if err := mount.Unmount(propRoot); err != nil { + logrus.Warn("Could not unmount %s: %v", propRoot, err) } } + pm.mu.RLock() + c := pm.cMap[p] + if c.exitChan != nil { + close(c.exitChan) + } + restart := c.restart + pm.mu.RUnlock() + + if restart { + pm.enable(p, c, true) + } return nil } @@ -333,23 +340,10 @@ func (l logHook) Fire(entry *logrus.Entry) error { return nil } -func attachToLog(id string) func(libcontainerd.IOPipe) error { - return func(iop libcontainerd.IOPipe) error { - iop.Stdin.Close() - - logger := logrus.New() - logger.Hooks.Add(logHook{id}) - // TODO: cache writer per id - w := logger.Writer() - go func() { - io.Copy(w, iop.Stdout) - }() - go func() { - // TODO: update logrus and use logger.WriterLevel - io.Copy(w, iop.Stderr) - }() - return nil - } +func makeLoggerStreams(id string) (stdout, stderr io.WriteCloser) { + logger := logrus.New() + logger.Hooks.Add(logHook{id}) + return logger.WriterLevel(logrus.InfoLevel), logger.WriterLevel(logrus.ErrorLevel) } func validatePrivileges(requiredPrivileges, privileges types.PluginPrivileges) error { diff --git a/components/engine/plugin/manager_linux.go b/components/engine/plugin/manager_linux.go index 7c832b55b2..beefc3dfba 100644 --- a/components/engine/plugin/manager_linux.go +++ b/components/engine/plugin/manager_linux.go @@ -11,7 +11,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/daemon/initlayer" - "github.com/docker/docker/libcontainerd" "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" @@ -63,7 +62,8 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { return errors.WithStack(err) } - if err := pm.containerdClient.Create(p.GetID(), "", "", *spec, attachToLog(p.GetID())); err != nil { + stdout, stderr := makeLoggerStreams(p.GetID()) + if err := pm.executor.Create(p.GetID(), *spec, stdout, stderr); err != nil { if p.PropagatedMount != "" { if err := mount.Unmount(p.PropagatedMount); err != nil { logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err) @@ -83,7 +83,7 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error { client, err := plugins.NewClientWithTimeout("unix://"+sockAddr, nil, time.Duration(c.timeoutInSecs)*time.Second) if err != nil { c.restart = false - shutdownPlugin(p, c, pm.containerdClient) + shutdownPlugin(p, c, pm.executor) return errors.WithStack(err) } @@ -109,7 +109,7 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error { c.restart = false // While restoring plugins, we need to explicitly set the state to disabled pm.config.Store.SetState(p, false) - shutdownPlugin(p, c, pm.containerdClient) + shutdownPlugin(p, c, pm.executor) return err } @@ -121,13 +121,14 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error { } func (pm *Manager) restore(p *v2.Plugin) error { - if err := pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())); err != nil { + stdout, stderr := makeLoggerStreams(p.GetID()) + if err := pm.executor.Restore(p.GetID(), stdout, stderr); err != nil { return err } if pm.config.LiveRestoreEnabled { c := &controller{} - if pids, _ := pm.containerdClient.GetPidsForContainer(p.GetID()); len(pids) == 0 { + if isRunning, _ := pm.executor.IsRunning(p.GetID()); !isRunning { // plugin is not running, so follow normal startup procedure return pm.enable(p, c, true) } @@ -143,10 +144,10 @@ func (pm *Manager) restore(p *v2.Plugin) error { return nil } -func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.Client) { +func shutdownPlugin(p *v2.Plugin, c *controller, executor Executor) { pluginID := p.GetID() - err := containerdClient.Signal(pluginID, int(unix.SIGTERM)) + err := executor.Signal(pluginID, int(unix.SIGTERM)) if err != nil { logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err) } else { @@ -155,7 +156,7 @@ func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd. logrus.Debug("Clean shutdown of plugin") case <-time.After(time.Second * 10): logrus.Debug("Force shutdown plugin") - if err := containerdClient.Signal(pluginID, int(unix.SIGKILL)); err != nil { + if err := executor.Signal(pluginID, int(unix.SIGKILL)); err != nil { logrus.Errorf("Sending SIGKILL to plugin failed with error: %v", err) } } @@ -175,7 +176,7 @@ func (pm *Manager) disable(p *v2.Plugin, c *controller) error { } c.restart = false - shutdownPlugin(p, c, pm.containerdClient) + shutdownPlugin(p, c, pm.executor) pm.config.Store.SetState(p, false) return pm.save(p) } @@ -192,9 +193,9 @@ func (pm *Manager) Shutdown() { logrus.Debug("Plugin active when liveRestore is set, skipping shutdown") continue } - if pm.containerdClient != nil && p.IsEnabled() { + if pm.executor != nil && p.IsEnabled() { c.restart = false - shutdownPlugin(p, c, pm.containerdClient) + shutdownPlugin(p, c, pm.executor) } } mount.Unmount(pm.config.Root) From ddb0ee3757a93c291e57450a70694d631634088e Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 7 Sep 2017 17:02:17 -0700 Subject: [PATCH 08/34] Revendor Microsoft/opengcs @ v0.3.4 Signed-off-by: John Howard Upstream-commit: 2798576b37aa99643a06366f00072b6026c0b77e Component: engine --- .../engine/libcontainerd/client_windows.go | 4 +- .../engine/libcontainerd/container_windows.go | 4 +- .../engine/libcontainerd/utils_windows.go | 2 +- components/engine/vendor.conf | 2 +- .../opengcs/client/createext4vhdx.go | 2 + .../Microsoft/opengcs/client/hotaddvhd.go | 2 + .../Microsoft/opengcs/client/hotremovevhd.go | 2 + .../Microsoft/opengcs/client/process.go | 45 +++++++++++++++++++ .../Microsoft/opengcs/client/tartovhd.go | 2 + .../Microsoft/opengcs/client/vhdtotar.go | 2 + 10 files changed, 59 insertions(+), 8 deletions(-) diff --git a/components/engine/libcontainerd/client_windows.go b/components/engine/libcontainerd/client_windows.go index a721b4a8cc..03e4eeb004 100644 --- a/components/engine/libcontainerd/client_windows.go +++ b/components/engine/libcontainerd/client_windows.go @@ -440,9 +440,7 @@ func (clnt *client) AddProcess(ctx context.Context, containerID, processFriendly return -1, err } - defer func() { - container.debugGCS() - }() + defer container.debugGCS() // Note we always tell HCS to // create stdout as it's required regardless of '-i' or '-t' options, so that diff --git a/components/engine/libcontainerd/container_windows.go b/components/engine/libcontainerd/container_windows.go index 5eeb4736f6..73fc6bd41b 100644 --- a/components/engine/libcontainerd/container_windows.go +++ b/components/engine/libcontainerd/container_windows.go @@ -59,9 +59,7 @@ func (ctr *container) start(attachStdio StdioCallback) error { return err } - defer func() { - ctr.debugGCS() - }() + defer ctr.debugGCS() // Note we always tell HCS to // create stdout as it's required regardless of '-i' or '-t' options, so that diff --git a/components/engine/libcontainerd/utils_windows.go b/components/engine/libcontainerd/utils_windows.go index fc2869b6b4..bca9fa2086 100644 --- a/components/engine/libcontainerd/utils_windows.go +++ b/components/engine/libcontainerd/utils_windows.go @@ -24,7 +24,7 @@ func (s *LCOWOption) Apply(interface{}) error { return nil } -// DebugGCS is a dirty hack for debugging for Linux Utility VMs. It simply +// debugGCS is a dirty hack for debugging for Linux Utility VMs. It simply // runs a bunch of commands inside the UVM, but seriously aides in advanced debugging. func (c *container) debugGCS() { if c == nil || c.isWindows || c.hcsContainer == nil { diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 5176b7d3d2..4ab0cb0dac 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -8,7 +8,7 @@ github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git github.com/gorilla/context v1.1 github.com/gorilla/mux v1.1 -github.com/Microsoft/opengcs v0.3.3 +github.com/Microsoft/opengcs v0.3.4 github.com/kr/pty 5cf931ef8f github.com/mattn/go-shellwords v1.0.3 github.com/sirupsen/logrus v1.0.1 diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go index b53ce25149..48daaeb048 100644 --- a/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go @@ -57,6 +57,8 @@ func (config *Config) CreateExt4Vhdx(destFile string, sizeGB uint32, cacheFile s return fmt.Errorf("failed to create VHDx %s: %s", destFile, err) } + defer config.DebugGCS() + // Attach it to the utility VM, but don't mount it (as there's no filesystem on it) if err := config.HotAddVhd(destFile, "", false, false); err != nil { return fmt.Errorf("opengcs: CreateExt4Vhdx: failed to hot-add %s to utility VM: %s", cacheFile, err) diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go index daf7c25f04..ef1e51fd65 100644 --- a/components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go @@ -20,6 +20,8 @@ func (config *Config) HotAddVhd(hostPath string, containerPath string, readOnly return fmt.Errorf("cannot hot-add VHD as no utility VM is in configuration") } + defer config.DebugGCS() + modification := &hcsshim.ResourceModificationRequestResponse{ Resource: "MappedVirtualDisk", Data: hcsshim.MappedVirtualDisk{ diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go index cf1971244d..be63189173 100644 --- a/components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go @@ -18,6 +18,8 @@ func (config *Config) HotRemoveVhd(hostPath string) error { return fmt.Errorf("cannot hot-add VHD as no utility VM is in configuration") } + defer config.DebugGCS() + modification := &hcsshim.ResourceModificationRequestResponse{ Resource: "MappedVirtualDisk", Data: hcsshim.MappedVirtualDisk{ diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/process.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/process.go index 984a95a32e..958fdb5d0b 100644 --- a/components/engine/vendor/github.com/Microsoft/opengcs/client/process.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/process.go @@ -3,8 +3,12 @@ package client import ( + "bytes" "fmt" "io" + "os" + "strings" + "time" "github.com/Microsoft/hcsshim" "github.com/sirupsen/logrus" @@ -110,3 +114,44 @@ func (config *Config) RunProcess(commandLine string, stdin io.Reader, stdout io. logrus.Debugf("opengcs: runProcess success: %s", commandLine) return process.Process, nil } + +func debugCommand(s string) string { + return fmt.Sprintf(`echo -e 'DEBUG COMMAND: %s\\n--------------\\n';%s;echo -e '\\n\\n';`, s, s) +} + +// DebugGCS extracts logs from the GCS. It's a useful hack for debugging, +// but not necessarily optimal, but all that is available to us in RS3. +func (config *Config) DebugGCS() { + if logrus.GetLevel() < logrus.DebugLevel || len(os.Getenv("OPENGCS_DEBUG_ENABLE")) == 0 { + return + } + + var out bytes.Buffer + cmd := os.Getenv("OPENGCS_DEBUG_COMMAND") + if cmd == "" { + cmd = `sh -c "` + cmd += debugCommand("ls -l /tmp") + cmd += debugCommand("cat /tmp/gcs.log") + cmd += debugCommand("ls -l /tmp/gcs") + cmd += debugCommand("ls -l /tmp/gcs/*") + cmd += debugCommand("cat /tmp/gcs/*/config.json") + cmd += debugCommand("ls -lR /var/run/gcsrunc") + cmd += debugCommand("cat /var/run/gcsrunc/log.log") + cmd += debugCommand("ps -ef") + cmd += `"` + } + proc, err := config.RunProcess(cmd, nil, &out, nil) + defer func() { + if proc != nil { + proc.Kill() + proc.Close() + } + }() + if err != nil { + logrus.Debugln("benign failure getting gcs logs: ", err) + } + if proc != nil { + proc.WaitTimeout(time.Duration(int(time.Second) * 30)) + } + logrus.Debugf("GCS Debugging:\n%s\n\nEnd GCS Debugging\n", strings.TrimSpace(out.String())) +} diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go index 9aa6609d48..29ee48957a 100644 --- a/components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go @@ -17,6 +17,8 @@ func (config *Config) TarToVhd(targetVHDFile string, reader io.Reader) (int64, e return 0, fmt.Errorf("cannot Tar2Vhd as no utility VM is in configuration") } + defer config.DebugGCS() + process, err := config.createUtilsProcess("tar2vhd") if err != nil { return 0, fmt.Errorf("failed to start tar2vhd for %s: %s", targetVHDFile, err) diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go index 27225a71e7..72e9a24ff9 100644 --- a/components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go @@ -20,6 +20,8 @@ func (config *Config) VhdToTar(vhdFile string, uvmMountPath string, isSandbox bo return nil, fmt.Errorf("cannot VhdToTar as no utility VM is in configuration") } + defer config.DebugGCS() + vhdHandle, err := os.Open(vhdFile) if err != nil { return nil, fmt.Errorf("opengcs: VhdToTar: failed to open %s: %s", vhdFile, err) From e2fa4c4d55cd772f047090edea68103430152db7 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 20 Sep 2017 15:19:16 +0200 Subject: [PATCH 09/34] Bump API version to 1.33 Signed-off-by: Sebastiaan van Stijn Upstream-commit: 15a59e763b6bdb44f28ffafb20e173606308ce2c Component: engine --- components/engine/api/common.go | 2 +- components/engine/api/swagger.yaml | 9 +++++---- components/engine/docs/api/version-history.md | 7 +++++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/components/engine/api/common.go b/components/engine/api/common.go index ff87a94b58..c8919f1900 100644 --- a/components/engine/api/common.go +++ b/components/engine/api/common.go @@ -3,7 +3,7 @@ package api // Common constants for daemon and client. const ( // DefaultVersion of Current REST API - DefaultVersion string = "1.32" + DefaultVersion string = "1.33" // NoBaseImageSpecifier is the symbol used by the FROM // command to specify that no base image is to be used. diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 75276c1b93..275c2e9cec 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -19,10 +19,10 @@ produces: consumes: - "application/json" - "text/plain" -basePath: "/v1.32" +basePath: "/v1.33" info: title: "Docker Engine API" - version: "1.32" + version: "1.33" x-logo: url: "https://docs.docker.com/images/logo-docker-main.png" description: | @@ -44,7 +44,7 @@ info: The API is usually changed in each release of Docker, so API calls are versioned to ensure that clients don't break. - For Docker Engine 17.07, the API version is 1.31. To lock to this version, you prefix the URL with `/v1.31`. For example, calling `/info` is the same as calling `/v1.31/info`. + For Docker Engine 17.09, the API version is 1.32. To lock to this version, you prefix the URL with `/v1.32`. For example, calling `/info` is the same as calling `/v1.32/info`. Engine releases in the near future should support this version of the API, so your client will continue to work even if it is talking to a newer Engine. @@ -52,10 +52,11 @@ info: The API uses an open schema model, which means server may add extra properties to responses. Likewise, the server will ignore any extra query parameters and request body properties. When you write clients, you need to ignore additional properties in responses to ensure they do not break when talking to newer Docker daemons. - This documentation is for version 1.32 of the API. Use this table to find documentation for previous versions of the API: + This documentation is for version 1.33 of the API. Use this table to find documentation for previous versions of the API: Docker version | API version | Changes ----------------|-------------|--------- + 17.09.x | [1.31](https://docs.docker.com/engine/api/v1.32/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-32-api-changes) 17.07.x | [1.31](https://docs.docker.com/engine/api/v1.31/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-31-api-changes) 17.06.x | [1.30](https://docs.docker.com/engine/api/v1.30/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-30-api-changes) 17.05.x | [1.29](https://docs.docker.com/engine/api/v1.29/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-29-api-changes) diff --git a/components/engine/docs/api/version-history.md b/components/engine/docs/api/version-history.md index 6921e1eec7..1144e95447 100644 --- a/components/engine/docs/api/version-history.md +++ b/components/engine/docs/api/version-history.md @@ -13,6 +13,13 @@ keywords: "API, Docker, rcli, REST, documentation" will be rejected. --> + +## v1.33 API changes + +[Docker Engine API v1.33](https://docs.docker.com/engine/api/v1.33/) documentation + + + ## v1.32 API changes [Docker Engine API v1.32](https://docs.docker.com/engine/api/v1.32/) documentation From de0f898e391bebc3744a80afb118908d1acae0bc Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 18 Sep 2017 09:26:34 -0400 Subject: [PATCH 10/34] Automatically set `may_detach_mounts=1` on startup This is kernel config available in RHEL7.4 based kernels that enables mountpoint removal where the mountpoint exists in other namespaces. In particular this is important for making this pattern work: ``` umount -l /some/path rm -r /some/path ``` Where `/some/path` exists in another mount namespace. Setting this value will prevent `device or resource busy` errors when attempting to the removal of `/some/path` in the example. This setting is the default, and non-configurable, on upstream kernels since 3.15. Signed-off-by: Brian Goff Upstream-commit: 83c2152de503012195bd26069fd8fbd2dea4b32f Component: engine --- components/engine/daemon/daemon_unix.go | 34 ++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index 655483c89e..31f8bc036b 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -1297,7 +1297,39 @@ func rootFSToAPIType(rootfs *image.RootFS) types.RootFS { // setupDaemonProcess sets various settings for the daemon's process func setupDaemonProcess(config *config.Config) error { // setup the daemons oom_score_adj - return setupOOMScoreAdj(config.OOMScoreAdjust) + if err := setupOOMScoreAdj(config.OOMScoreAdjust); err != nil { + return err + } + return setMayDetachMounts() +} + +// This is used to allow removal of mountpoints that may be mounted in other +// namespaces on RHEL based kernels starting from RHEL 7.4. +// Without this setting, removals on these RHEL based kernels may fail with +// "device or resource busy". +// This setting is not available in upstream kernels as it is not configurable, +// but has been in the upstream kernels since 3.15. +func setMayDetachMounts() error { + f, err := os.OpenFile("/proc/sys/fs/may_detach_mounts", os.O_WRONLY, 0) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.Wrap(err, "error opening may_detach_mounts kernel config file") + } + defer f.Close() + + _, err = f.WriteString("1") + if os.IsPermission(err) { + // Setting may_detach_mounts does not work in an + // unprivileged container. Ignore the error, but log + // it if we appear not to be in that situation. + if !rsystem.RunningInUserNS() { + logrus.Debugf("Permission denied writing %q to /proc/sys/fs/may_detach_mounts", "1") + } + return nil + } + return err } func setupOOMScoreAdj(score int) error { From 20c1a2b928647fa80931caa781f238a11deca5c5 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Wed, 20 Sep 2017 18:00:55 +0200 Subject: [PATCH 11/34] Handle plugin list not implemented Signed-off-by: Christopher Crone Upstream-commit: e7e11bdd44878d28c642d72761aa41eb9ffce3d1 Component: engine --- components/engine/client/errors.go | 22 ++++++++++++++++++++++ components/engine/client/plugin_list.go | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/components/engine/client/errors.go b/components/engine/client/errors.go index cad4521757..3f52dfe5f6 100644 --- a/components/engine/client/errors.go +++ b/components/engine/client/errors.go @@ -64,6 +64,8 @@ func wrapResponseError(err error, resp serverResponse, object, id string) error return nil case resp.statusCode == http.StatusNotFound: return objectNotFoundError{object: object, id: id} + case resp.statusCode == http.StatusNotImplemented: + return notImplementedError{message: err.Error()} default: return err } @@ -157,6 +159,26 @@ func IsErrPluginPermissionDenied(err error) bool { return ok } +type notImplementedError struct { + message string +} + +func (e notImplementedError) Error() string { + return e.message +} + +func (e notImplementedError) NotImplemented() bool { + return true +} + +// IsNotImplementedError returns true if the error is a NotImplemented error. +// This is returned by the API when a requested feature has not been +// implemented. +func IsNotImplementedError(err error) bool { + te, ok := err.(notImplementedError) + return ok && te.NotImplemented() +} + // NewVersionError returns an error if the APIVersion required // if less than the current supported version func (cli *Client) NewVersionError(APIrequired, feature string) error { diff --git a/components/engine/client/plugin_list.go b/components/engine/client/plugin_list.go index 3acde3b966..78dbeb8be3 100644 --- a/components/engine/client/plugin_list.go +++ b/components/engine/client/plugin_list.go @@ -23,7 +23,7 @@ func (cli *Client) PluginList(ctx context.Context, filter filters.Args) (types.P } resp, err := cli.get(ctx, "/plugins", query, nil) if err != nil { - return plugins, err + return plugins, wrapResponseError(err, resp, "plugin", "") } err = json.NewDecoder(resp.body).Decode(&plugins) From da0223d36580f45322f7f6053273017f322de6d7 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Wed, 20 Sep 2017 14:47:49 +0200 Subject: [PATCH 12/34] Set integration test OSType with environment variable Signed-off-by: Christopher Crone Upstream-commit: f0e5b3d7d89c0c87d001faa18bc60fd1b4531901 Component: engine --- .../cli/build/fakestorage/fixtures.go | 2 +- components/engine/integration-cli/cli/cli.go | 2 +- .../environment/environment.go | 4 ++-- .../integration-cli/requirements_test.go | 12 +++++------ .../engine/integration/system/version_test.go | 2 +- .../engine/internal/test/environment/clean.go | 2 +- .../internal/test/environment/environment.go | 21 +++++++++++++++---- .../internal/test/environment/protect.go | 4 ++-- 8 files changed, 31 insertions(+), 18 deletions(-) diff --git a/components/engine/integration-cli/cli/build/fakestorage/fixtures.go b/components/engine/integration-cli/cli/build/fakestorage/fixtures.go index b76a7d9207..2eb72847f4 100644 --- a/components/engine/integration-cli/cli/build/fakestorage/fixtures.go +++ b/components/engine/integration-cli/cli/build/fakestorage/fixtures.go @@ -30,7 +30,7 @@ func ensureHTTPServerImage(t testingT) { } defer os.RemoveAll(tmp) - goos := testEnv.DaemonInfo.OSType + goos := testEnv.OSType if goos == "" { goos = "linux" } diff --git a/components/engine/integration-cli/cli/cli.go b/components/engine/integration-cli/cli/cli.go index b8230b2da4..813c3ad437 100644 --- a/components/engine/integration-cli/cli/cli.go +++ b/components/engine/integration-cli/cli/cli.go @@ -115,7 +115,7 @@ func Docker(cmd icmd.Cmd, cmdOperators ...CmdOperator) *icmd.Result { // validateArgs is a checker to ensure tests are not running commands which are // not supported on platforms. Specifically on Windows this is 'busybox top'. func validateArgs(args ...string) error { - if testEnv.DaemonInfo.OSType != "windows" { + if testEnv.OSType != "windows" { return nil } foundBusybox := -1 diff --git a/components/engine/integration-cli/environment/environment.go b/components/engine/integration-cli/environment/environment.go index 4e04ba76f3..0decc06983 100644 --- a/components/engine/integration-cli/environment/environment.go +++ b/components/engine/integration-cli/environment/environment.go @@ -67,9 +67,9 @@ func (e *Execution) ExperimentalDaemon() bool { // decisions on how to configure themselves according to the platform // of the daemon. This is initialized in docker_utils by sending // a version call to the daemon and examining the response header. -// Deprecated: use Execution.DaemonInfo.OSType +// Deprecated: use Execution.OSType func (e *Execution) DaemonPlatform() string { - return e.DaemonInfo.OSType + return e.OSType } // MinimalBaseImage is the image used for minimal builds (it depends on the platform) diff --git a/components/engine/integration-cli/requirements_test.go b/components/engine/integration-cli/requirements_test.go index 411248195b..28c70c70ce 100644 --- a/components/engine/integration-cli/requirements_test.go +++ b/components/engine/integration-cli/requirements_test.go @@ -21,12 +21,12 @@ func ArchitectureIsNot(arch string) bool { } func DaemonIsWindows() bool { - return testEnv.DaemonInfo.OSType == "windows" + return testEnv.OSType == "windows" } func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool { return func() bool { - if testEnv.DaemonInfo.OSType != "windows" { + if testEnv.OSType != "windows" { return false } version := testEnv.DaemonInfo.KernelVersion @@ -36,7 +36,7 @@ func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool { } func DaemonIsLinux() bool { - return testEnv.DaemonInfo.OSType == "linux" + return testEnv.OSType == "linux" } func OnlyDefaultNetworks() bool { @@ -178,21 +178,21 @@ func UserNamespaceInKernel() bool { } func IsPausable() bool { - if testEnv.DaemonInfo.OSType == "windows" { + if testEnv.OSType == "windows" { return testEnv.DaemonInfo.Isolation == "hyperv" } return true } func NotPausable() bool { - if testEnv.DaemonInfo.OSType == "windows" { + if testEnv.OSType == "windows" { return testEnv.DaemonInfo.Isolation == "process" } return false } func IsolationIs(expectedIsolation string) bool { - return testEnv.DaemonInfo.OSType == "windows" && string(testEnv.DaemonInfo.Isolation) == expectedIsolation + return testEnv.OSType == "windows" && string(testEnv.DaemonInfo.Isolation) == expectedIsolation } func IsolationIsHyperv() bool { diff --git a/components/engine/integration/system/version_test.go b/components/engine/integration/system/version_test.go index ac47891e9b..110bd9f56f 100644 --- a/components/engine/integration/system/version_test.go +++ b/components/engine/integration/system/version_test.go @@ -20,5 +20,5 @@ func TestVersion(t *testing.T) { assert.NotNil(t, version.Version) assert.NotNil(t, version.MinAPIVersion) assert.Equal(t, testEnv.DaemonInfo.ExperimentalBuild, version.Experimental) - assert.Equal(t, testEnv.DaemonInfo.OSType, version.Os) + assert.Equal(t, testEnv.OSType, version.Os) } diff --git a/components/engine/internal/test/environment/clean.go b/components/engine/internal/test/environment/clean.go index 702d10711b..1bdd21080a 100644 --- a/components/engine/internal/test/environment/clean.go +++ b/components/engine/internal/test/environment/clean.go @@ -28,7 +28,7 @@ type logT interface { func (e *Execution) Clean(t testingT) { client := e.APIClient() - platform := e.DaemonInfo.OSType + platform := e.OSType if (platform != "windows") || (platform == "windows" && e.DaemonInfo.Isolation == "hyperv") { unpauseAllContainers(t, client) } diff --git a/components/engine/internal/test/environment/environment.go b/components/engine/internal/test/environment/environment.go index afe8929350..eba92b2b31 100644 --- a/components/engine/internal/test/environment/environment.go +++ b/components/engine/internal/test/environment/environment.go @@ -17,6 +17,7 @@ import ( type Execution struct { client client.APIClient DaemonInfo types.Info + OSType string PlatformDefaults PlatformDefaults protectedElements protectedElements } @@ -40,19 +41,31 @@ func New() (*Execution, error) { return nil, errors.Wrapf(err, "failed to get info from daemon") } + osType := getOSType(info) + return &Execution{ client: client, DaemonInfo: info, - PlatformDefaults: getPlatformDefaults(info), + OSType: osType, + PlatformDefaults: getPlatformDefaults(info, osType), protectedElements: newProtectedElements(), }, nil } -func getPlatformDefaults(info types.Info) PlatformDefaults { +func getOSType(info types.Info) string { + // Docker EE does not set the OSType so allow the user to override this value. + userOsType := os.Getenv("TEST_OSTYPE") + if userOsType != "" { + return userOsType + } + return info.OSType +} + +func getPlatformDefaults(info types.Info, osType string) PlatformDefaults { volumesPath := filepath.Join(info.DockerRootDir, "volumes") containersPath := filepath.Join(info.DockerRootDir, "containers") - switch info.OSType { + switch osType { case "linux": return PlatformDefaults{ BaseImage: "scratch", @@ -71,7 +84,7 @@ func getPlatformDefaults(info types.Info) PlatformDefaults { ContainerStoragePath: filepath.FromSlash(containersPath), } default: - panic(fmt.Sprintf("unknown info.OSType for daemon: %s", info.OSType)) + panic(fmt.Sprintf("unknown OSType for daemon: %s", osType)) } } diff --git a/components/engine/internal/test/environment/protect.go b/components/engine/internal/test/environment/protect.go index 2e882c8470..3c74fcf1bb 100644 --- a/components/engine/internal/test/environment/protect.go +++ b/components/engine/internal/test/environment/protect.go @@ -35,7 +35,7 @@ func ProtectAll(t testingT, testEnv *Execution) { ProtectImages(t, testEnv) ProtectNetworks(t, testEnv) ProtectVolumes(t, testEnv) - if testEnv.DaemonInfo.OSType == "linux" { + if testEnv.OSType == "linux" { ProtectPlugins(t, testEnv) } } @@ -81,7 +81,7 @@ func (e *Execution) ProtectImage(t testingT, images ...string) { func ProtectImages(t testingT, testEnv *Execution) { images := getExistingImages(t, testEnv) - if testEnv.DaemonInfo.OSType == "linux" { + if testEnv.OSType == "linux" { images = append(images, ensureFrozenImagesLinux(t, testEnv)...) } testEnv.ProtectImage(t, images...) From ccdce91e6587e5f5b54b2a41c42e9037ac4985c7 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 20 Sep 2017 15:20:43 -0700 Subject: [PATCH 13/34] overlay2: fix faulty errcheck The change in 7a7357dae1bcccb17e9b2d4c7c8f5c025fce56ca inadvertently changed the `defer` error code into a no-op. This restores its behavior prior to that code change, and also introduces a little more error logging. Signed-off-by: Euan Kemp Upstream-commit: 639ab92f011245e17e9a293455a8dae1eb034022 Component: engine --- components/engine/daemon/graphdriver/overlay2/overlay.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index 9650975b3c..f350ca9c0b 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -515,7 +515,7 @@ func (d *Driver) Remove(id string) error { } // Get creates and mounts the required file system for the given id and returns the mount path. -func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) { +func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr error) { d.locker.Lock(id) defer d.locker.Unlock(id) dir := d.dir(id) @@ -538,9 +538,11 @@ func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) { return containerfs.NewLocalContainerFS(mergedDir), nil } defer func() { - if err != nil { + if retErr != nil { if c := d.ctr.Decrement(mergedDir); c <= 0 { - unix.Unmount(mergedDir, 0) + if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil { + logrus.Errorf("error unmounting %v: %v", mergedDir, mntErr) + } } } }() From aebe8e8ce7dcd77a50cc1fb2c4ce71dca375651a Mon Sep 17 00:00:00 2001 From: chchliang Date: Mon, 18 Sep 2017 09:58:08 +0800 Subject: [PATCH 14/34] add Images testcase Signed-off-by: chchliang Upstream-commit: 832f39c2ed53fc4a91265798198273044448bc7f Component: engine --- components/engine/image/image_test.go | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/components/engine/image/image_test.go b/components/engine/image/image_test.go index e04587edae..899666a60e 100644 --- a/components/engine/image/image_test.go +++ b/components/engine/image/image_test.go @@ -2,6 +2,7 @@ package image import ( "encoding/json" + "runtime" "sort" "strings" "testing" @@ -54,6 +55,39 @@ func TestMarshalKeyOrder(t *testing.T) { } } +func TestImage(t *testing.T) { + cid := "50a16564e727" + config := &container.Config{ + Hostname: "hostname", + Domainname: "domain", + User: "root", + } + platform := runtime.GOOS + + img := &Image{ + V1Image: V1Image{ + Config: config, + }, + computedID: ID(cid), + } + + assert.Equal(t, cid, img.ImageID()) + assert.Equal(t, cid, img.ID().String()) + assert.Equal(t, platform, img.Platform()) + assert.Equal(t, config, img.RunConfig()) +} + +func TestImagePlatformNotEmpty(t *testing.T) { + platform := "platform" + img := &Image{ + V1Image: V1Image{ + OS: platform, + }, + OSVersion: "osversion", + } + assert.Equal(t, platform, img.Platform()) +} + func TestNewChildImageFromImageWithRootFS(t *testing.T) { rootFS := NewRootFS() rootFS.Append(layer.DiffID("ba5e")) From 6700f361c5d2f481dbcebd69a701111e98ca7253 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Thu, 21 Sep 2017 17:56:45 -0700 Subject: [PATCH 15/34] pkg/package: remove promise package The promise package represents a simple enough concurrency pattern that replicating it in place is sufficient. To end the propagation of this package, it has been removed and the uses have been inlined. While this code could likely be refactored to be simpler without the package, the changes have been minimized to reduce the possibility of defects. Someone else may want to do further refactoring to remove closures and reduce the number of goroutines in use. Signed-off-by: Stephen J Day Upstream-commit: 0cd4ab3f9a3f242468484fc62b46e632fdba5e13 Component: engine --- components/engine/container/stream/attach.go | 66 ++++++++++--------- components/engine/pkg/archive/archive.go | 59 +++++++++-------- components/engine/pkg/containerfs/archiver.go | 66 ++++++++++--------- components/engine/pkg/promise/promise.go | 11 ---- components/engine/pkg/promise/promise_test.go | 25 ------- 5 files changed, 103 insertions(+), 124 deletions(-) delete mode 100644 components/engine/pkg/promise/promise.go delete mode 100644 components/engine/pkg/promise/promise_test.go diff --git a/components/engine/container/stream/attach.go b/components/engine/container/stream/attach.go index 24b68863d7..4b4a4541e6 100644 --- a/components/engine/container/stream/attach.go +++ b/components/engine/container/stream/attach.go @@ -7,7 +7,6 @@ import ( "golang.org/x/net/context" "github.com/docker/docker/pkg/pools" - "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/term" "github.com/sirupsen/logrus" ) @@ -58,7 +57,7 @@ func (c *Config) AttachStreams(cfg *AttachConfig) { } // CopyStreams starts goroutines to copy data in and out to/from the container -func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) chan error { +func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) <-chan error { var ( wg sync.WaitGroup errors = make(chan error, 3) @@ -137,35 +136,42 @@ func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) chan error go attachStream("stdout", cfg.Stdout, cfg.CStdout) go attachStream("stderr", cfg.Stderr, cfg.CStderr) - return promise.Go(func() error { - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) + errs := make(chan error, 1) + + go func() { + defer close(errs) + errs <- func() error { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-ctx.Done(): + // close all pipes + if cfg.CStdin != nil { + cfg.CStdin.Close() + } + if cfg.CStdout != nil { + cfg.CStdout.Close() + } + if cfg.CStderr != nil { + cfg.CStderr.Close() + } + <-done + } + close(errors) + for err := range errors { + if err != nil { + return err + } + } + return nil }() - select { - case <-done: - case <-ctx.Done(): - // close all pipes - if cfg.CStdin != nil { - cfg.CStdin.Close() - } - if cfg.CStdout != nil { - cfg.CStdout.Close() - } - if cfg.CStderr != nil { - cfg.CStderr.Close() - } - <-done - } - close(errors) - for err := range errors { - if err != nil { - return err - } - } - return nil - }) + }() + + return errs } func copyEscapable(dst io.Writer, src io.ReadCloser, keys []byte) (written int64, err error) { diff --git a/components/engine/pkg/archive/archive.go b/components/engine/pkg/archive/archive.go index 876e605680..aa55637565 100644 --- a/components/engine/pkg/archive/archive.go +++ b/components/engine/pkg/archive/archive.go @@ -20,7 +20,6 @@ import ( "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/pools" - "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/system" "github.com/sirupsen/logrus" ) @@ -1095,36 +1094,42 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { } r, w := io.Pipe() - errC := promise.Go(func() error { - defer w.Close() + errC := make(chan error, 1) - srcF, err := os.Open(src) - if err != nil { - return err - } - defer srcF.Close() + go func() { + defer close(errC) - hdr, err := tar.FileInfoHeader(srcSt, "") - if err != nil { - return err - } - hdr.Name = filepath.Base(dst) - hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) + errC <- func() error { + defer w.Close() - if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { - return err - } + srcF, err := os.Open(src) + if err != nil { + return err + } + defer srcF.Close() - tw := tar.NewWriter(w) - defer tw.Close() - if err := tw.WriteHeader(hdr); err != nil { - return err - } - if _, err := io.Copy(tw, srcF); err != nil { - return err - } - return nil - }) + hdr, err := tar.FileInfoHeader(srcSt, "") + if err != nil { + return err + } + hdr.Name = filepath.Base(dst) + hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) + + if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { + return err + } + + tw := tar.NewWriter(w) + defer tw.Close() + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := io.Copy(tw, srcF); err != nil { + return err + } + return nil + }() + }() defer func() { if er := <-errC; err == nil && er != nil { err = er diff --git a/components/engine/pkg/containerfs/archiver.go b/components/engine/pkg/containerfs/archiver.go index 7fffa00036..3eeab49912 100644 --- a/components/engine/pkg/containerfs/archiver.go +++ b/components/engine/pkg/containerfs/archiver.go @@ -9,7 +9,6 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/system" "github.com/sirupsen/logrus" ) @@ -122,40 +121,45 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { } r, w := io.Pipe() - errC := promise.Go(func() error { - defer w.Close() + errC := make(chan error, 1) - srcF, err := srcDriver.Open(src) - if err != nil { - return err - } - defer srcF.Close() + go func() { + defer close(errC) + errC <- func() error { + defer w.Close() - hdr, err := tar.FileInfoHeader(srcSt, "") - if err != nil { - return err - } - hdr.Name = dstDriver.Base(dst) - if dstDriver.OS() == "windows" { - hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) - } else { - hdr.Mode = int64(os.FileMode(hdr.Mode)) - } + srcF, err := srcDriver.Open(src) + if err != nil { + return err + } + defer srcF.Close() - if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { - return err - } + hdr, err := tar.FileInfoHeader(srcSt, "") + if err != nil { + return err + } + hdr.Name = dstDriver.Base(dst) + if dstDriver.OS() == "windows" { + hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) + } else { + hdr.Mode = int64(os.FileMode(hdr.Mode)) + } - tw := tar.NewWriter(w) - defer tw.Close() - if err := tw.WriteHeader(hdr); err != nil { - return err - } - if _, err := io.Copy(tw, srcF); err != nil { - return err - } - return nil - }) + if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil { + return err + } + + tw := tar.NewWriter(w) + defer tw.Close() + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := io.Copy(tw, srcF); err != nil { + return err + } + return nil + }() + }() defer func() { if er := <-errC; err == nil && er != nil { err = er diff --git a/components/engine/pkg/promise/promise.go b/components/engine/pkg/promise/promise.go deleted file mode 100644 index dd52b9082f..0000000000 --- a/components/engine/pkg/promise/promise.go +++ /dev/null @@ -1,11 +0,0 @@ -package promise - -// Go is a basic promise implementation: it wraps calls a function in a goroutine, -// and returns a channel which will later return the function's return value. -func Go(f func() error) chan error { - ch := make(chan error, 1) - go func() { - ch <- f() - }() - return ch -} diff --git a/components/engine/pkg/promise/promise_test.go b/components/engine/pkg/promise/promise_test.go deleted file mode 100644 index 287213b504..0000000000 --- a/components/engine/pkg/promise/promise_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package promise - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGo(t *testing.T) { - errCh := Go(functionWithError) - er := <-errCh - require.EqualValues(t, "Error Occurred", er.Error()) - - noErrCh := Go(functionWithNoError) - er = <-noErrCh - require.Nil(t, er) -} - -func functionWithError() (err error) { - return errors.New("Error Occurred") -} -func functionWithNoError() (err error) { - return nil -} From d1d8439f3f6e559c23e3e934643764d7f88c3d30 Mon Sep 17 00:00:00 2001 From: Yuanhong Peng Date: Sat, 20 May 2017 10:38:45 +0800 Subject: [PATCH 16/34] Fixes #29654: take reference to RWLayer while committing/exporting Take an extra reference to rwlayer while the container is being committed or exported to avoid the removal of that layer. Also add some checks before commit/export. Signed-off-by: Yuanhong Peng Upstream-commit: 8c32659979150630a2c4eae4e7da944806c46297 Component: engine --- components/engine/container/state.go | 17 +++++++++++++ components/engine/daemon/commit.go | 38 ++++++++++++++++++++++++---- components/engine/daemon/export.go | 32 +++++++++++++++++++---- 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/components/engine/container/state.go b/components/engine/container/state.go index 32f3f5b7a5..cdf51d37d2 100644 --- a/components/engine/container/state.go +++ b/components/engine/container/state.go @@ -357,6 +357,15 @@ func (s *State) ResetRemovalInProgress() { s.Unlock() } +// IsRemovalInProgress returns whether the RemovalInProgress flag is set. +// Used by Container to check whether a container is being removed. +func (s *State) IsRemovalInProgress() bool { + s.Lock() + res := s.RemovalInProgress + s.Unlock() + return res +} + // SetDead sets the container state to "dead" func (s *State) SetDead() { s.Lock() @@ -364,6 +373,14 @@ func (s *State) SetDead() { s.Unlock() } +// IsDead returns whether the Dead flag is set. Used by Container to check whether a container is dead. +func (s *State) IsDead() bool { + s.Lock() + res := s.Dead + s.Unlock() + return res +} + // SetRemoved assumes this container is already in the "dead" state and // closes the internal waitRemove channel to unblock callers waiting for a // container to be removed. diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 084f488583..2684f614d7 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -2,6 +2,7 @@ package daemon import ( "encoding/json" + "fmt" "io" "runtime" "strings" @@ -133,6 +134,16 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str return "", errors.Errorf("%+v does not support commit of a running container", runtime.GOOS) } + if container.IsDead() { + err := fmt.Errorf("You cannot commit container %s which is Dead", container.ID) + return "", stateConflictError{err} + } + + if container.IsRemovalInProgress() { + err := fmt.Errorf("You cannot commit container %s which is being removed", container.ID) + return "", stateConflictError{err} + } + if c.Pause && !container.IsPaused() { daemon.containerPause(container) defer daemon.containerUnpause(container) @@ -234,19 +245,36 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str return id.String(), nil } -func (daemon *Daemon) exportContainerRw(container *container.Container) (io.ReadCloser, error) { - if err := daemon.Mount(container); err != nil { +func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io.ReadCloser, err error) { + rwlayer, err := daemon.stores[container.Platform].layerStore.GetRWLayer(container.ID) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer) + } + }() + + // TODO: this mount call is not necessary as we assume that TarStream() should + // mount the layer if needed. But the Diff() function for windows requests that + // the layer should be mounted when calling it. So we reserve this mount call + // until windows driver can implement Diff() interface correctly. + _, err = rwlayer.Mount(container.GetMountLabel()) + if err != nil { return nil, err } - archive, err := container.RWLayer.TarStream() + archive, err := rwlayer.TarStream() if err != nil { - daemon.Unmount(container) // logging is already handled in the `Unmount` function + rwlayer.Unmount() return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { archive.Close() - return container.RWLayer.Unmount() + err = rwlayer.Unmount() + daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer) + return err }), nil } diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 730387d76c..465ccac283 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -22,6 +22,16 @@ func (daemon *Daemon) ContainerExport(name string, out io.Writer) error { return fmt.Errorf("the daemon on this platform does not support exporting Windows containers") } + if container.IsDead() { + err := fmt.Errorf("You cannot export container %s which is Dead", container.ID) + return stateConflictError{err} + } + + if container.IsRemovalInProgress() { + err := fmt.Errorf("You cannot export container %s which is being removed", container.ID) + return stateConflictError{err} + } + data, err := daemon.containerExport(container) if err != nil { return fmt.Errorf("Error exporting container %s: %v", name, err) @@ -35,8 +45,19 @@ func (daemon *Daemon) ContainerExport(name string, out io.Writer) error { return nil } -func (daemon *Daemon) containerExport(container *container.Container) (io.ReadCloser, error) { - if err := daemon.Mount(container); err != nil { +func (daemon *Daemon) containerExport(container *container.Container) (arch io.ReadCloser, err error) { + rwlayer, err := daemon.stores[container.Platform].layerStore.GetRWLayer(container.ID) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer) + } + }() + + _, err = rwlayer.Mount(container.GetMountLabel()) + if err != nil { return nil, err } @@ -46,12 +67,13 @@ func (daemon *Daemon) containerExport(container *container.Container) (io.ReadCl GIDMaps: daemon.idMappings.GIDs(), }) if err != nil { - daemon.Unmount(container) + rwlayer.Unmount() return nil, err } - arch := ioutils.NewReadCloserWrapper(archive, func() error { + arch = ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() - daemon.Unmount(container) + rwlayer.Unmount() + daemon.stores[container.Platform].layerStore.ReleaseRWLayer(rwlayer) return err }) daemon.LogContainerEvent(container, "export") From 2d68241660139ecec7241637d0ba6d5c51a1534a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 21 Sep 2017 17:54:27 -0700 Subject: [PATCH 17/34] vendor: update buildkit Signed-off-by: Tonis Tiigi Upstream-commit: ed6fd3d95bde4651ffb66d37cd1e5e76ee3c1f7b Component: engine --- components/engine/vendor.conf | 4 +- .../vendor/github.com/moby/buildkit/README.md | 60 ++++++++-- .../moby/buildkit/session/context.go | 22 ++++ .../buildkit/session/filesync/diffcopy.go | 36 +++++- .../buildkit/session/filesync/filesync.go | 92 ++++++++++++--- .../buildkit/session/filesync/filesync.pb.go | 110 ++++++++++++++++-- .../buildkit/session/filesync/filesync.proto | 5 + .../buildkit/session/filesync/tarstream.go | 83 ------------- .../moby/buildkit/session/manager.go | 32 +++-- .../moby/buildkit/session/session.go | 17 +-- .../github.com/moby/buildkit/vendor.conf | 26 +++-- .../github.com/tonistiigi/fsutil/diff.go | 7 ++ .../tonistiigi/fsutil/diskwriter.go | 89 +++++--------- .../tonistiigi/fsutil/diskwriter_darwin.go | 7 ++ .../tonistiigi/fsutil/diskwriter_linux.go | 44 ------- .../tonistiigi/fsutil/diskwriter_unix.go | 51 ++++++++ .../github.com/tonistiigi/fsutil/receive.go | 65 ++++++++--- .../tonistiigi/fsutil/receive_unsupported.go | 14 --- .../github.com/tonistiigi/fsutil/validator.go | 18 +-- .../github.com/tonistiigi/fsutil/walker.go | 14 ++- 20 files changed, 499 insertions(+), 297 deletions(-) create mode 100644 components/engine/vendor/github.com/moby/buildkit/session/context.go delete mode 100644 components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go create mode 100644 components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_darwin.go create mode 100644 components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_unix.go delete mode 100644 components/engine/vendor/github.com/tonistiigi/fsutil/receive_unsupported.go diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index e90550db30..98689c4b0b 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -2,7 +2,6 @@ github.com/Azure/go-ansiterm 19f72df4d05d31cbe1c56bfc8045c96babff6c7e github.com/Microsoft/hcsshim v0.6.5 github.com/Microsoft/go-winio v0.4.5 -github.com/moby/buildkit da2b9dc7dab99e824b2b1067ad7d0523e32dd2d9 https://github.com/dmcgowan/buildkit.git github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git @@ -28,6 +27,8 @@ github.com/imdario/mergo 0.2.1 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0 github.com/containerd/continuity 22694c680ee48fb8f50015b44618517e2bde77e8 +github.com/moby/buildkit c2dbdeb457ea665699a5d97f79eebfac4ab4726f https://github.com/tonistiigi/buildkit.git +github.com/tonistiigi/fsutil 1dedf6e90084bd88c4c518a15e68a37ed1370203 #get libnetwork packages github.com/docker/libnetwork 60e002dd61885e1cd909582f00f7eb4da634518a @@ -107,7 +108,6 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 github.com/containerd/containerd 06b9cb35161009dcb7123345749fef02f7cea8e0 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4 github.com/stevvooe/continuity cd7a8e21e2b6f84799f5dd4b65faf49c8d3ee02d -github.com/tonistiigi/fsutil 0ac4c11b053b9c5c7c47558f81f96c7100ce50fb # cluster github.com/docker/swarmkit bd7bafb8a61de1f5f23c8215ce7b9ecbcb30ff21 diff --git a/components/engine/vendor/github.com/moby/buildkit/README.md b/components/engine/vendor/github.com/moby/buildkit/README.md index ddcbb01ceb..4101bf43b5 100644 --- a/components/engine/vendor/github.com/moby/buildkit/README.md +++ b/components/engine/vendor/github.com/moby/buildkit/README.md @@ -1,10 +1,16 @@ -### Important: This repository is in an early development phase and not suitable for practical workloads. It does not compare with `docker build` features yet. +### Important: This repository is in an early development phase [![asciicinema example](https://asciinema.org/a/gPEIEo1NzmDTUu2bEPsUboqmU.png)](https://asciinema.org/a/gPEIEo1NzmDTUu2bEPsUboqmU) ## BuildKit + +[![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb) +[![Build Status](https://travis-ci.org/moby/buildkit.svg?branch=master)](https://travis-ci.org/moby/buildkit) +[![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit) + + BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner. Key features: @@ -23,7 +29,7 @@ Read the proposal from https://github.com/moby/moby/issues/32925 #### Quick start -BuildKit daemon can be built in two different versions: one that uses [containerd](https://github.com/containerd/containerd) for execution and distribution, and a standalone version that doesn't have other dependencies apart from [runc](https://github.com/opencontainers/runc). We are open for adding more backends. `buildd` is a CLI utility for running the gRPC API. +BuildKit daemon can be built in two different versions: one that uses [containerd](https://github.com/containerd/containerd) for execution and distribution, and a standalone version that doesn't have other dependencies apart from [runc](https://github.com/opencontainers/runc). We are open for adding more backends. `buildd` is a CLI utility for serving the gRPC API. ```bash # buildd daemon (choose one) @@ -36,17 +42,15 @@ go build -o buildctl ./cmd/buildctl You can also use `make binaries` that prepares all binaries into the `bin/` directory. -The first thing to test could be to try building BuildKit with BuildKit. BuildKit provides a low-level solver format that could be used by multiple build definitions. Preparation work for making the Dockerfile parser reusable as a frontend is tracked in https://github.com/moby/moby/pull/33492. As no frontends have been integrated yet we currently have to use a client library to generate this low-level definition. - `examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit and its dependencies using the `client` package. Running one of these script generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build. -You can use `buildctl debug dump-llb` to see what data is this definition. +You can use `buildctl debug dump-llb` to see what data is in this definition. Add `--dot` to generate dot layout. ```bash go run examples/buildkit0/buildkit.go | buildctl debug dump-llb | jq . ``` -To start building use `buildctl build` command. The script accepts `--target` flag to choose between `containerd` and `standalone` configurations. In standalone mode BuildKit binaries are built together with `runc`. In containerd mode, the `containerd` binary is built as well from the upstream repo. +To start building use `buildctl build` command. The example script accepts `--target` flag to choose between `containerd` and `standalone` configurations. In standalone mode BuildKit binaries are built together with `runc`. In containerd mode, the `containerd` binary is built as well from the upstream repo. ```bash go run examples/buildkit0/buildkit.go | buildctl build @@ -59,10 +63,52 @@ Different versions of the example scripts show different ways of describing the - `./examples/buildkit0` - uses only exec operations, defines a full stage per component. - `./examples/buildkit1` - cloning git repositories has been separated for extra concurrency. - `./examples/buildkit2` - uses git sources directly instead of running `git clone`, allowing better performance and much safer caching. +- `./examples/buildkit3` - allows using local source files for separate components eg. `./buildkit3 --runc=local | buildctl build --local runc-src=some/local/path` +- `./examples/dockerfile2llb` - can be used to convert a Dockerfile to LLB for debugging purposes +- `./examples/gobuild` - shows how to use nested invocation to generate LLB for Go package internal dependencies + + +#### Examples + +##### Starting the buildd daemon: + +``` +buildd-standalone --debug --root /var/lib/buildkit +``` + +##### Building a Dockerfile: + +``` +buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. +``` + +`context` and `dockerfile` should point to local directories for build context and Dockerfile location. + + +##### Exporting resulting image to containerd + +Containerd version of buildd needs to be used + +``` +buildctl build ... --exporter=image --exporter-opt name=docker.io/username/image +ctr --namespace=buildkit images ls +``` + +##### Exporting build result back to client + +``` +buildctl build ... --exporter=local --exporter-opt output=path/to/output-dir +``` + +#### View build cache + +``` +buildctl du -v +``` #### Supported runc version -During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/3707703a694187c7d08e2f333da6ddd58bcb729d/RUNC.md) for more information. +During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/d1e11f17ec7b325f89608dd46c128300b8727d50/RUNC.md) for more information. #### Contributing diff --git a/components/engine/vendor/github.com/moby/buildkit/session/context.go b/components/engine/vendor/github.com/moby/buildkit/session/context.go new file mode 100644 index 0000000000..31a29f0868 --- /dev/null +++ b/components/engine/vendor/github.com/moby/buildkit/session/context.go @@ -0,0 +1,22 @@ +package session + +import "context" + +type contextKeyT string + +var contextKey = contextKeyT("buildkit/session-id") + +func NewContext(ctx context.Context, id string) context.Context { + if id != "" { + return context.WithValue(ctx, contextKey, id) + } + return ctx +} + +func FromContext(ctx context.Context) string { + v := ctx.Value(contextKey) + if v == nil { + return "" + } + return v.(string) +} diff --git a/components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go index 58b29686cc..c5a3b5bd6e 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go @@ -1,31 +1,55 @@ package filesync import ( + "os" "time" - "google.golang.org/grpc" - "github.com/sirupsen/logrus" "github.com/tonistiigi/fsutil" + "google.golang.org/grpc" ) -func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb) error { +func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error { return fsutil.Send(stream.Context(), stream, dir, &fsutil.WalkOpt{ ExcludePatterns: excludes, - IncludePaths: includes, // TODO: rename IncludePatterns + IncludePatterns: includes, + Map: _map, }, progress) } -func recvDiffCopy(ds grpc.Stream, dest string, cu CacheUpdater) error { +func recvDiffCopy(ds grpc.Stream, dest string, cu CacheUpdater, progress progressCb) error { st := time.Now() defer func() { logrus.Debugf("diffcopy took: %v", time.Since(st)) }() var cf fsutil.ChangeFunc + var ch fsutil.ContentHasher if cu != nil { cu.MarkSupported(true) cf = cu.HandleChange + ch = cu.ContentHasher() } + return fsutil.Receive(ds.Context(), ds, dest, fsutil.ReceiveOpt{ + NotifyHashed: cf, + ContentHasher: ch, + ProgressCb: progress, + }) +} - return fsutil.Receive(ds.Context(), ds, dest, cf) +func syncTargetDiffCopy(ds grpc.Stream, dest string) error { + if err := os.MkdirAll(dest, 0700); err != nil { + return err + } + return fsutil.Receive(ds.Context(), ds, dest, fsutil.ReceiveOpt{ + Merge: true, + Filter: func() func(*fsutil.Stat) bool { + uid := os.Getuid() + gid := os.Getgid() + return func(st *fsutil.Stat) bool { + st.Uid = uint32(uid) + st.Gid = uint32(gid) + return true + } + }(), + }) } diff --git a/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go index fe4d00a729..5642f07ac4 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go @@ -1,6 +1,7 @@ package filesync import ( + "fmt" "os" "strings" @@ -15,20 +16,29 @@ import ( const ( keyOverrideExcludes = "override-excludes" keyIncludePatterns = "include-patterns" + keyDirName = "dir-name" ) type fsSyncProvider struct { - root string - excludes []string - p progressCb - doneCh chan error + dirs map[string]SyncedDir + p progressCb + doneCh chan error +} + +type SyncedDir struct { + Name string + Dir string + Excludes []string + Map func(*fsutil.Stat) bool } // NewFSSyncProvider creates a new provider for sending files from client -func NewFSSyncProvider(root string, excludes []string) session.Attachable { +func NewFSSyncProvider(dirs []SyncedDir) session.Attachable { p := &fsSyncProvider{ - root: root, - excludes: excludes, + dirs: map[string]SyncedDir{}, + } + for _, d := range dirs { + p.dirs[d.Name] = d } return p } @@ -58,9 +68,19 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error opts, _ := metadata.FromContext(stream.Context()) // if no metadata continue with empty object + name, ok := opts[keyDirName] + if !ok || len(name) != 1 { + return errors.New("no dir name in request") + } + + dir, ok := sp.dirs[name[0]] + if !ok { + return errors.Errorf("no access allowed to dir %q", name[0]) + } + var excludes []string if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" { - excludes = sp.excludes + excludes = dir.Excludes } includes := opts[keyIncludePatterns] @@ -75,7 +95,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error doneCh = sp.doneCh sp.doneCh = nil } - err := pr.sendFn(stream, sp.root, includes, excludes, progress) + err := pr.sendFn(stream, dir.Dir, includes, excludes, progress, dir.Map) if doneCh != nil { if err != nil { doneCh <- err @@ -94,8 +114,8 @@ type progressCb func(int, bool) type protocol struct { name string - sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb) error - recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater) error + sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb, _map func(*fsutil.Stat) bool) error + recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater, progress progressCb) error } func isProtoSupported(p string) bool { @@ -112,25 +132,23 @@ var supportedProtocols = []protocol{ sendFn: sendDiffCopy, recvFn: recvDiffCopy, }, - { - name: "tarstream", - sendFn: sendTarStream, - recvFn: recvTarStream, - }, } // FSSendRequestOpt defines options for FSSend request type FSSendRequestOpt struct { + Name string IncludePatterns []string OverrideExcludes bool DestDir string CacheUpdater CacheUpdater + ProgressCb func(int, bool) } // CacheUpdater is an object capable of sending notifications for the cache hash changes type CacheUpdater interface { MarkSupported(bool) HandleChange(fsutil.ChangeKind, string, os.FileInfo, error) error + ContentHasher() fsutil.ContentHasher } // FSSync initializes a transfer of files @@ -155,6 +173,8 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { opts[keyIncludePatterns] = opt.IncludePatterns } + opts[keyDirName] = []string{opt.Name} + ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -177,7 +197,45 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { return err } stream = cc + default: + panic(fmt.Sprintf("invalid protocol: %q", pr.name)) } - return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater) + return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb) +} + +// NewFSSyncTarget allows writing into a directory +func NewFSSyncTarget(outdir string) session.Attachable { + p := &fsSyncTarget{ + outdir: outdir, + } + return p +} + +type fsSyncTarget struct { + outdir string +} + +func (sp *fsSyncTarget) Register(server *grpc.Server) { + RegisterFileSendServer(server, sp) +} + +func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) error { + return syncTargetDiffCopy(stream, sp.outdir) +} + +func CopyToCaller(ctx context.Context, srcPath string, c session.Caller, progress func(int, bool)) error { + method := session.MethodURL(_FileSend_serviceDesc.ServiceName, "diffcopy") + if !c.Supports(method) { + return errors.Errorf("method %s not supported by the client", method) + } + + client := NewFileSendClient(c.Conn()) + + cc, err := client.DiffCopy(ctx) + if err != nil { + return err + } + + return sendDiffCopy(cc, srcPath, nil, nil, progress, nil) } diff --git a/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.pb.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.pb.go index c6ed666383..69c78886f2 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.pb.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.pb.go @@ -277,6 +277,102 @@ var _FileSync_serviceDesc = grpc.ServiceDesc{ Metadata: "filesync.proto", } +// Client API for FileSend service + +type FileSendClient interface { + DiffCopy(ctx context.Context, opts ...grpc.CallOption) (FileSend_DiffCopyClient, error) +} + +type fileSendClient struct { + cc *grpc.ClientConn +} + +func NewFileSendClient(cc *grpc.ClientConn) FileSendClient { + return &fileSendClient{cc} +} + +func (c *fileSendClient) DiffCopy(ctx context.Context, opts ...grpc.CallOption) (FileSend_DiffCopyClient, error) { + stream, err := grpc.NewClientStream(ctx, &_FileSend_serviceDesc.Streams[0], c.cc, "/moby.filesync.v1.FileSend/DiffCopy", opts...) + if err != nil { + return nil, err + } + x := &fileSendDiffCopyClient{stream} + return x, nil +} + +type FileSend_DiffCopyClient interface { + Send(*BytesMessage) error + Recv() (*BytesMessage, error) + grpc.ClientStream +} + +type fileSendDiffCopyClient struct { + grpc.ClientStream +} + +func (x *fileSendDiffCopyClient) Send(m *BytesMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *fileSendDiffCopyClient) Recv() (*BytesMessage, error) { + m := new(BytesMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// Server API for FileSend service + +type FileSendServer interface { + DiffCopy(FileSend_DiffCopyServer) error +} + +func RegisterFileSendServer(s *grpc.Server, srv FileSendServer) { + s.RegisterService(&_FileSend_serviceDesc, srv) +} + +func _FileSend_DiffCopy_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(FileSendServer).DiffCopy(&fileSendDiffCopyServer{stream}) +} + +type FileSend_DiffCopyServer interface { + Send(*BytesMessage) error + Recv() (*BytesMessage, error) + grpc.ServerStream +} + +type fileSendDiffCopyServer struct { + grpc.ServerStream +} + +func (x *fileSendDiffCopyServer) Send(m *BytesMessage) error { + return x.ServerStream.SendMsg(m) +} + +func (x *fileSendDiffCopyServer) Recv() (*BytesMessage, error) { + m := new(BytesMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _FileSend_serviceDesc = grpc.ServiceDesc{ + ServiceName: "moby.filesync.v1.FileSend", + HandlerType: (*FileSendServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "DiffCopy", + Handler: _FileSend_DiffCopy_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "filesync.proto", +} + func (m *BytesMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -558,7 +654,7 @@ var ( func init() { proto.RegisterFile("filesync.proto", fileDescriptorFilesync) } var fileDescriptorFilesync = []byte{ - // 198 bytes of a gzipped FileDescriptorProto + // 208 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0xcb, 0xcc, 0x49, 0x2d, 0xae, 0xcc, 0x4b, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0xc8, 0xcd, 0x4f, 0xaa, 0xd4, 0x83, 0x0b, 0x96, 0x19, 0x2a, 0x29, 0x71, 0xf1, 0x38, 0x55, 0x96, 0xa4, 0x16, 0xfb, 0xa6, @@ -566,10 +662,10 @@ var fileDescriptorFilesync = []byte{ 0x30, 0x6a, 0xf0, 0x04, 0x81, 0xd9, 0x46, 0xab, 0x19, 0xb9, 0x38, 0xdc, 0x32, 0x73, 0x52, 0x83, 0x2b, 0xf3, 0x92, 0x85, 0xfc, 0xb8, 0x38, 0x5c, 0x32, 0xd3, 0xd2, 0x9c, 0xf3, 0x0b, 0x2a, 0x85, 0xe4, 0xf4, 0xd0, 0xcd, 0xd3, 0x43, 0x36, 0x4c, 0x8a, 0x80, 0xbc, 0x06, 0xa3, 0x01, 0xa3, 0x90, - 0x3f, 0x17, 0x67, 0x48, 0x62, 0x51, 0x70, 0x49, 0x51, 0x6a, 0x62, 0x2e, 0x35, 0x0c, 0x74, 0x32, - 0xbb, 0xf0, 0x50, 0x8e, 0xe1, 0xc6, 0x43, 0x39, 0x86, 0x0f, 0x0f, 0xe5, 0x18, 0x1b, 0x1e, 0xc9, - 0x31, 0xae, 0x78, 0x24, 0xc7, 0x78, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, - 0xc9, 0x31, 0xbe, 0x78, 0x24, 0xc7, 0xf0, 0xe1, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x51, - 0x1c, 0x30, 0xb3, 0x92, 0xd8, 0xc0, 0x41, 0x64, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x5f, 0x0c, - 0x8d, 0xc5, 0x34, 0x01, 0x00, 0x00, + 0x3f, 0x17, 0x67, 0x48, 0x62, 0x51, 0x70, 0x49, 0x51, 0x6a, 0x62, 0x2e, 0x35, 0x0c, 0x34, 0x8a, + 0x82, 0x3a, 0x36, 0x35, 0x2f, 0x85, 0xda, 0x8e, 0x75, 0x32, 0xbb, 0xf0, 0x50, 0x8e, 0xe1, 0xc6, + 0x43, 0x39, 0x86, 0x0f, 0x0f, 0xe5, 0x18, 0x1b, 0x1e, 0xc9, 0x31, 0xae, 0x78, 0x24, 0xc7, 0x78, + 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0xbe, 0x78, 0x24, 0xc7, + 0xf0, 0xe1, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x51, 0x1c, 0x30, 0xb3, 0x92, 0xd8, 0xc0, + 0xc1, 0x6f, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x72, 0x81, 0x1a, 0x91, 0x90, 0x01, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.proto b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.proto index 2fd5b3ec8d..0ae2937368 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.proto +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.proto @@ -9,6 +9,11 @@ service FileSync{ rpc TarStream(stream BytesMessage) returns (stream BytesMessage); } +service FileSend{ + rpc DiffCopy(stream BytesMessage) returns (stream BytesMessage); +} + + // BytesMessage contains a chunk of byte data message BytesMessage{ bytes data = 1; diff --git a/components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go deleted file mode 100644 index 5cab867498..0000000000 --- a/components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go +++ /dev/null @@ -1,83 +0,0 @@ -package filesync - -import ( - "io" - - "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/chrootarchive" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "google.golang.org/grpc" -) - -func sendTarStream(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb) error { - a, err := archive.TarWithOptions(dir, &archive.TarOptions{ - ExcludePatterns: excludes, - }) - if err != nil { - return err - } - - size := 0 - buf := make([]byte, 1<<15) - t := new(BytesMessage) - for { - n, err := a.Read(buf) - if err != nil { - if err == io.EOF { - break - } - return err - } - t.Data = buf[:n] - - if err := stream.SendMsg(t); err != nil { - return err - } - size += n - if progress != nil { - progress(size, false) - } - } - if progress != nil { - progress(size, true) - } - return nil -} - -func recvTarStream(ds grpc.Stream, dest string, cs CacheUpdater) error { - - pr, pw := io.Pipe() - - go func() { - var ( - err error - t = new(BytesMessage) - ) - for { - if err = ds.RecvMsg(t); err != nil { - if err == io.EOF { - err = nil - } - break - } - _, err = pw.Write(t.Data) - if err != nil { - break - } - } - if err = pw.CloseWithError(err); err != nil { - logrus.Errorf("failed to close tar transfer pipe") - } - }() - - decompressedStream, err := archive.DecompressStream(pr) - if err != nil { - return errors.Wrap(err, "failed to decompress stream") - } - - if err := chrootarchive.Untar(decompressedStream, dest, nil); err != nil { - return errors.Wrap(err, "failed to untar context") - } - return nil -} diff --git a/components/engine/vendor/github.com/moby/buildkit/session/manager.go b/components/engine/vendor/github.com/moby/buildkit/session/manager.go index 9523e6f317..b3e5955652 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/manager.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/manager.go @@ -49,14 +49,14 @@ func (sm *Manager) HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, return errors.New("handler does not support hijack") } - uuid := r.Header.Get(headerSessionUUID) + id := r.Header.Get(headerSessionID) proto := r.Header.Get("Upgrade") sm.mu.Lock() - if _, ok := sm.sessions[uuid]; ok { + if _, ok := sm.sessions[id]; ok { sm.mu.Unlock() - return errors.Errorf("session %s already exists", uuid) + return errors.Errorf("session %s already exists", id) } if proto == "" { @@ -102,8 +102,10 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin ctx, cancel := context.WithCancel(ctx) defer cancel() + opts = canonicalHeaders(opts) + h := http.Header(opts) - uuid := h.Get(headerSessionUUID) + id := h.Get(headerSessionID) name := h.Get(headerSessionName) sharedKey := h.Get(headerSessionSharedKey) @@ -115,7 +117,7 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin c := &client{ Session: Session{ - uuid: uuid, + id: id, name: name, sharedKey: sharedKey, ctx: ctx, @@ -129,13 +131,13 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin for _, m := range opts[headerSessionMethod] { c.supported[strings.ToLower(m)] = struct{}{} } - sm.sessions[uuid] = c + sm.sessions[id] = c sm.updateCondition.Broadcast() sm.mu.Unlock() defer func() { sm.mu.Lock() - delete(sm.sessions, uuid) + delete(sm.sessions, id) sm.mu.Unlock() }() @@ -146,8 +148,8 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin return nil } -// Get returns a session by UUID -func (sm *Manager) Get(ctx context.Context, uuid string) (Caller, error) { +// Get returns a session by ID +func (sm *Manager) Get(ctx context.Context, id string) (Caller, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -165,11 +167,11 @@ func (sm *Manager) Get(ctx context.Context, uuid string) (Caller, error) { select { case <-ctx.Done(): sm.mu.Unlock() - return nil, errors.Wrapf(ctx.Err(), "no active session for %s", uuid) + return nil, errors.Wrapf(ctx.Err(), "no active session for %s", id) default: } var ok bool - c, ok = sm.sessions[uuid] + c, ok = sm.sessions[id] if !ok || c.closed() { sm.updateCondition.Wait() continue @@ -200,3 +202,11 @@ func (c *client) Supports(url string) bool { func (c *client) Conn() *grpc.ClientConn { return c.cc } + +func canonicalHeaders(in map[string][]string) map[string][]string { + out := map[string][]string{} + for k := range in { + out[http.CanonicalHeaderKey(k)] = in[k] + } + return out +} diff --git a/components/engine/vendor/github.com/moby/buildkit/session/session.go b/components/engine/vendor/github.com/moby/buildkit/session/session.go index 147486a75b..454c3d7f3f 100644 --- a/components/engine/vendor/github.com/moby/buildkit/session/session.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/session.go @@ -12,7 +12,7 @@ import ( ) const ( - headerSessionUUID = "X-Docker-Expose-Session-Uuid" + headerSessionID = "X-Docker-Expose-Session-Uuid" headerSessionName = "X-Docker-Expose-Session-Name" headerSessionSharedKey = "X-Docker-Expose-Session-Sharedkey" headerSessionMethod = "X-Docker-Expose-Session-Grpc-Method" @@ -28,7 +28,7 @@ type Attachable interface { // Session is a long running connection between client and a daemon type Session struct { - uuid string + id string name string sharedKey string ctx context.Context @@ -39,9 +39,9 @@ type Session struct { // NewSession returns a new long running session func NewSession(name, sharedKey string) (*Session, error) { - uuid := stringid.GenerateRandomID() + id := stringid.GenerateRandomID() s := &Session{ - uuid: uuid, + id: id, name: name, sharedKey: sharedKey, grpcServer: grpc.NewServer(), @@ -57,9 +57,9 @@ func (s *Session) Allow(a Attachable) { a.Register(s.grpcServer) } -// UUID returns unique identifier for the session -func (s *Session) UUID() string { - return s.uuid +// ID returns unique identifier for the session +func (s *Session) ID() string { + return s.id } // Run activates the session @@ -72,7 +72,7 @@ func (s *Session) Run(ctx context.Context, dialer Dialer) error { defer close(s.done) meta := make(map[string][]string) - meta[headerSessionUUID] = []string{s.uuid} + meta[headerSessionID] = []string{s.id} meta[headerSessionName] = []string{s.name} meta[headerSessionSharedKey] = []string{s.sharedKey} @@ -92,6 +92,7 @@ func (s *Session) Run(ctx context.Context, dialer Dialer) error { // Close closes the session func (s *Session) Close() error { if s.cancelCtx != nil && s.done != nil { + s.grpcServer.Stop() s.cancelCtx() <-s.done } diff --git a/components/engine/vendor/github.com/moby/buildkit/vendor.conf b/components/engine/vendor/github.com/moby/buildkit/vendor.conf index b13cfa9675..f3760bd0f7 100644 --- a/components/engine/vendor/github.com/moby/buildkit/vendor.conf +++ b/components/engine/vendor/github.com/moby/buildkit/vendor.conf @@ -6,26 +6,26 @@ github.com/davecgh/go-spew v1.1.0 github.com/pmezard/go-difflib v1.0.0 golang.org/x/sys 739734461d1c916b6c72a63d7efda2b27edb369f -github.com/containerd/containerd 3707703a694187c7d08e2f333da6ddd58bcb729d -golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c -github.com/Sirupsen/logrus v0.11.0 +github.com/containerd/containerd d1e11f17ec7b325f89608dd46c128300b8727d50 +golang.org/x/sync f52d1811a62927559de87708c8913c1650ce4f26 +github.com/sirupsen/logrus v1.0.0 google.golang.org/grpc v1.3.0 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 golang.org/x/net 1f9224279e98554b6a6432d4dd998a739f8b2b7c github.com/gogo/protobuf d2e1ade2d719b78fe5b061b4c18a9f7111b5bdc8 github.com/golang/protobuf 5a0f697c9ed9d68fef0116532c6e05cfeae00e55 github.com/containerd/continuity 86cec1535a968310e7532819f699ff2830ed7463 -github.com/opencontainers/image-spec v1.0.0-rc6 -github.com/opencontainers/runc 429a5387123625040bacfbb60d96b1cbd02293ab +github.com/opencontainers/image-spec v1.0.0 +github.com/opencontainers/runc e775f0fba3ea329b8b766451c892c41a3d49594d github.com/Microsoft/go-winio v0.4.1 github.com/containerd/fifo 69b99525e472735860a5269b75af1970142b3062 -github.com/opencontainers/runtime-spec 198f23f827eea397d4331d7eb048d9d4c7ff7bee +github.com/opencontainers/runtime-spec 96de01bbb42c7af89bff100e10a9f0fb62e75bfb github.com/containerd/go-runc 2774a2ea124a5c2d0aba13b5c2dd8a5a9a48775d github.com/containerd/console 7fed77e673ca4abcd0cbd6d4d0e0e22137cbd778 -github.com/Azure/go-ansiterm fa152c58bc15761d0200cb75fe958b89a9d4888e +github.com/Azure/go-ansiterm 19f72df4d05d31cbe1c56bfc8045c96babff6c7e google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 -github.com/docker/go-events aa2e3b613fbbfdddbe055a7b9e3ce271cfd83eca +github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/urfave/cli d70f47eeca3afd795160003bc6e28b001d60c67c github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52 @@ -33,8 +33,14 @@ github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716 golang.org/x/time 8be79e1e0910c292df4e79c241bb7e8f7e725959 github.com/BurntSushi/locker 392720b78f44e9d0249fcac6c43b111b47a370b8 -github.com/docker/docker 05c7c311390911daebcf5d9519dee813fc02a887 +github.com/docker/docker 6f723db8c6f0c7f0b252674a9673a25b5978db04 https://github.com/tonistiigi/docker.git github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f -github.com/tonistiigi/fsutil 0ac4c11b053b9c5c7c47558f81f96c7100ce50fb +github.com/tonistiigi/fsutil 1dedf6e90084bd88c4c518a15e68a37ed1370203 github.com/stevvooe/continuity 86cec1535a968310e7532819f699ff2830ed7463 +github.com/dmcgowan/go-tar 2e2c51242e8993c50445dab7c03c8e7febddd0cf +github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git +github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 +github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b +github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d +github.com/docker/distribution 30578ca32960a4d368bf6db67b0a33c2a1f3dc6f diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/diff.go b/components/engine/vendor/github.com/tonistiigi/fsutil/diff.go index 1530973784..6125ef73af 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/diff.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/diff.go @@ -1,6 +1,7 @@ package fsutil import ( + "hash" "os" "golang.org/x/net/context" @@ -14,6 +15,8 @@ func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error { type HandleChangeFn func(ChangeKind, string, os.FileInfo, error) error +type ContentHasher func(*Stat) (hash.Hash, error) + func GetWalkerFn(root string) walkerFn { return func(ctx context.Context, pathC chan<- *currentPath) error { return Walk(ctx, root, nil, func(path string, f os.FileInfo, err error) error { @@ -35,3 +38,7 @@ func GetWalkerFn(root string) walkerFn { }) } } + +func emptyWalker(ctx context.Context, pathC chan<- *currentPath) error { + return nil +} diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter.go b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter.go index a54b4a737a..a465615c3b 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter.go @@ -1,11 +1,6 @@ -// +build linux windows - package fsutil import ( - "archive/tar" - "crypto/sha256" - "encoding/hex" "hash" "io" "os" @@ -14,8 +9,7 @@ import ( "sync" "time" - "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/tarsum" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sync/errgroup" @@ -24,11 +18,15 @@ import ( type WriteToFunc func(context.Context, string, io.WriteCloser) error type DiskWriterOpt struct { - AsyncDataCb WriteToFunc - SyncDataCb WriteToFunc - NotifyCb func(ChangeKind, string, os.FileInfo, error) error + AsyncDataCb WriteToFunc + SyncDataCb WriteToFunc + NotifyCb func(ChangeKind, string, os.FileInfo, error) error + ContentHasher ContentHasher + Filter FilterFunc } +type FilterFunc func(*Stat) bool + type DiskWriter struct { opt DiskWriterOpt dest string @@ -37,6 +35,7 @@ type DiskWriter struct { ctx context.Context cancel func() eg *errgroup.Group + filter FilterFunc } func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error) { @@ -102,6 +101,12 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er return errors.Errorf("%s invalid change without stat information", p) } + if dw.filter != nil { + if ok := dw.filter(stat); !ok { + return nil + } + } + rename := true oldFi, err := os.Lstat(destPath) if err != nil { @@ -202,7 +207,7 @@ func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w var hw *hashedWriter if dw.opt.NotifyCb != nil { var err error - if hw, err = newHashWriter(p, fi, w); err != nil { + if hw, err = newHashWriter(dw.opt.ContentHasher, fi, w); err != nil { return err } w = hw @@ -229,13 +234,18 @@ func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w type hashedWriter struct { os.FileInfo io.Writer - h hash.Hash - w io.WriteCloser - sum string + h hash.Hash + w io.WriteCloser + dgst digest.Digest } -func newHashWriter(p string, fi os.FileInfo, w io.WriteCloser) (*hashedWriter, error) { - h, err := NewTarsumHash(p, fi) +func newHashWriter(ch ContentHasher, fi os.FileInfo, w io.WriteCloser) (*hashedWriter, error) { + stat, ok := fi.Sys().(*Stat) + if !ok { + return nil, errors.Errorf("invalid change without stat information") + } + + h, err := ch(stat) if err != nil { return nil, err } @@ -249,15 +259,15 @@ func newHashWriter(p string, fi os.FileInfo, w io.WriteCloser) (*hashedWriter, e } func (hw *hashedWriter) Close() error { - hw.sum = string(hex.EncodeToString(hw.h.Sum(nil))) + hw.dgst = digest.NewDigest(digest.SHA256, hw.h) if hw.w != nil { return hw.w.Close() } return nil } -func (hw *hashedWriter) Hash() string { - return hw.sum +func (hw *hashedWriter) Digest() digest.Digest { + return hw.dgst } type lazyFileWriter struct { @@ -310,44 +320,3 @@ func nextSuffix() string { randmu.Unlock() return strconv.Itoa(int(1e9 + r%1e9))[1:] } - -func NewTarsumHash(p string, fi os.FileInfo) (hash.Hash, error) { - stat, ok := fi.Sys().(*Stat) - link := "" - if ok { - link = stat.Linkname - } - if fi.IsDir() { - p += string(os.PathSeparator) - } - h, err := archive.FileInfoHeader(p, fi, link) - if err != nil { - return nil, err - } - h.Name = p - if ok { - h.Uid = int(stat.Uid) - h.Gid = int(stat.Gid) - h.Linkname = stat.Linkname - if stat.Xattrs != nil { - h.Xattrs = make(map[string]string) - for k, v := range stat.Xattrs { - h.Xattrs[k] = string(v) - } - } - } - tsh := &tarsumHash{h: h, Hash: sha256.New()} - tsh.Reset() - return tsh, nil -} - -// Reset resets the Hash to its initial state. -func (tsh *tarsumHash) Reset() { - tsh.Hash.Reset() - tarsum.WriteV1Header(tsh.h, tsh.Hash) -} - -type tarsumHash struct { - hash.Hash - h *tar.Header -} diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_darwin.go b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_darwin.go new file mode 100644 index 0000000000..94d3324acf --- /dev/null +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_darwin.go @@ -0,0 +1,7 @@ +// +build darwin + +package fsutil + +func chtimes(path string, un int64) error { + return nil +} diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_linux.go b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_linux.go index c6d97eb0a6..74f08a15ca 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_linux.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_linux.go @@ -3,36 +3,10 @@ package fsutil import ( - "os" - "syscall" - "github.com/pkg/errors" - "github.com/stevvooe/continuity/sysx" "golang.org/x/sys/unix" ) -func rewriteMetadata(p string, stat *Stat) error { - for key, value := range stat.Xattrs { - sysx.Setxattr(p, key, value, 0) - } - - if err := os.Lchown(p, int(stat.Uid), int(stat.Gid)); err != nil { - return errors.Wrapf(err, "failed to lchown %s", p) - } - - if os.FileMode(stat.Mode)&os.ModeSymlink == 0 { - if err := os.Chmod(p, os.FileMode(stat.Mode)); err != nil { - return errors.Wrapf(err, "failed to chown %s", p) - } - } - - if err := chtimes(p, stat.ModTime); err != nil { - return errors.Wrapf(err, "failed to chtimes %s", p) - } - - return nil -} - func chtimes(path string, un int64) error { var utimes [2]unix.Timespec utimes[0] = unix.NsecToTimespec(un) @@ -44,21 +18,3 @@ func chtimes(path string, un int64) error { return nil } - -// handleTarTypeBlockCharFifo is an OS-specific helper function used by -// createTarFile to handle the following types of header: Block; Char; Fifo -func handleTarTypeBlockCharFifo(path string, stat *Stat) error { - mode := uint32(stat.Mode & 07777) - if os.FileMode(stat.Mode)&os.ModeCharDevice != 0 { - mode |= syscall.S_IFCHR - } else if os.FileMode(stat.Mode)&os.ModeNamedPipe != 0 { - mode |= syscall.S_IFIFO - } else { - mode |= syscall.S_IFBLK - } - - if err := syscall.Mknod(path, mode, int(mkdev(stat.Devmajor, stat.Devminor))); err != nil { - return err - } - return nil -} diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_unix.go b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_unix.go new file mode 100644 index 0000000000..5f51fce3b4 --- /dev/null +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/diskwriter_unix.go @@ -0,0 +1,51 @@ +// +build !windows + +package fsutil + +import ( + "os" + "syscall" + + "github.com/pkg/errors" + "github.com/stevvooe/continuity/sysx" +) + +func rewriteMetadata(p string, stat *Stat) error { + for key, value := range stat.Xattrs { + sysx.Setxattr(p, key, value, 0) + } + + if err := os.Lchown(p, int(stat.Uid), int(stat.Gid)); err != nil { + return errors.Wrapf(err, "failed to lchown %s", p) + } + + if os.FileMode(stat.Mode)&os.ModeSymlink == 0 { + if err := os.Chmod(p, os.FileMode(stat.Mode)); err != nil { + return errors.Wrapf(err, "failed to chown %s", p) + } + } + + if err := chtimes(p, stat.ModTime); err != nil { + return errors.Wrapf(err, "failed to chtimes %s", p) + } + + return nil +} + +// handleTarTypeBlockCharFifo is an OS-specific helper function used by +// createTarFile to handle the following types of header: Block; Char; Fifo +func handleTarTypeBlockCharFifo(path string, stat *Stat) error { + mode := uint32(stat.Mode & 07777) + if os.FileMode(stat.Mode)&os.ModeCharDevice != 0 { + mode |= syscall.S_IFCHR + } else if os.FileMode(stat.Mode)&os.ModeNamedPipe != 0 { + mode |= syscall.S_IFIFO + } else { + mode |= syscall.S_IFBLK + } + + if err := syscall.Mknod(path, mode, int(mkdev(stat.Devmajor, stat.Devminor))); err != nil { + return err + } + return nil +} diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/receive.go b/components/engine/vendor/github.com/tonistiigi/fsutil/receive.go index e7cee2b7ce..233c28b70e 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/receive.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/receive.go @@ -1,5 +1,3 @@ -// +build linux windows - package fsutil import ( @@ -12,29 +10,45 @@ import ( "golang.org/x/sync/errgroup" ) -func Receive(ctx context.Context, conn Stream, dest string, notifyHashed ChangeFunc) error { +type ReceiveOpt struct { + NotifyHashed ChangeFunc + ContentHasher ContentHasher + ProgressCb func(int, bool) + Merge bool + Filter FilterFunc +} + +func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() r := &receiver{ - conn: &syncStream{Stream: conn}, - dest: dest, - files: make(map[string]uint32), - pipes: make(map[uint32]io.WriteCloser), - notifyHashed: notifyHashed, + conn: &syncStream{Stream: conn}, + dest: dest, + files: make(map[string]uint32), + pipes: make(map[uint32]io.WriteCloser), + notifyHashed: opt.NotifyHashed, + contentHasher: opt.ContentHasher, + progressCb: opt.ProgressCb, + merge: opt.Merge, + filter: opt.Filter, } return r.run(ctx) } type receiver struct { - dest string - conn Stream - files map[string]uint32 - pipes map[uint32]io.WriteCloser - mu sync.RWMutex - muPipes sync.RWMutex + dest string + conn Stream + files map[string]uint32 + pipes map[uint32]io.WriteCloser + mu sync.RWMutex + muPipes sync.RWMutex + progressCb func(int, bool) + merge bool + filter FilterFunc notifyHashed ChangeFunc + contentHasher ContentHasher orderValidator Validator hlValidator Hardlinks } @@ -81,8 +95,10 @@ func (r *receiver) run(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) dw, err := NewDiskWriter(ctx, r.dest, DiskWriterOpt{ - AsyncDataCb: r.asyncDataFunc, - NotifyCb: r.notifyHashed, + AsyncDataCb: r.asyncDataFunc, + NotifyCb: r.notifyHashed, + ContentHasher: r.contentHasher, + Filter: r.filter, }) if err != nil { return err @@ -91,7 +107,11 @@ func (r *receiver) run(ctx context.Context) error { w := newDynamicWalker() g.Go(func() error { - err := doubleWalkDiff(ctx, dw.HandleChange, GetWalkerFn(r.dest), w.fill) + destWalker := emptyWalker + if !r.merge { + destWalker = GetWalkerFn(r.dest) + } + err := doubleWalkDiff(ctx, dw.HandleChange, destWalker, w.fill) if err != nil { return err } @@ -105,12 +125,23 @@ func (r *receiver) run(ctx context.Context) error { g.Go(func() error { var i uint32 = 0 + size := 0 + if r.progressCb != nil { + defer func() { + r.progressCb(size, true) + }() + } var p Packet for { p = Packet{Data: p.Data[:0]} if err := r.conn.RecvMsg(&p); err != nil { return err } + if r.progressCb != nil { + size += p.Size() + r.progressCb(size, false) + } + switch p.Type { case PACKET_STAT: if p.Stat == nil { diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/receive_unsupported.go b/components/engine/vendor/github.com/tonistiigi/fsutil/receive_unsupported.go deleted file mode 100644 index 8e83342374..0000000000 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/receive_unsupported.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !linux,!windows - -package fsutil - -import ( - "runtime" - - "github.com/pkg/errors" - "golang.org/x/net/context" -) - -func Receive(ctx context.Context, conn Stream, dest string, notifyHashed ChangeFunc) error { - return errors.Errorf("receive is unsupported in %s", runtime.GOOS) -} diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/validator.go b/components/engine/vendor/github.com/tonistiigi/fsutil/validator.go index e4a5eba66b..2bd1287a85 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/validator.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/validator.go @@ -2,7 +2,8 @@ package fsutil import ( "os" - "path/filepath" + "path" + "runtime" "sort" "strings" @@ -26,14 +27,17 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err if v.parentDirs == nil { v.parentDirs = make([]parent, 1, 10) } - if p != filepath.Clean(p) { + if runtime.GOOS == "windows" { + p = strings.Replace(p, "\\", "", -1) + } + if p != path.Clean(p) { return errors.Errorf("invalid unclean path %s", p) } - if filepath.IsAbs(p) { + if path.IsAbs(p) { return errors.Errorf("abolute path %s not allowed", p) } - dir := filepath.Dir(p) - base := filepath.Base(p) + dir := path.Dir(p) + base := path.Base(p) if dir == "." { dir = "" } @@ -51,12 +55,12 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err } if dir != v.parentDirs[len(v.parentDirs)-1].dir || v.parentDirs[i].last >= base { - return errors.Errorf("changes out of order: %q %q", p, filepath.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) + return errors.Errorf("changes out of order: %q %q", p, path.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) } v.parentDirs[i].last = base if kind != ChangeKindDelete && fi.IsDir() { v.parentDirs = append(v.parentDirs, parent{ - dir: filepath.Join(dir, base), + dir: path.Join(dir, base), last: "", }) } diff --git a/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go b/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go index bfec609b5a..db1af56b49 100644 --- a/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go +++ b/components/engine/vendor/github.com/tonistiigi/fsutil/walker.go @@ -13,8 +13,9 @@ import ( ) type WalkOpt struct { - IncludePaths []string // todo: remove? + IncludePatterns []string ExcludePatterns []string + Map func(*Stat) bool } func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error { @@ -57,9 +58,9 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if opt != nil { - if opt.IncludePaths != nil { + if opt.IncludePatterns != nil { matched := false - for _, p := range opt.IncludePaths { + for _, p := range opt.IncludePatterns { if m, _ := filepath.Match(p, path); m { matched = true break @@ -138,7 +139,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err case <-ctx.Done(): return ctx.Err() default: - if err := fn(path, &StatInfo{stat}, nil); err != nil { + if opt != nil && opt.Map != nil { + if allowed := opt.Map(stat); !allowed { + return nil + } + } + if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil { return err } } From afac6be12395359157baf09c5febd0927d9f786d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 21 Sep 2017 22:07:45 -0700 Subject: [PATCH 18/34] builder: updates to session after vendor Signed-off-by: Tonis Tiigi Upstream-commit: d4729749023fde0c57cded0c8159dc85cd7ee448 Component: engine --- components/engine/builder/fscache/fscache.go | 46 +++++++++++++++++++ .../engine/builder/remotecontext/tarsum.go | 8 ++-- .../integration-cli/docker_api_build_test.go | 6 ++- components/engine/vendor.conf | 2 +- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/components/engine/builder/fscache/fscache.go b/components/engine/builder/fscache/fscache.go index faef7d1ac1..880cc9d10d 100644 --- a/components/engine/builder/fscache/fscache.go +++ b/components/engine/builder/fscache/fscache.go @@ -1,7 +1,10 @@ package fscache import ( + "archive/tar" + "crypto/sha256" "encoding/json" + "hash" "os" "path/filepath" "sort" @@ -11,8 +14,10 @@ import ( "github.com/boltdb/bolt" "github.com/docker/docker/builder" "github.com/docker/docker/builder/remotecontext" + "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/tarsum" "github.com/moby/buildkit/session/filesync" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -578,6 +583,10 @@ func (dc *detectChanges) MarkSupported(v bool) { dc.supported = v } +func (dc *detectChanges) ContentHasher() fsutil.ContentHasher { + return newTarsumHash +} + type wrappedContext struct { builder.Source closer func() error @@ -607,3 +616,40 @@ func (s sortableCacheSources) Less(i, j int) bool { func (s sortableCacheSources) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func newTarsumHash(stat *fsutil.Stat) (hash.Hash, error) { + fi := &fsutil.StatInfo{stat} + p := stat.Path + if fi.IsDir() { + p += string(os.PathSeparator) + } + h, err := archive.FileInfoHeader(p, fi, stat.Linkname) + if err != nil { + return nil, err + } + h.Name = p + h.Uid = int(stat.Uid) + h.Gid = int(stat.Gid) + h.Linkname = stat.Linkname + if stat.Xattrs != nil { + h.Xattrs = make(map[string]string) + for k, v := range stat.Xattrs { + h.Xattrs[k] = string(v) + } + } + + tsh := &tarsumHash{h: h, Hash: sha256.New()} + tsh.Reset() + return tsh, nil +} + +// Reset resets the Hash to its initial state. +func (tsh *tarsumHash) Reset() { + tsh.Hash.Reset() + tarsum.WriteV1Header(tsh.h, tsh.Hash) +} + +type tarsumHash struct { + hash.Hash + h *tar.Header +} diff --git a/components/engine/builder/remotecontext/tarsum.go b/components/engine/builder/remotecontext/tarsum.go index 6770eed871..370f13d80b 100644 --- a/components/engine/builder/remotecontext/tarsum.go +++ b/components/engine/builder/remotecontext/tarsum.go @@ -5,15 +5,15 @@ import ( "os" "sync" - iradix "github.com/hashicorp/go-immutable-radix" - "github.com/docker/docker/pkg/containerfs" + iradix "github.com/hashicorp/go-immutable-radix" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/tonistiigi/fsutil" ) type hashed interface { - Hash() string + Digest() digest.Digest } // CachableSource is a source that contains cache records for its contents @@ -110,7 +110,7 @@ func (cs *CachableSource) HandleChange(kind fsutil.ChangeKind, p string, fi os.F } hfi := &fileInfo{ - sum: h.Hash(), + sum: h.Digest().Hex(), } cs.txn.Insert([]byte(p), hfi) cs.mu.Unlock() diff --git a/components/engine/integration-cli/docker_api_build_test.go b/components/engine/integration-cli/docker_api_build_test.go index 423ebcf9fc..8c494f1427 100644 --- a/components/engine/integration-cli/docker_api_build_test.go +++ b/components/engine/integration-cli/docker_api_build_test.go @@ -586,7 +586,9 @@ func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) { sess, err := session.NewSession("foo1", "foo") assert.Nil(c, err) - fsProvider := filesync.NewFSSyncProvider(dir, nil) + fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{ + {Dir: dir}, + }) sess.Allow(fsProvider) g, ctx := errgroup.WithContext(context.Background()) @@ -596,7 +598,7 @@ func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) { }) g.Go(func() error { - res, body, err := request.Post("/build?remote=client-session&session="+sess.UUID(), func(req *http.Request) error { + res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error { req.Body = ioutil.NopCloser(strings.NewReader(dockerfile)) return nil }) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 98689c4b0b..246fdc14cc 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -27,7 +27,7 @@ github.com/imdario/mergo 0.2.1 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0 github.com/containerd/continuity 22694c680ee48fb8f50015b44618517e2bde77e8 -github.com/moby/buildkit c2dbdeb457ea665699a5d97f79eebfac4ab4726f https://github.com/tonistiigi/buildkit.git +github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8 github.com/tonistiigi/fsutil 1dedf6e90084bd88c4c518a15e68a37ed1370203 #get libnetwork packages From 28762b72424ce6ff7d2c1dcd5cfb87fd7bd202e9 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Sun, 24 Sep 2017 16:44:16 -0700 Subject: [PATCH 19/34] Vendoring libnetwork Fix for networkDB garbage collection (PR: https://github.com/docker/libnetwork/pull/1944) Added extra logs to monitor the netowrkDB status and number of entries per network Signed-off-by: Flavio Crisciani Upstream-commit: 04043428ea5ce679618aec2007b77ac51d0b6af0 Component: engine --- components/engine/vendor.conf | 2 +- .../libnetwork/drivers/bridge/bridge.go | 6 +- .../drivers/bridge/setup_ip_tables.go | 11 +- .../libnetwork/drivers/overlay/ov_endpoint.go | 11 +- .../drivers/solaris/overlay/ov_endpoint.go | 11 +- .../docker/libnetwork/endpoint_info.go | 6 +- .../docker/libnetwork/iptables/iptables.go | 11 +- .../docker/libnetwork/networkdb/broadcast.go | 2 + .../docker/libnetwork/networkdb/cluster.go | 94 +- .../docker/libnetwork/networkdb/delegate.go | 38 +- .../docker/libnetwork/networkdb/networkdb.go | 70 +- .../libnetwork/networkdb/networkdb.pb.go | 811 +++++++++++------- .../libnetwork/networkdb/networkdb.proto | 8 +- .../docker/libnetwork/sandbox_dns_unix.go | 6 +- 14 files changed, 642 insertions(+), 445 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index e90550db30..6c4f9aa25d 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -30,7 +30,7 @@ golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0 github.com/containerd/continuity 22694c680ee48fb8f50015b44618517e2bde77e8 #get libnetwork packages -github.com/docker/libnetwork 60e002dd61885e1cd909582f00f7eb4da634518a +github.com/docker/libnetwork 0f08d31bf0e640e0cdc6d5161227f87602d605c5 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go index 64a2743dcf..1742c8df31 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -763,11 +763,7 @@ func (d *driver) createNetwork(config *networkConfiguration) error { // Apply the prepared list of steps, and abort at the first error. bridgeSetup.queueStep(setupDeviceUp) - if err = bridgeSetup.apply(); err != nil { - return err - } - - return nil + return bridgeSetup.apply() } func (d *driver) DeleteNetwork(nid string) error { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go index f01b08dea0..da654449d2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go @@ -169,11 +169,7 @@ func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairp } // Set Accept on all non-intercontainer outgoing packets. - if err := programChainRule(outRule, "ACCEPT NON_ICC OUTGOING", enable); err != nil { - return err - } - - return nil + return programChainRule(outRule, "ACCEPT NON_ICC OUTGOING", enable) } func programChainRule(rule iptRule, ruleDescr string, insert bool) error { @@ -304,10 +300,7 @@ func setupInternalNetworkRules(bridgeIface string, addr net.Addr, icc, insert bo return err } // Set Inter Container Communication. - if err := setIcc(bridgeIface, icc, insert); err != nil { - return err - } - return nil + return setIcc(bridgeIface, icc, insert) } func clearEndpointConnections(nlh *netlink.Handle, ep *bridgeEndpoint) { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go index d0e63f6b7e..bb08de465c 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go @@ -144,11 +144,7 @@ func (d *driver) deleteEndpointFromStore(e *endpoint) error { return fmt.Errorf("overlay local store not initialized, ep not deleted") } - if err := d.localStore.DeleteObjectAtomic(e); err != nil { - return err - } - - return nil + return d.localStore.DeleteObjectAtomic(e) } func (d *driver) writeEndpointToStore(e *endpoint) error { @@ -156,10 +152,7 @@ func (d *driver) writeEndpointToStore(e *endpoint) error { return fmt.Errorf("overlay local store not initialized, ep not added") } - if err := d.localStore.PutObjectAtomic(e); err != nil { - return err - } - return nil + return d.localStore.PutObjectAtomic(e) } func (ep *endpoint) DataScope() string { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go index 21b33ffe85..9df7a5874b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go @@ -134,11 +134,7 @@ func (d *driver) deleteEndpointFromStore(e *endpoint) error { return fmt.Errorf("overlay local store not initialized, ep not deleted") } - if err := d.localStore.DeleteObjectAtomic(e); err != nil { - return err - } - - return nil + return d.localStore.DeleteObjectAtomic(e) } func (d *driver) writeEndpointToStore(e *endpoint) error { @@ -146,10 +142,7 @@ func (d *driver) writeEndpointToStore(e *endpoint) error { return fmt.Errorf("overlay local store not initialized, ep not added") } - if err := d.localStore.PutObjectAtomic(e); err != nil { - return err - } - return nil + return d.localStore.PutObjectAtomic(e) } func (ep *endpoint) DataScope() string { diff --git a/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go b/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go index b5d3fabcb2..68d3e8673d 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go +++ b/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go @@ -202,11 +202,7 @@ func (ep *endpoint) Info() EndpointInfo { return ep } - if epi := sb.getEndpoint(ep.ID()); epi != nil { - return epi - } - - return nil + return sb.getEndpoint(ep.ID()) } func (ep *endpoint) Iface() InterfaceInfo { diff --git a/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go b/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go index ecce0d3b2a..3e120059de 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go +++ b/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go @@ -276,11 +276,7 @@ func (c *ChainInfo) Forward(action Action, ip net.IP, port int, proto, destAddr "--dport", strconv.Itoa(destPort), "-j", "MASQUERADE", } - if err := ProgramRule(Nat, "POSTROUTING", action, args); err != nil { - return err - } - - return nil + return ProgramRule(Nat, "POSTROUTING", action, args) } // Link adds reciprocal ACCEPT rule for two supplied IP addresses. @@ -301,10 +297,7 @@ func (c *ChainInfo) Link(action Action, ip1, ip2 net.IP, port int, proto string, // reverse args[7], args[9] = args[9], args[7] args[10] = "--sport" - if err := ProgramRule(Filter, c.Name, action, args); err != nil { - return err - } - return nil + return ProgramRule(Filter, c.Name, action, args) } // ProgramRule adds the rule specified by args only if the diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go index 52e96ec639..8317ed03f6 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go @@ -134,6 +134,8 @@ func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname st TableName: tname, Key: key, Value: entry.value, + // The duration in second is a float that below would be truncated + ResidualReapTime: int32(entry.reapTime.Seconds()), } raw, err := encodeMessage(MessageTypeTableEvent, &tEvent) diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go index d15a5767ff..af6f5d9f7b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go @@ -17,11 +17,15 @@ import ( ) const ( - reapInterval = 30 * time.Minute - reapPeriod = 5 * time.Second - retryInterval = 1 * time.Second - nodeReapInterval = 24 * time.Hour - nodeReapPeriod = 2 * time.Hour + // The garbage collection logic for entries leverage the presence of the network. + // For this reason the expiration time of the network is put slightly higher than the entry expiration so that + // there is at least 5 extra cycle to make sure that all the entries are properly deleted before deleting the network. + reapEntryInterval = 30 * time.Minute + reapNetworkInterval = reapEntryInterval + 5*reapPeriod + reapPeriod = 5 * time.Second + retryInterval = 1 * time.Second + nodeReapInterval = 24 * time.Hour + nodeReapPeriod = 2 * time.Hour ) type logWriter struct{} @@ -300,8 +304,9 @@ func (nDB *NetworkDB) reconnectNode() { // the reaper runs. NOTE nDB.reapTableEntries updates the reapTime with a readlock. This // is safe as long as no other concurrent path touches the reapTime field. func (nDB *NetworkDB) reapState() { - nDB.reapNetworks() + // The reapTableEntries leverage the presence of the network so garbage collect entries first nDB.reapTableEntries() + nDB.reapNetworks() } func (nDB *NetworkDB) reapNetworks() { @@ -321,43 +326,51 @@ func (nDB *NetworkDB) reapNetworks() { } func (nDB *NetworkDB) reapTableEntries() { - var paths []string - + var nodeNetworks []string + // This is best effort, if the list of network changes will be picked up in the next cycle nDB.RLock() - nDB.indexes[byTable].Walk(func(path string, v interface{}) bool { - entry, ok := v.(*entry) - if !ok { - return false - } - - if !entry.deleting { - return false - } - if entry.reapTime > 0 { - entry.reapTime -= reapPeriod - return false - } - paths = append(paths, path) - return false - }) + for nid := range nDB.networks[nDB.config.NodeName] { + nodeNetworks = append(nodeNetworks, nid) + } nDB.RUnlock() - nDB.Lock() - for _, path := range paths { - params := strings.Split(path[1:], "/") - tname := params[0] - nid := params[1] - key := params[2] + cycleStart := time.Now() + // In order to avoid blocking the database for a long time, apply the garbage collection logic by network + // The lock is taken at the beginning of the cycle and the deletion is inline + for _, nid := range nodeNetworks { + nDB.Lock() + nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid), func(path string, v interface{}) bool { + // timeCompensation compensate in case the lock took some time to be released + timeCompensation := time.Since(cycleStart) + entry, ok := v.(*entry) + if !ok || !entry.deleting { + return false + } - if _, ok := nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)); !ok { - logrus.Errorf("Could not delete entry in table %s with network id %s and key %s as it does not exist", tname, nid, key) - } + // In this check we are adding an extra 1 second to guarantee that when the number is truncated to int32 to fit the packet + // for the tableEvent the number is always strictly > 1 and never 0 + if entry.reapTime > reapPeriod+timeCompensation+time.Second { + entry.reapTime -= reapPeriod + timeCompensation + return false + } - if _, ok := nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)); !ok { - logrus.Errorf("Could not delete entry in network %s with table name %s and key %s as it does not exist", nid, tname, key) - } + params := strings.Split(path[1:], "/") + nid := params[0] + tname := params[1] + key := params[2] + + okTable, okNetwork := nDB.deleteEntry(nid, tname, key) + if !okTable { + logrus.Errorf("Table tree delete failed, entry with key:%s does not exists in the table:%s network:%s", key, tname, nid) + } + if !okNetwork { + logrus.Errorf("Network tree delete failed, entry with key:%s does not exists in the network:%s table:%s", key, nid, tname) + } + + return false + }) + nDB.Unlock() } - nDB.Unlock() } func (nDB *NetworkDB) gossip() { @@ -406,8 +419,9 @@ func (nDB *NetworkDB) gossip() { // Collect stats and print the queue info, note this code is here also to have a view of the queues empty network.qMessagesSent += len(msgs) if printStats { - logrus.Infof("NetworkDB stats - Queue net:%s qLen:%d netPeers:%d netMsg/s:%d", - nid, broadcastQ.NumQueued(), broadcastQ.NumNodes(), network.qMessagesSent/int((nDB.config.StatsPrintPeriod/time.Second))) + logrus.Infof("NetworkDB stats - netID:%s leaving:%t netPeers:%d entries:%d Queue qLen:%d netMsg/s:%d", + nid, network.leaving, broadcastQ.NumNodes(), network.entriesNumber, broadcastQ.NumQueued(), + network.qMessagesSent/int((nDB.config.StatsPrintPeriod/time.Second))) network.qMessagesSent = 0 } @@ -572,6 +586,8 @@ func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited b TableName: params[1], Key: params[2], Value: entry.value, + // The duration in second is a float that below would be truncated + ResidualReapTime: int32(entry.reapTime.Seconds()), } msg, err := encodeMessage(MessageTypeTableEvent, &tEvent) diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go index ffaf94e8c8..28919cf3d2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go @@ -1,9 +1,9 @@ package networkdb import ( - "fmt" "net" "strings" + "time" "github.com/gogo/protobuf/proto" "github.com/sirupsen/logrus" @@ -165,7 +165,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool { n.ltime = nEvent.LTime n.leaving = nEvent.Type == NetworkEventTypeLeave if n.leaving { - n.reapTime = reapInterval + n.reapTime = reapNetworkInterval // The remote node is leaving the network, but not the gossip cluster. // Mark all its entries in deleted state, this will guarantee that @@ -198,8 +198,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool { } func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool { - // Update our local clock if the received messages has newer - // time. + // Update our local clock if the received messages has newer time. nDB.tableClock.Witness(tEvent.LTime) // Ignore the table events for networks that are in the process of going away @@ -235,20 +234,26 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool { node: tEvent.NodeName, value: tEvent.Value, deleting: tEvent.Type == TableEventTypeDelete, + reapTime: time.Duration(tEvent.ResidualReapTime) * time.Second, } - if e.deleting { - e.reapTime = reapInterval + // All the entries marked for deletion should have a reapTime set greater than 0 + // This case can happen if the cluster is running different versions of the engine where the old version does not have the + // field. If that is not the case, this can be a BUG + if e.deleting && e.reapTime == 0 { + logrus.Warnf("handleTableEvent object %+v has a 0 reapTime, is the cluster running the same docker engine version?", tEvent) + e.reapTime = reapEntryInterval } nDB.Lock() - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.TableName, tEvent.NetworkID, tEvent.Key), e) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), e) + nDB.createOrUpdateEntry(tEvent.NetworkID, tEvent.TableName, tEvent.Key, e) nDB.Unlock() if err != nil && tEvent.Type == TableEventTypeDelete { - // If it is a delete event and we didn't have the entry here don't repropagate - return true + // If it is a delete event and we did not have a state for it, don't propagate to the application + // If the residual reapTime is lower or equal to 1/6 of the total reapTime don't bother broadcasting it around + // most likely the cluster is already aware of it, if not who will sync with this node will catch the state too. + return e.reapTime > reapPeriod/6 } var op opType @@ -303,22 +308,17 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) { n, ok := nDB.networks[nDB.config.NodeName][tEvent.NetworkID] nDB.RUnlock() - if !ok { + // if the network is not there anymore, OR we are leaving the network OR the broadcast queue is not present + if !ok || n.leaving || n.tableBroadcasts == nil { return } - broadcastQ := n.tableBroadcasts - - if broadcastQ == nil { - return - } - - broadcastQ.QueueBroadcast(&tableEventMessage{ + n.tableBroadcasts.QueueBroadcast(&tableEventMessage{ msg: buf, id: tEvent.NetworkID, tname: tEvent.TableName, key: tEvent.Key, - node: nDB.config.NodeName, + node: tEvent.NodeName, }) } } diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go index 73dd999097..afdf32e2c2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go @@ -141,6 +141,11 @@ type network struct { // Number of gossip messages sent related to this network during the last stats collection period qMessagesSent int + + // Number of entries on the network. This value is the sum of all the entries of all the tables of a specific network. + // Its use is for statistics purposes. It keep tracks of database size and is printed per network every StatsPrintPeriod + // interval + entriesNumber int } // Config represents the configuration of the networdb instance and @@ -338,8 +343,7 @@ func (nDB *NetworkDB) CreateEntry(tname, nid, key string, value []byte) error { } nDB.Lock() - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + nDB.createOrUpdateEntry(nid, tname, key, entry) nDB.Unlock() return nil @@ -365,8 +369,7 @@ func (nDB *NetworkDB) UpdateEntry(tname, nid, key string, value []byte) error { } nDB.Lock() - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + nDB.createOrUpdateEntry(nid, tname, key, entry) nDB.Unlock() return nil @@ -402,7 +405,7 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error { node: nDB.config.NodeName, value: value, deleting: true, - reapTime: reapInterval, + reapTime: reapEntryInterval, } if err := nDB.sendTableEvent(TableEventTypeDelete, nid, tname, key, entry); err != nil { @@ -410,8 +413,7 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error { } nDB.Lock() - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + nDB.createOrUpdateEntry(nid, tname, key, entry) nDB.Unlock() return nil @@ -473,10 +475,10 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) { entry := &entry{ ltime: oldEntry.ltime, - node: node, + node: oldEntry.node, value: oldEntry.value, deleting: true, - reapTime: reapInterval, + reapTime: reapEntryInterval, } // we arrived at this point in 2 cases: @@ -488,12 +490,10 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) { // without doing a delete of all the objects entry.ltime++ } - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + nDB.createOrUpdateEntry(nid, tname, key, entry) } else { // the local node is leaving the network, all the entries of remote nodes can be safely removed - nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)) - nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)) + nDB.deleteEntry(nid, tname, key) } nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value)) @@ -513,8 +513,7 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) { nid := params[1] key := params[2] - nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)) - nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)) + nDB.deleteEntry(nid, tname, key) nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value)) return false @@ -558,7 +557,12 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error { nodeNetworks = make(map[string]*network) nDB.networks[nDB.config.NodeName] = nodeNetworks } - nodeNetworks[nid] = &network{id: nid, ltime: ltime} + n, ok := nodeNetworks[nid] + var entries int + if ok { + entries = n.entriesNumber + } + nodeNetworks[nid] = &network{id: nid, ltime: ltime, entriesNumber: entries} nodeNetworks[nid].tableBroadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { nDB.RLock() @@ -567,6 +571,7 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error { }, RetransmitMult: 4, } + nDB.addNetworkNode(nid, nDB.config.NodeName) networkNodes := nDB.networkNodes[nid] nDB.Unlock() @@ -614,8 +619,9 @@ func (nDB *NetworkDB) LeaveNetwork(nid string) error { return fmt.Errorf("could not find network %s while trying to leave", nid) } + logrus.Debugf("%s: leaving network %s", nDB.config.NodeName, nid) n.ltime = ltime - n.reapTime = reapInterval + n.reapTime = reapNetworkInterval n.leaving = true return nil } @@ -679,3 +685,33 @@ func (nDB *NetworkDB) updateLocalNetworkTime() { n.ltime = ltime } } + +// createOrUpdateEntry this function handles the creation or update of entries into the local +// tree store. It is also used to keep in sync the entries number of the network (all tables are aggregated) +func (nDB *NetworkDB) createOrUpdateEntry(nid, tname, key string, entry interface{}) (bool, bool) { + _, okTable := nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) + _, okNetwork := nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + if !okNetwork { + // Add only if it is an insert not an update + n, ok := nDB.networks[nDB.config.NodeName][nid] + if ok { + n.entriesNumber++ + } + } + return okTable, okNetwork +} + +// deleteEntry this function handles the deletion of entries into the local tree store. +// It is also used to keep in sync the entries number of the network (all tables are aggregated) +func (nDB *NetworkDB) deleteEntry(nid, tname, key string) (bool, bool) { + _, okTable := nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)) + _, okNetwork := nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)) + if okNetwork { + // Remove only if the delete is successful + n, ok := nDB.networks[nDB.config.NodeName][nid] + if ok { + n.entriesNumber-- + } + } + return okTable, okNetwork +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.pb.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.pb.go index dfbc7131fb..7087a57ca0 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.pb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: networkdb.proto -// DO NOT EDIT! /* Package networkdb is a generated protocol buffer package. @@ -28,9 +27,6 @@ import _ "github.com/gogo/protobuf/gogoproto" import github_com_hashicorp_serf_serf "github.com/hashicorp/serf/serf" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -42,7 +38,9 @@ var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. -const _ = proto.GoGoProtoPackageIsVersion1 +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package // MessageType enum defines all the core message types that networkdb // uses to communicate to peers. @@ -192,6 +190,20 @@ func (m *GossipMessage) Reset() { *m = GossipMessage{} } func (*GossipMessage) ProtoMessage() {} func (*GossipMessage) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{0} } +func (m *GossipMessage) GetType() MessageType { + if m != nil { + return m.Type + } + return MessageTypeInvalid +} + +func (m *GossipMessage) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + // NodeEvent message payload definition. type NodeEvent struct { Type NodeEvent_Type `protobuf:"varint,1,opt,name=type,proto3,enum=networkdb.NodeEvent_Type" json:"type,omitempty"` @@ -207,6 +219,20 @@ func (m *NodeEvent) Reset() { *m = NodeEvent{} } func (*NodeEvent) ProtoMessage() {} func (*NodeEvent) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{1} } +func (m *NodeEvent) GetType() NodeEvent_Type { + if m != nil { + return m.Type + } + return NodeEventTypeInvalid +} + +func (m *NodeEvent) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + // NetworkEvent message payload definition. type NetworkEvent struct { Type NetworkEvent_Type `protobuf:"varint,1,opt,name=type,proto3,enum=networkdb.NetworkEvent_Type" json:"type,omitempty"` @@ -224,6 +250,27 @@ func (m *NetworkEvent) Reset() { *m = NetworkEvent{} } func (*NetworkEvent) ProtoMessage() {} func (*NetworkEvent) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{2} } +func (m *NetworkEvent) GetType() NetworkEvent_Type { + if m != nil { + return m.Type + } + return NetworkEventTypeInvalid +} + +func (m *NetworkEvent) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + +func (m *NetworkEvent) GetNetworkID() string { + if m != nil { + return m.NetworkID + } + return "" +} + // NetworkEntry for push pull of networks. type NetworkEntry struct { // ID of the network @@ -241,6 +288,27 @@ func (m *NetworkEntry) Reset() { *m = NetworkEntry{} } func (*NetworkEntry) ProtoMessage() {} func (*NetworkEntry) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{3} } +func (m *NetworkEntry) GetNetworkID() string { + if m != nil { + return m.NetworkID + } + return "" +} + +func (m *NetworkEntry) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + +func (m *NetworkEntry) GetLeaving() bool { + if m != nil { + return m.Leaving + } + return false +} + // NetworkPushpull message payload definition. type NetworkPushPull struct { // Lamport time when this push pull was initiated. @@ -261,6 +329,13 @@ func (m *NetworkPushPull) GetNetworks() []*NetworkEntry { return nil } +func (m *NetworkPushPull) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + // TableEvent message payload definition. type TableEvent struct { Type TableEvent_Type `protobuf:"varint,1,opt,name=type,proto3,enum=networkdb.TableEvent_Type" json:"type,omitempty"` @@ -276,12 +351,63 @@ type TableEvent struct { Key string `protobuf:"bytes,6,opt,name=key,proto3" json:"key,omitempty"` // Entry value. Value []byte `protobuf:"bytes,7,opt,name=value,proto3" json:"value,omitempty"` + // Residual reap time for the entry before getting deleted in seconds + ResidualReapTime int32 `protobuf:"varint,8,opt,name=residual_reap_time,json=residualReapTime,proto3" json:"residual_reap_time,omitempty"` } func (m *TableEvent) Reset() { *m = TableEvent{} } func (*TableEvent) ProtoMessage() {} func (*TableEvent) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{5} } +func (m *TableEvent) GetType() TableEvent_Type { + if m != nil { + return m.Type + } + return TableEventTypeInvalid +} + +func (m *TableEvent) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + +func (m *TableEvent) GetNetworkID() string { + if m != nil { + return m.NetworkID + } + return "" +} + +func (m *TableEvent) GetTableName() string { + if m != nil { + return m.TableName + } + return "" +} + +func (m *TableEvent) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *TableEvent) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *TableEvent) GetResidualReapTime() int32 { + if m != nil { + return m.ResidualReapTime + } + return 0 +} + // BulkSync message payload definition. type BulkSyncMessage struct { // Lamport time when this bulk sync was initiated. @@ -302,6 +428,34 @@ func (m *BulkSyncMessage) Reset() { *m = BulkSyncMessage{} } func (*BulkSyncMessage) ProtoMessage() {} func (*BulkSyncMessage) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{6} } +func (m *BulkSyncMessage) GetUnsolicited() bool { + if m != nil { + return m.Unsolicited + } + return false +} + +func (m *BulkSyncMessage) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + +func (m *BulkSyncMessage) GetNetworks() []string { + if m != nil { + return m.Networks + } + return nil +} + +func (m *BulkSyncMessage) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + // Compound message payload definition. type CompoundMessage struct { // A list of simple messages. @@ -322,7 +476,7 @@ func (m *CompoundMessage) GetMessages() []*CompoundMessage_SimpleMessage { type CompoundMessage_SimpleMessage struct { // Bytestring payload of a message constructed using // other message type definitions. - Payload []byte `protobuf:"bytes,1,opt,name=Payload,json=payload,proto3" json:"Payload,omitempty"` + Payload []byte `protobuf:"bytes,1,opt,name=Payload,proto3" json:"Payload,omitempty"` } func (m *CompoundMessage_SimpleMessage) Reset() { *m = CompoundMessage_SimpleMessage{} } @@ -331,6 +485,13 @@ func (*CompoundMessage_SimpleMessage) Descriptor() ([]byte, []int) { return fileDescriptorNetworkdb, []int{7, 0} } +func (m *CompoundMessage_SimpleMessage) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + func init() { proto.RegisterType((*GossipMessage)(nil), "networkdb.GossipMessage") proto.RegisterType((*NodeEvent)(nil), "networkdb.NodeEvent") @@ -413,7 +574,7 @@ func (this *TableEvent) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&networkdb.TableEvent{") s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n") s = append(s, "LTime: "+fmt.Sprintf("%#v", this.LTime)+",\n") @@ -422,6 +583,7 @@ func (this *TableEvent) GoString() string { s = append(s, "TableName: "+fmt.Sprintf("%#v", this.TableName)+",\n") s = append(s, "Key: "+fmt.Sprintf("%#v", this.Key)+",\n") s = append(s, "Value: "+fmt.Sprintf("%#v", this.Value)+",\n") + s = append(s, "ResidualReapTime: "+fmt.Sprintf("%#v", this.ResidualReapTime)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -469,197 +631,180 @@ func valueToGoStringNetworkdb(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringNetworkdb(e map[int32]github_com_gogo_protobuf_proto.Extension) string { - if e == nil { - return "nil" - } - s := "map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "}" - return s -} -func (m *GossipMessage) Marshal() (data []byte, err error) { +func (m *GossipMessage) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *GossipMessage) MarshalTo(data []byte) (int, error) { +func (m *GossipMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.Type != 0 { - data[i] = 0x8 + dAtA[i] = 0x8 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.Type)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.Type)) } if len(m.Data) > 0 { - data[i] = 0x12 + dAtA[i] = 0x12 i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.Data))) - i += copy(data[i:], m.Data) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) } return i, nil } -func (m *NodeEvent) Marshal() (data []byte, err error) { +func (m *NodeEvent) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *NodeEvent) MarshalTo(data []byte) (int, error) { +func (m *NodeEvent) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.Type != 0 { - data[i] = 0x8 + dAtA[i] = 0x8 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.Type)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.Type)) } if m.LTime != 0 { - data[i] = 0x10 + dAtA[i] = 0x10 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.LTime)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.LTime)) } if len(m.NodeName) > 0 { - data[i] = 0x1a + dAtA[i] = 0x1a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NodeName))) - i += copy(data[i:], m.NodeName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) } return i, nil } -func (m *NetworkEvent) Marshal() (data []byte, err error) { +func (m *NetworkEvent) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *NetworkEvent) MarshalTo(data []byte) (int, error) { +func (m *NetworkEvent) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.Type != 0 { - data[i] = 0x8 + dAtA[i] = 0x8 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.Type)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.Type)) } if m.LTime != 0 { - data[i] = 0x10 + dAtA[i] = 0x10 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.LTime)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.LTime)) } if len(m.NodeName) > 0 { - data[i] = 0x1a + dAtA[i] = 0x1a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NodeName))) - i += copy(data[i:], m.NodeName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) } if len(m.NetworkID) > 0 { - data[i] = 0x22 + dAtA[i] = 0x22 i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NetworkID))) - i += copy(data[i:], m.NetworkID) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NetworkID))) + i += copy(dAtA[i:], m.NetworkID) } return i, nil } -func (m *NetworkEntry) Marshal() (data []byte, err error) { +func (m *NetworkEntry) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *NetworkEntry) MarshalTo(data []byte) (int, error) { +func (m *NetworkEntry) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if len(m.NetworkID) > 0 { - data[i] = 0xa + dAtA[i] = 0xa i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NetworkID))) - i += copy(data[i:], m.NetworkID) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NetworkID))) + i += copy(dAtA[i:], m.NetworkID) } if m.LTime != 0 { - data[i] = 0x10 + dAtA[i] = 0x10 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.LTime)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.LTime)) } if len(m.NodeName) > 0 { - data[i] = 0x1a + dAtA[i] = 0x1a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NodeName))) - i += copy(data[i:], m.NodeName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) } if m.Leaving { - data[i] = 0x20 + dAtA[i] = 0x20 i++ if m.Leaving { - data[i] = 1 + dAtA[i] = 1 } else { - data[i] = 0 + dAtA[i] = 0 } i++ } return i, nil } -func (m *NetworkPushPull) Marshal() (data []byte, err error) { +func (m *NetworkPushPull) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *NetworkPushPull) MarshalTo(data []byte) (int, error) { +func (m *NetworkPushPull) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.LTime != 0 { - data[i] = 0x8 + dAtA[i] = 0x8 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.LTime)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.LTime)) } if len(m.Networks) > 0 { for _, msg := range m.Networks { - data[i] = 0x12 + dAtA[i] = 0x12 i++ - i = encodeVarintNetworkdb(data, i, uint64(msg.Size())) - n, err := msg.MarshalTo(data[i:]) + i = encodeVarintNetworkdb(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -667,153 +812,158 @@ func (m *NetworkPushPull) MarshalTo(data []byte) (int, error) { } } if len(m.NodeName) > 0 { - data[i] = 0x1a + dAtA[i] = 0x1a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NodeName))) - i += copy(data[i:], m.NodeName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) } return i, nil } -func (m *TableEvent) Marshal() (data []byte, err error) { +func (m *TableEvent) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *TableEvent) MarshalTo(data []byte) (int, error) { +func (m *TableEvent) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.Type != 0 { - data[i] = 0x8 + dAtA[i] = 0x8 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.Type)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.Type)) } if m.LTime != 0 { - data[i] = 0x10 + dAtA[i] = 0x10 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.LTime)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.LTime)) } if len(m.NodeName) > 0 { - data[i] = 0x1a + dAtA[i] = 0x1a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NodeName))) - i += copy(data[i:], m.NodeName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) } if len(m.NetworkID) > 0 { - data[i] = 0x22 + dAtA[i] = 0x22 i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NetworkID))) - i += copy(data[i:], m.NetworkID) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NetworkID))) + i += copy(dAtA[i:], m.NetworkID) } if len(m.TableName) > 0 { - data[i] = 0x2a + dAtA[i] = 0x2a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.TableName))) - i += copy(data[i:], m.TableName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.TableName))) + i += copy(dAtA[i:], m.TableName) } if len(m.Key) > 0 { - data[i] = 0x32 + dAtA[i] = 0x32 i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.Key))) - i += copy(data[i:], m.Key) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) } if len(m.Value) > 0 { - data[i] = 0x3a + dAtA[i] = 0x3a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.Value))) - i += copy(data[i:], m.Value) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.Value))) + i += copy(dAtA[i:], m.Value) + } + if m.ResidualReapTime != 0 { + dAtA[i] = 0x40 + i++ + i = encodeVarintNetworkdb(dAtA, i, uint64(m.ResidualReapTime)) } return i, nil } -func (m *BulkSyncMessage) Marshal() (data []byte, err error) { +func (m *BulkSyncMessage) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *BulkSyncMessage) MarshalTo(data []byte) (int, error) { +func (m *BulkSyncMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if m.LTime != 0 { - data[i] = 0x8 + dAtA[i] = 0x8 i++ - i = encodeVarintNetworkdb(data, i, uint64(m.LTime)) + i = encodeVarintNetworkdb(dAtA, i, uint64(m.LTime)) } if m.Unsolicited { - data[i] = 0x10 + dAtA[i] = 0x10 i++ if m.Unsolicited { - data[i] = 1 + dAtA[i] = 1 } else { - data[i] = 0 + dAtA[i] = 0 } i++ } if len(m.NodeName) > 0 { - data[i] = 0x1a + dAtA[i] = 0x1a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.NodeName))) - i += copy(data[i:], m.NodeName) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.NodeName))) + i += copy(dAtA[i:], m.NodeName) } if len(m.Networks) > 0 { for _, s := range m.Networks { - data[i] = 0x22 + dAtA[i] = 0x22 i++ l = len(s) for l >= 1<<7 { - data[i] = uint8(uint64(l)&0x7f | 0x80) + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) l >>= 7 i++ } - data[i] = uint8(l) + dAtA[i] = uint8(l) i++ - i += copy(data[i:], s) + i += copy(dAtA[i:], s) } } if len(m.Payload) > 0 { - data[i] = 0x2a + dAtA[i] = 0x2a i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.Payload))) - i += copy(data[i:], m.Payload) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.Payload))) + i += copy(dAtA[i:], m.Payload) } return i, nil } -func (m *CompoundMessage) Marshal() (data []byte, err error) { +func (m *CompoundMessage) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *CompoundMessage) MarshalTo(data []byte) (int, error) { +func (m *CompoundMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if len(m.Messages) > 0 { for _, msg := range m.Messages { - data[i] = 0xa + dAtA[i] = 0xa i++ - i = encodeVarintNetworkdb(data, i, uint64(msg.Size())) - n, err := msg.MarshalTo(data[i:]) + i = encodeVarintNetworkdb(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -823,55 +973,55 @@ func (m *CompoundMessage) MarshalTo(data []byte) (int, error) { return i, nil } -func (m *CompoundMessage_SimpleMessage) Marshal() (data []byte, err error) { +func (m *CompoundMessage_SimpleMessage) Marshal() (dAtA []byte, err error) { size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } - return data[:n], nil + return dAtA[:n], nil } -func (m *CompoundMessage_SimpleMessage) MarshalTo(data []byte) (int, error) { +func (m *CompoundMessage_SimpleMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l if len(m.Payload) > 0 { - data[i] = 0xa + dAtA[i] = 0xa i++ - i = encodeVarintNetworkdb(data, i, uint64(len(m.Payload))) - i += copy(data[i:], m.Payload) + i = encodeVarintNetworkdb(dAtA, i, uint64(len(m.Payload))) + i += copy(dAtA[i:], m.Payload) } return i, nil } -func encodeFixed64Networkdb(data []byte, offset int, v uint64) int { - data[offset] = uint8(v) - data[offset+1] = uint8(v >> 8) - data[offset+2] = uint8(v >> 16) - data[offset+3] = uint8(v >> 24) - data[offset+4] = uint8(v >> 32) - data[offset+5] = uint8(v >> 40) - data[offset+6] = uint8(v >> 48) - data[offset+7] = uint8(v >> 56) +func encodeFixed64Networkdb(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) return offset + 8 } -func encodeFixed32Networkdb(data []byte, offset int, v uint32) int { - data[offset] = uint8(v) - data[offset+1] = uint8(v >> 8) - data[offset+2] = uint8(v >> 16) - data[offset+3] = uint8(v >> 24) +func encodeFixed32Networkdb(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) return offset + 4 } -func encodeVarintNetworkdb(data []byte, offset int, v uint64) int { +func encodeVarintNetworkdb(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { - data[offset] = uint8(v&0x7f | 0x80) + dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } - data[offset] = uint8(v) + dAtA[offset] = uint8(v) return offset + 1 } func (m *GossipMessage) Size() (n int) { @@ -991,6 +1141,9 @@ func (m *TableEvent) Size() (n int) { if l > 0 { n += 1 + l + sovNetworkdb(uint64(l)) } + if m.ResidualReapTime != 0 { + n += 1 + sovNetworkdb(uint64(m.ResidualReapTime)) + } return n } @@ -1128,6 +1281,7 @@ func (this *TableEvent) String() string { `TableName:` + fmt.Sprintf("%v", this.TableName) + `,`, `Key:` + fmt.Sprintf("%v", this.Key) + `,`, `Value:` + fmt.Sprintf("%v", this.Value) + `,`, + `ResidualReapTime:` + fmt.Sprintf("%v", this.ResidualReapTime) + `,`, `}`, }, "") return s @@ -1174,8 +1328,8 @@ func valueToStringNetworkdb(v interface{}) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("*%v", pv) } -func (m *GossipMessage) Unmarshal(data []byte) error { - l := len(data) +func (m *GossipMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1187,7 +1341,7 @@ func (m *GossipMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1215,7 +1369,7 @@ func (m *GossipMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.Type |= (MessageType(b) & 0x7F) << shift if b < 0x80 { @@ -1234,7 +1388,7 @@ func (m *GossipMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -1248,14 +1402,14 @@ func (m *GossipMessage) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Data = append(m.Data[:0], data[iNdEx:postIndex]...) + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) if m.Data == nil { m.Data = []byte{} } iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -1274,8 +1428,8 @@ func (m *GossipMessage) Unmarshal(data []byte) error { } return nil } -func (m *NodeEvent) Unmarshal(data []byte) error { - l := len(data) +func (m *NodeEvent) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1287,7 +1441,7 @@ func (m *NodeEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1315,7 +1469,7 @@ func (m *NodeEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.Type |= (NodeEvent_Type(b) & 0x7F) << shift if b < 0x80 { @@ -1334,7 +1488,7 @@ func (m *NodeEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.LTime |= (github_com_hashicorp_serf_serf.LamportTime(b) & 0x7F) << shift if b < 0x80 { @@ -1353,7 +1507,7 @@ func (m *NodeEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1368,11 +1522,11 @@ func (m *NodeEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeName = string(data[iNdEx:postIndex]) + m.NodeName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -1391,8 +1545,8 @@ func (m *NodeEvent) Unmarshal(data []byte) error { } return nil } -func (m *NetworkEvent) Unmarshal(data []byte) error { - l := len(data) +func (m *NetworkEvent) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1404,7 +1558,7 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1432,7 +1586,7 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.Type |= (NetworkEvent_Type(b) & 0x7F) << shift if b < 0x80 { @@ -1451,7 +1605,7 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.LTime |= (github_com_hashicorp_serf_serf.LamportTime(b) & 0x7F) << shift if b < 0x80 { @@ -1470,7 +1624,7 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1485,7 +1639,7 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeName = string(data[iNdEx:postIndex]) + m.NodeName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { @@ -1499,7 +1653,7 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1514,11 +1668,11 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NetworkID = string(data[iNdEx:postIndex]) + m.NetworkID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -1537,8 +1691,8 @@ func (m *NetworkEvent) Unmarshal(data []byte) error { } return nil } -func (m *NetworkEntry) Unmarshal(data []byte) error { - l := len(data) +func (m *NetworkEntry) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1550,7 +1704,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1578,7 +1732,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1593,7 +1747,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NetworkID = string(data[iNdEx:postIndex]) + m.NetworkID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 0 { @@ -1607,7 +1761,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.LTime |= (github_com_hashicorp_serf_serf.LamportTime(b) & 0x7F) << shift if b < 0x80 { @@ -1626,7 +1780,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1641,7 +1795,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeName = string(data[iNdEx:postIndex]) + m.NodeName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 0 { @@ -1655,7 +1809,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ v |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -1665,7 +1819,7 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { m.Leaving = bool(v != 0) default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -1684,8 +1838,8 @@ func (m *NetworkEntry) Unmarshal(data []byte) error { } return nil } -func (m *NetworkPushPull) Unmarshal(data []byte) error { - l := len(data) +func (m *NetworkPushPull) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1697,7 +1851,7 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1725,7 +1879,7 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.LTime |= (github_com_hashicorp_serf_serf.LamportTime(b) & 0x7F) << shift if b < 0x80 { @@ -1744,7 +1898,7 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ msglen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -1759,7 +1913,7 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { return io.ErrUnexpectedEOF } m.Networks = append(m.Networks, &NetworkEntry{}) - if err := m.Networks[len(m.Networks)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + if err := m.Networks[len(m.Networks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -1775,7 +1929,7 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1790,11 +1944,11 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeName = string(data[iNdEx:postIndex]) + m.NodeName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -1813,8 +1967,8 @@ func (m *NetworkPushPull) Unmarshal(data []byte) error { } return nil } -func (m *TableEvent) Unmarshal(data []byte) error { - l := len(data) +func (m *TableEvent) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -1826,7 +1980,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1854,7 +2008,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.Type |= (TableEvent_Type(b) & 0x7F) << shift if b < 0x80 { @@ -1873,7 +2027,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.LTime |= (github_com_hashicorp_serf_serf.LamportTime(b) & 0x7F) << shift if b < 0x80 { @@ -1892,7 +2046,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1907,7 +2061,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeName = string(data[iNdEx:postIndex]) + m.NodeName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { @@ -1921,7 +2075,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1936,7 +2090,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NetworkID = string(data[iNdEx:postIndex]) + m.NetworkID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 5: if wireType != 2 { @@ -1950,7 +2104,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1965,7 +2119,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.TableName = string(data[iNdEx:postIndex]) + m.TableName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 6: if wireType != 2 { @@ -1979,7 +2133,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -1994,7 +2148,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Key = string(data[iNdEx:postIndex]) + m.Key = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 7: if wireType != 2 { @@ -2008,7 +2162,7 @@ func (m *TableEvent) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -2022,14 +2176,33 @@ func (m *TableEvent) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Value = append(m.Value[:0], data[iNdEx:postIndex]...) + m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) if m.Value == nil { m.Value = []byte{} } iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ResidualReapTime", wireType) + } + m.ResidualReapTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNetworkdb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ResidualReapTime |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -2048,8 +2221,8 @@ func (m *TableEvent) Unmarshal(data []byte) error { } return nil } -func (m *BulkSyncMessage) Unmarshal(data []byte) error { - l := len(data) +func (m *BulkSyncMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -2061,7 +2234,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2089,7 +2262,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ m.LTime |= (github_com_hashicorp_serf_serf.LamportTime(b) & 0x7F) << shift if b < 0x80 { @@ -2108,7 +2281,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ v |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -2128,7 +2301,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2143,7 +2316,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NodeName = string(data[iNdEx:postIndex]) + m.NodeName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { @@ -2157,7 +2330,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2172,7 +2345,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Networks = append(m.Networks, string(data[iNdEx:postIndex])) + m.Networks = append(m.Networks, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 5: if wireType != 2 { @@ -2186,7 +2359,7 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -2200,14 +2373,14 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Payload = append(m.Payload[:0], data[iNdEx:postIndex]...) + m.Payload = append(m.Payload[:0], dAtA[iNdEx:postIndex]...) if m.Payload == nil { m.Payload = []byte{} } iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -2226,8 +2399,8 @@ func (m *BulkSyncMessage) Unmarshal(data []byte) error { } return nil } -func (m *CompoundMessage) Unmarshal(data []byte) error { - l := len(data) +func (m *CompoundMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -2239,7 +2412,7 @@ func (m *CompoundMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2267,7 +2440,7 @@ func (m *CompoundMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ msglen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -2282,13 +2455,13 @@ func (m *CompoundMessage) Unmarshal(data []byte) error { return io.ErrUnexpectedEOF } m.Messages = append(m.Messages, &CompoundMessage_SimpleMessage{}) - if err := m.Messages[len(m.Messages)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + if err := m.Messages[len(m.Messages)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -2307,8 +2480,8 @@ func (m *CompoundMessage) Unmarshal(data []byte) error { } return nil } -func (m *CompoundMessage_SimpleMessage) Unmarshal(data []byte) error { - l := len(data) +func (m *CompoundMessage_SimpleMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx @@ -2320,7 +2493,7 @@ func (m *CompoundMessage_SimpleMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2348,7 +2521,7 @@ func (m *CompoundMessage_SimpleMessage) Unmarshal(data []byte) error { if iNdEx >= l { return io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -2362,14 +2535,14 @@ func (m *CompoundMessage_SimpleMessage) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Payload = append(m.Payload[:0], data[iNdEx:postIndex]...) + m.Payload = append(m.Payload[:0], dAtA[iNdEx:postIndex]...) if m.Payload == nil { m.Payload = []byte{} } iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipNetworkdb(data[iNdEx:]) + skippy, err := skipNetworkdb(dAtA[iNdEx:]) if err != nil { return err } @@ -2388,8 +2561,8 @@ func (m *CompoundMessage_SimpleMessage) Unmarshal(data []byte) error { } return nil } -func skipNetworkdb(data []byte) (n int, err error) { - l := len(data) +func skipNetworkdb(dAtA []byte) (n int, err error) { + l := len(dAtA) iNdEx := 0 for iNdEx < l { var wire uint64 @@ -2400,7 +2573,7 @@ func skipNetworkdb(data []byte) (n int, err error) { if iNdEx >= l { return 0, io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2418,7 +2591,7 @@ func skipNetworkdb(data []byte) (n int, err error) { return 0, io.ErrUnexpectedEOF } iNdEx++ - if data[iNdEx-1] < 0x80 { + if dAtA[iNdEx-1] < 0x80 { break } } @@ -2435,7 +2608,7 @@ func skipNetworkdb(data []byte) (n int, err error) { if iNdEx >= l { return 0, io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { @@ -2458,7 +2631,7 @@ func skipNetworkdb(data []byte) (n int, err error) { if iNdEx >= l { return 0, io.ErrUnexpectedEOF } - b := data[iNdEx] + b := dAtA[iNdEx] iNdEx++ innerWire |= (uint64(b) & 0x7F) << shift if b < 0x80 { @@ -2469,7 +2642,7 @@ func skipNetworkdb(data []byte) (n int, err error) { if innerWireType == 4 { break } - next, err := skipNetworkdb(data[start:]) + next, err := skipNetworkdb(dAtA[start:]) if err != nil { return 0, err } @@ -2493,62 +2666,68 @@ var ( ErrIntOverflowNetworkdb = fmt.Errorf("proto: integer overflow") ) +func init() { proto.RegisterFile("networkdb.proto", fileDescriptorNetworkdb) } + var fileDescriptorNetworkdb = []byte{ - // 887 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x96, 0xc1, 0x6e, 0xe3, 0x44, - 0x18, 0xc7, 0xeb, 0xc4, 0x49, 0xe3, 0xaf, 0x0d, 0x1b, 0xbc, 0xdd, 0xad, 0xd7, 0x0b, 0x49, 0x31, - 0xcb, 0x2a, 0x44, 0xe0, 0xa2, 0xee, 0x13, 0x24, 0xb1, 0x05, 0xd9, 0xf5, 0x3a, 0x91, 0x93, 0x14, - 0x71, 0x8a, 0x9c, 0x78, 0x48, 0xac, 0x3a, 0xb6, 0x15, 0x3b, 0x45, 0x39, 0x81, 0x38, 0xad, 0x78, - 0x07, 0x4e, 0xcb, 0x99, 0x07, 0xe0, 0xc0, 0x89, 0xc3, 0x8a, 0x13, 0xdc, 0x10, 0x87, 0x8a, 0xee, - 0x13, 0xf0, 0x08, 0x8c, 0xc7, 0x76, 0x32, 0x4e, 0xa3, 0x5e, 0x40, 0xc0, 0xc1, 0xad, 0x67, 0xe6, - 0xe7, 0xcf, 0xdf, 0xf7, 0x9f, 0xff, 0xe7, 0x09, 0xdc, 0x71, 0x51, 0xf8, 0x85, 0xb7, 0xb8, 0xb0, - 0xc6, 0xb2, 0xbf, 0xf0, 0x42, 0x8f, 0xe7, 0xd6, 0x13, 0xe2, 0xd1, 0xd4, 0x9b, 0x7a, 0x64, 0xf6, - 0x34, 0xba, 0x8b, 0x01, 0xa9, 0x0b, 0xe5, 0x8f, 0xbd, 0x20, 0xb0, 0xfd, 0xe7, 0x28, 0x08, 0xcc, - 0x29, 0xe2, 0x1b, 0xc0, 0x86, 0x2b, 0x1f, 0x09, 0xcc, 0x09, 0x53, 0x7f, 0xe3, 0xec, 0xbe, 0xbc, - 0x89, 0x98, 0x10, 0x03, 0xbc, 0x6a, 0x10, 0x86, 0xe7, 0x81, 0xb5, 0xcc, 0xd0, 0x14, 0x72, 0x98, - 0x3d, 0x34, 0xc8, 0xbd, 0xf4, 0x32, 0x07, 0x9c, 0xee, 0x59, 0x48, 0xbd, 0x44, 0x6e, 0xc8, 0x7f, - 0x98, 0x89, 0xf6, 0x80, 0x8a, 0xb6, 0x66, 0x64, 0x2a, 0x60, 0x07, 0x8a, 0xce, 0x28, 0xb4, 0xe7, - 0x88, 0x84, 0x64, 0x5b, 0x67, 0xaf, 0xae, 0x6a, 0x7b, 0xbf, 0x5f, 0xd5, 0x1a, 0x53, 0x3b, 0x9c, - 0x2d, 0xc7, 0xf2, 0xc4, 0x9b, 0x9f, 0xce, 0xcc, 0x60, 0x66, 0x4f, 0xbc, 0x85, 0x7f, 0x1a, 0xa0, - 0xc5, 0xe7, 0xe4, 0x8f, 0xac, 0x99, 0x73, 0xdf, 0x5b, 0x84, 0x03, 0xfc, 0xa4, 0x51, 0x70, 0xa2, - 0x7f, 0xfc, 0x43, 0xe0, 0x5c, 0xfc, 0x8a, 0x91, 0x6b, 0xe2, 0x68, 0x79, 0x1c, 0x8d, 0x33, 0x4a, - 0xd1, 0x84, 0x8e, 0xc7, 0xd2, 0x97, 0xc0, 0x46, 0x6f, 0xe5, 0xdf, 0x83, 0xfd, 0x8e, 0x7e, 0xde, - 0xd4, 0x3a, 0x4a, 0x65, 0x4f, 0x14, 0xbe, 0xf9, 0xf6, 0xe4, 0x68, 0x9d, 0x56, 0xb4, 0xde, 0x71, - 0x2f, 0x4d, 0xc7, 0xb6, 0xf8, 0x1a, 0xb0, 0x4f, 0xbb, 0x1d, 0xbd, 0xc2, 0x88, 0xf7, 0x30, 0xf3, - 0x66, 0x86, 0x79, 0xea, 0xd9, 0x2e, 0xff, 0x0e, 0x14, 0x34, 0xb5, 0x79, 0xae, 0x56, 0x72, 0xe2, - 0x7d, 0x4c, 0xf0, 0x19, 0x42, 0x43, 0xe6, 0x25, 0x12, 0x0f, 0x5f, 0xbc, 0xac, 0xee, 0xfd, 0xf0, - 0x5d, 0x95, 0xbc, 0x58, 0xba, 0xce, 0xc1, 0xa1, 0x1e, 0x6b, 0x11, 0x0b, 0xf5, 0x51, 0x46, 0xa8, - 0xb7, 0x68, 0xa1, 0x28, 0xec, 0x3f, 0xd0, 0x8a, 0xff, 0x00, 0x20, 0x49, 0x66, 0x64, 0x5b, 0x02, - 0x1b, 0xad, 0xb6, 0xca, 0xaf, 0xaf, 0x6a, 0x5c, 0x92, 0x58, 0x47, 0x31, 0x52, 0x97, 0x75, 0x2c, - 0xe9, 0x05, 0x93, 0x48, 0x5b, 0xa7, 0xa5, 0x7d, 0x88, 0x45, 0x39, 0xa6, 0x0b, 0xa1, 0xd5, 0x95, - 0xd6, 0xea, 0xc6, 0x3b, 0xb0, 0x85, 0x11, 0x81, 0x1f, 0x6d, 0x04, 0x7e, 0x80, 0xa1, 0x7b, 0xdb, - 0xd0, 0x2e, 0x8d, 0x7f, 0x64, 0x36, 0x1a, 0xbb, 0xe1, 0x62, 0xb5, 0x55, 0x09, 0x73, 0x7b, 0x25, - 0xff, 0x9a, 0xbe, 0x02, 0xec, 0x3b, 0x38, 0x7b, 0xdb, 0x9d, 0x12, 0x71, 0x4b, 0x46, 0x3a, 0x94, - 0xbe, 0x67, 0xe0, 0x4e, 0x92, 0x5a, 0x6f, 0x19, 0xcc, 0x7a, 0x4b, 0xc7, 0xa1, 0xb2, 0x62, 0xfe, - 0x6e, 0x56, 0x4f, 0xa0, 0x94, 0x54, 0x1b, 0xe0, 0x12, 0xf3, 0xf5, 0x83, 0xb3, 0xe3, 0x1d, 0xb6, - 0x8b, 0x94, 0x33, 0xd6, 0xe0, 0xed, 0x6d, 0xf5, 0x73, 0x1e, 0x60, 0x60, 0x8e, 0x9d, 0xa4, 0xf9, - 0xe5, 0x8c, 0xa7, 0x45, 0x2a, 0xf8, 0x06, 0xfa, 0xdf, 0x3b, 0x9a, 0x7f, 0x1b, 0x20, 0x8c, 0xd2, - 0x8d, 0x63, 0x15, 0x48, 0x2c, 0x8e, 0xcc, 0x90, 0x60, 0x15, 0xc8, 0x5f, 0xa0, 0x95, 0x50, 0x24, - 0xf3, 0xd1, 0x2d, 0x7f, 0x04, 0x05, 0x6c, 0xec, 0x25, 0x12, 0xf6, 0xc9, 0x67, 0x31, 0x1e, 0x44, - 0x9b, 0x19, 0x37, 0xc6, 0x63, 0xba, 0x31, 0x88, 0x99, 0x37, 0x6a, 0xd0, 0x6d, 0xf1, 0x08, 0x8a, - 0x6d, 0x43, 0x6d, 0x0e, 0xd4, 0xb4, 0x31, 0xb2, 0x58, 0x7b, 0x81, 0xcc, 0x10, 0x45, 0xd4, 0xb0, - 0xa7, 0x44, 0x54, 0x6e, 0x17, 0x35, 0xf4, 0xad, 0x84, 0x52, 0x54, 0x4d, 0xc5, 0x54, 0x7e, 0x17, - 0xa5, 0x20, 0x07, 0x85, 0xdb, 0xed, 0xf3, 0x2b, 0x76, 0x5f, 0x6b, 0xe9, 0x5c, 0xf4, 0x57, 0xee, - 0x24, 0x3d, 0x1c, 0xfe, 0x41, 0xf7, 0x9d, 0xc0, 0xc1, 0xd2, 0x0d, 0x3c, 0xc7, 0x9e, 0xd8, 0x21, - 0xb2, 0xc8, 0x8e, 0x97, 0x0c, 0x7a, 0xea, 0xf6, 0x3d, 0x14, 0x29, 0xf3, 0xb2, 0xd8, 0xbc, 0x1c, - 0xe5, 0x51, 0xdc, 0x51, 0xbe, 0xb9, 0x72, 0x3c, 0xd3, 0x22, 0xdb, 0x75, 0x68, 0xa4, 0x43, 0xe9, - 0x6b, 0x5c, 0x53, 0xdb, 0xc3, 0xb9, 0x2c, 0x5d, 0x2b, 0xad, 0x49, 0x81, 0xd2, 0x3c, 0xbe, 0x0d, - 0x70, 0x55, 0x51, 0x1b, 0xd4, 0x29, 0xa7, 0x6e, 0xd1, 0x72, 0xdf, 0x9e, 0xfb, 0x0e, 0x4a, 0x46, - 0xc6, 0xfa, 0x49, 0xf1, 0x7d, 0x28, 0x67, 0x96, 0xa2, 0x24, 0x7a, 0x49, 0x12, 0x4c, 0x26, 0x89, - 0xc6, 0x4f, 0x39, 0x38, 0xa0, 0xce, 0x52, 0xfe, 0x5d, 0xda, 0x10, 0xe4, 0xf8, 0xa0, 0x56, 0x53, - 0x37, 0xc8, 0x50, 0xd6, 0xd5, 0xc1, 0xa7, 0x5d, 0xe3, 0xd9, 0x48, 0x3d, 0x57, 0xf5, 0x01, 0x36, - 0x05, 0xf9, 0xa8, 0x52, 0x68, 0xe6, 0x3c, 0x69, 0xc0, 0xc1, 0xa0, 0xd9, 0xd2, 0xd4, 0x84, 0x4e, - 0x3e, 0x9b, 0x14, 0x4d, 0xf5, 0xe9, 0x63, 0xe0, 0x7a, 0xc3, 0xfe, 0x27, 0xa3, 0xde, 0x50, 0xd3, - 0xb0, 0x41, 0x8e, 0x31, 0x79, 0x97, 0x22, 0xd7, 0xdf, 0x1e, 0xcc, 0xb5, 0x86, 0xda, 0xb3, 0x51, - 0xff, 0x33, 0xbd, 0x5d, 0x61, 0x6f, 0x70, 0xa9, 0x59, 0xf0, 0xa9, 0x5a, 0x6a, 0x77, 0x9f, 0xf7, - 0xba, 0x43, 0x5d, 0xa9, 0x14, 0x6e, 0x60, 0xa9, 0xa2, 0xf8, 0x84, 0x00, 0xbd, 0xab, 0xa4, 0x19, - 0x16, 0x63, 0x63, 0xd2, 0xf5, 0xa4, 0x87, 0xa8, 0x78, 0x37, 0x31, 0x26, 0x2d, 0x5b, 0x4b, 0xf8, - 0xed, 0xba, 0xba, 0xf7, 0xe7, 0x75, 0x95, 0xf9, 0xea, 0x75, 0x95, 0x79, 0x85, 0xaf, 0x5f, 0xf0, - 0xf5, 0x07, 0xbe, 0xc6, 0x45, 0xf2, 0xd3, 0xe6, 0xc9, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x21, - 0x78, 0x72, 0xc3, 0x0e, 0x09, 0x00, 0x00, + // 953 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x96, 0xcd, 0x6e, 0xe3, 0x54, + 0x14, 0xc7, 0x7b, 0xf3, 0xd5, 0xe4, 0x34, 0xa5, 0xe6, 0x4e, 0x67, 0xc6, 0xe3, 0x81, 0xc4, 0x98, + 0x99, 0x2a, 0x53, 0x41, 0x8a, 0x3a, 0x4f, 0xd0, 0x24, 0x16, 0x64, 0x26, 0xe3, 0x44, 0x6e, 0x52, + 0xc4, 0x2a, 0xba, 0xad, 0x2f, 0xa9, 0x55, 0xc7, 0xb6, 0x6c, 0x27, 0x28, 0x2b, 0x10, 0xab, 0x51, + 0x16, 0xbc, 0x41, 0x56, 0xc3, 0x9a, 0x07, 0x40, 0x2c, 0x59, 0xcc, 0x82, 0x05, 0xec, 0x10, 0x8b, + 0x88, 0xe6, 0x09, 0x78, 0x04, 0xe4, 0x6b, 0x3b, 0xb9, 0x49, 0xab, 0x91, 0x10, 0x23, 0xc1, 0x26, + 0xb9, 0x1f, 0xbf, 0x1c, 0x9f, 0xf3, 0xf7, 0xff, 0xdc, 0x1b, 0xd8, 0xb3, 0x69, 0xf0, 0x95, 0xe3, + 0x5d, 0x19, 0xe7, 0x55, 0xd7, 0x73, 0x02, 0x07, 0x17, 0x96, 0x0b, 0xd2, 0xfe, 0xc0, 0x19, 0x38, + 0x6c, 0xf5, 0x28, 0x1c, 0x45, 0x80, 0xd2, 0x86, 0xdd, 0x4f, 0x1d, 0xdf, 0x37, 0xdd, 0x17, 0xd4, + 0xf7, 0xc9, 0x80, 0xe2, 0x43, 0xc8, 0x04, 0x13, 0x97, 0x8a, 0x48, 0x46, 0x95, 0x77, 0x8e, 0xef, + 0x55, 0x57, 0x11, 0x63, 0xa2, 0x3b, 0x71, 0xa9, 0xce, 0x18, 0x8c, 0x21, 0x63, 0x90, 0x80, 0x88, + 0x29, 0x19, 0x55, 0x8a, 0x3a, 0x1b, 0x2b, 0xaf, 0x52, 0x50, 0xd0, 0x1c, 0x83, 0xaa, 0x63, 0x6a, + 0x07, 0xf8, 0xe3, 0xb5, 0x68, 0x0f, 0xb8, 0x68, 0x4b, 0xa6, 0xca, 0x05, 0x6c, 0x42, 0xce, 0xea, + 0x07, 0xe6, 0x90, 0xb2, 0x90, 0x99, 0xda, 0xf1, 0xeb, 0x79, 0x79, 0xeb, 0x8f, 0x79, 0xf9, 0x70, + 0x60, 0x06, 0x97, 0xa3, 0xf3, 0xea, 0x85, 0x33, 0x3c, 0xba, 0x24, 0xfe, 0xa5, 0x79, 0xe1, 0x78, + 0xee, 0x91, 0x4f, 0xbd, 0x2f, 0xd9, 0x47, 0xb5, 0x45, 0x86, 0xae, 0xe3, 0x05, 0x5d, 0x73, 0x48, + 0xf5, 0xac, 0x15, 0x7e, 0xe1, 0x87, 0x50, 0xb0, 0x1d, 0x83, 0xf6, 0x6d, 0x32, 0xa4, 0x62, 0x5a, + 0x46, 0x95, 0x82, 0x9e, 0x0f, 0x17, 0x34, 0x32, 0xa4, 0xca, 0xd7, 0x90, 0x09, 0x9f, 0x8a, 0x1f, + 0xc3, 0x76, 0x53, 0x3b, 0x3b, 0x69, 0x35, 0x1b, 0xc2, 0x96, 0x24, 0x4e, 0x67, 0xf2, 0xfe, 0x32, + 0xad, 0x70, 0xbf, 0x69, 0x8f, 0x89, 0x65, 0x1a, 0xb8, 0x0c, 0x99, 0x67, 0xed, 0xa6, 0x26, 0x20, + 0xe9, 0xee, 0x74, 0x26, 0xbf, 0xbb, 0xc6, 0x3c, 0x73, 0x4c, 0x1b, 0x7f, 0x00, 0xd9, 0x96, 0x7a, + 0x72, 0xa6, 0x0a, 0x29, 0xe9, 0xde, 0x74, 0x26, 0xe3, 0x35, 0xa2, 0x45, 0xc9, 0x98, 0x4a, 0xc5, + 0x97, 0xaf, 0x4a, 0x5b, 0x3f, 0x7e, 0x5f, 0x62, 0x0f, 0x56, 0xae, 0x53, 0x50, 0xd4, 0x22, 0x2d, + 0x22, 0xa1, 0x3e, 0x59, 0x13, 0xea, 0x3d, 0x5e, 0x28, 0x0e, 0xfb, 0x0f, 0xb4, 0xc2, 0x1f, 0x01, + 0xc4, 0xc9, 0xf4, 0x4d, 0x43, 0xcc, 0x84, 0xbb, 0xb5, 0xdd, 0xc5, 0xbc, 0x5c, 0x88, 0x13, 0x6b, + 0x36, 0xf4, 0xc4, 0x65, 0x4d, 0x43, 0x79, 0x89, 0x62, 0x69, 0x2b, 0xbc, 0xb4, 0x0f, 0xa7, 0x33, + 0xf9, 0x3e, 0x5f, 0x08, 0xaf, 0xae, 0xb2, 0x54, 0x37, 0x7a, 0x03, 0x1b, 0x18, 0x13, 0xf8, 0xd1, + 0x4a, 0xe0, 0x07, 0xd3, 0x99, 0x7c, 0x77, 0x13, 0xba, 0x4d, 0xe3, 0x5f, 0xd0, 0x4a, 0x63, 0x3b, + 0xf0, 0x26, 0x1b, 0x95, 0xa0, 0x37, 0x57, 0xf2, 0x36, 0xf5, 0x7d, 0x72, 0x43, 0xdf, 0x5a, 0x71, + 0x31, 0x2f, 0xe7, 0xb5, 0x58, 0x63, 0x4e, 0x6d, 0x11, 0xb6, 0x2d, 0x4a, 0xc6, 0xa6, 0x3d, 0x60, + 0x52, 0xe7, 0xf5, 0x64, 0xaa, 0xfc, 0x84, 0x60, 0x2f, 0x4e, 0xb4, 0x33, 0xf2, 0x2f, 0x3b, 0x23, + 0xcb, 0xe2, 0x72, 0x44, 0xff, 0x36, 0xc7, 0xa7, 0x90, 0x8f, 0x6b, 0xf7, 0xc5, 0x94, 0x9c, 0xae, + 0xec, 0x1c, 0xdf, 0xbf, 0xc5, 0x84, 0xa1, 0x8e, 0xfa, 0x12, 0xfc, 0x07, 0x85, 0x29, 0xdf, 0x65, + 0x00, 0xba, 0xe4, 0xdc, 0x8a, 0x0f, 0x86, 0xea, 0x9a, 0xdf, 0x25, 0xee, 0x51, 0x2b, 0xe8, 0x7f, + 0xef, 0x76, 0xfc, 0x3e, 0x40, 0x10, 0xa6, 0x1b, 0xc5, 0xca, 0xb2, 0x58, 0x05, 0xb6, 0xc2, 0x82, + 0x09, 0x90, 0xbe, 0xa2, 0x13, 0x31, 0xc7, 0xd6, 0xc3, 0x21, 0xde, 0x87, 0xec, 0x98, 0x58, 0x23, + 0x2a, 0x6e, 0xb3, 0x23, 0x33, 0x9a, 0xe0, 0x1a, 0x60, 0x8f, 0xfa, 0xa6, 0x31, 0x22, 0x56, 0xdf, + 0xa3, 0xc4, 0x8d, 0x0a, 0xcd, 0xcb, 0xa8, 0x92, 0xad, 0xed, 0x2f, 0xe6, 0x65, 0x41, 0x8f, 0x77, + 0x75, 0x4a, 0x5c, 0x56, 0x8a, 0xe0, 0x6d, 0xac, 0x28, 0x3f, 0x24, 0x8d, 0x77, 0xc0, 0x37, 0x1e, + 0x6b, 0x96, 0x95, 0xa2, 0x7c, 0xdb, 0x3d, 0x82, 0x5c, 0x5d, 0x57, 0x4f, 0xba, 0x6a, 0xd2, 0x78, + 0xeb, 0x58, 0xdd, 0xa3, 0x24, 0xa0, 0x21, 0xd5, 0xeb, 0x34, 0x42, 0x2a, 0x75, 0x1b, 0xd5, 0x73, + 0x8d, 0x98, 0x6a, 0xa8, 0x2d, 0xb5, 0xab, 0x0a, 0xe9, 0xdb, 0xa8, 0x06, 0xb5, 0x68, 0xb0, 0xd9, + 0x9e, 0xbf, 0x21, 0xd8, 0xab, 0x8d, 0xac, 0xab, 0xd3, 0x89, 0x7d, 0x91, 0x5c, 0x3e, 0x6f, 0xd1, + 0xcf, 0x32, 0xec, 0x8c, 0x6c, 0xdf, 0xb1, 0xcc, 0x0b, 0x33, 0xa0, 0x06, 0x73, 0x4d, 0x5e, 0xe7, + 0x97, 0xde, 0xec, 0x03, 0x89, 0x6b, 0x87, 0x8c, 0x9c, 0x66, 0x7b, 0x89, 0xeb, 0x45, 0xd8, 0x76, + 0xc9, 0xc4, 0x72, 0x88, 0xc1, 0x5e, 0x79, 0x51, 0x4f, 0xa6, 0xca, 0xb7, 0x08, 0xf6, 0xea, 0xce, + 0xd0, 0x75, 0x46, 0xb6, 0x91, 0xd4, 0xd4, 0x80, 0xfc, 0x30, 0x1a, 0xfa, 0x22, 0x62, 0x8d, 0x55, + 0xe1, 0xdc, 0xbe, 0x41, 0x57, 0x4f, 0xcd, 0xa1, 0x6b, 0xd1, 0x78, 0xa6, 0x2f, 0x7f, 0x29, 0x3d, + 0x81, 0xdd, 0xb5, 0xad, 0x30, 0x89, 0x4e, 0x9c, 0x04, 0x8a, 0x92, 0x88, 0xa7, 0x87, 0x3f, 0xa7, + 0x60, 0x87, 0xbb, 0xab, 0xf1, 0x87, 0xbc, 0x21, 0xd8, 0xf5, 0xc4, 0xed, 0x26, 0x6e, 0xa8, 0xc2, + 0xae, 0xa6, 0x76, 0x3f, 0x6f, 0xeb, 0xcf, 0xfb, 0xea, 0x99, 0xaa, 0x75, 0x05, 0x14, 0x1d, 0xda, + 0x1c, 0xba, 0x76, 0x5f, 0x1d, 0xc2, 0x4e, 0xf7, 0xa4, 0xd6, 0x52, 0x63, 0x3a, 0x3e, 0x96, 0x39, + 0x9a, 0xeb, 0xf5, 0x03, 0x28, 0x74, 0x7a, 0xa7, 0x9f, 0xf5, 0x3b, 0xbd, 0x56, 0x4b, 0x48, 0x4b, + 0xf7, 0xa7, 0x33, 0xf9, 0x0e, 0x47, 0x2e, 0x4f, 0xb3, 0x03, 0x28, 0xd4, 0x7a, 0xad, 0xe7, 0xfd, + 0xd3, 0x2f, 0xb4, 0xba, 0x90, 0xb9, 0xc1, 0x25, 0x66, 0xc1, 0x8f, 0x21, 0x5f, 0x6f, 0xbf, 0xe8, + 0xb4, 0x7b, 0x5a, 0x43, 0xc8, 0xde, 0xc0, 0x12, 0x45, 0x71, 0x05, 0x40, 0x6b, 0x37, 0x92, 0x0c, + 0x73, 0x91, 0x31, 0xf9, 0x7a, 0x92, 0x4b, 0x5a, 0xba, 0x13, 0x1b, 0x93, 0x97, 0xad, 0x26, 0xfe, + 0x7e, 0x5d, 0xda, 0xfa, 0xeb, 0xba, 0x84, 0xbe, 0x59, 0x94, 0xd0, 0xeb, 0x45, 0x09, 0xfd, 0xba, + 0x28, 0xa1, 0x3f, 0x17, 0x25, 0x74, 0x9e, 0x63, 0x7f, 0x9d, 0x9e, 0xfe, 0x1d, 0x00, 0x00, 0xff, + 0xff, 0x92, 0x82, 0xdb, 0x1a, 0x6e, 0x09, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.proto b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.proto index 7df1b42dca..0b8490be7a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.proto +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.proto @@ -109,7 +109,7 @@ message NetworkEntry { // network event was recorded. uint64 l_time = 2 [(gogoproto.customtype) = "github.com/hashicorp/serf/serf.LamportTime", (gogoproto.nullable) = false]; // Source node name where this network attachment happened. - string node_name = 3; + string node_name = 3 [(gogoproto.customname) = "NodeName"]; // Indicates if a leave from this network is in progress. bool leaving = 4; } @@ -119,6 +119,8 @@ message NetworkPushPull { // Lamport time when this push pull was initiated. uint64 l_time = 1 [(gogoproto.customtype) = "github.com/hashicorp/serf/serf.LamportTime", (gogoproto.nullable) = false]; repeated NetworkEntry networks = 2; + // Name of the node sending this push pull payload. + string node_name = 3 [(gogoproto.customname) = "NodeName"]; } // TableEvent message payload definition. @@ -152,6 +154,8 @@ message TableEvent { string key = 6; // Entry value. bytes value = 7; + // Residual reap time for the entry before getting deleted in seconds + int32 residual_reap_time = 8 [(gogoproto.customname) = "ResidualReapTime"];; } // BulkSync message payload definition. @@ -180,4 +184,4 @@ message CompoundMessage { // A list of simple messages. repeated SimpleMessage messages = 1; -} \ No newline at end of file +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go b/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go index d196330558..afa3e793fe 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go +++ b/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go @@ -67,11 +67,7 @@ func (sb *sandbox) setupResolutionFiles() error { return err } - if err := sb.setupDNS(); err != nil { - return err - } - - return nil + return sb.setupDNS() } func (sb *sandbox) buildHostsFile() error { From 08619d0c7ccc0ad7b9c30aa7b72a8ee37c0caf5e Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 25 Sep 2017 10:42:20 +0200 Subject: [PATCH 20/34] Fix TestMount under a selinux system Signed-off-by: Vincent Demeester Upstream-commit: 8bebd42df2d8eaa0ecdc9c78bc1e395a752eb5c9 Component: engine --- components/engine/pkg/mount/mounter_linux_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/engine/pkg/mount/mounter_linux_test.go b/components/engine/pkg/mount/mounter_linux_test.go index 47c03b3631..f02bd13786 100644 --- a/components/engine/pkg/mount/mounter_linux_test.go +++ b/components/engine/pkg/mount/mounter_linux_test.go @@ -8,6 +8,8 @@ import ( "os" "strings" "testing" + + selinux "github.com/opencontainers/selinux/go-selinux" ) func TestMount(t *testing.T) { @@ -101,7 +103,11 @@ func TestMount(t *testing.T) { t.Fatal(err) } defer ensureUnmount(t, target) - validateMount(t, target, tc.expectedOpts, tc.expectedOptional, tc.expectedVFS) + expectedVFS := tc.expectedVFS + if selinux.GetEnabled() && expectedVFS != "" { + expectedVFS = expectedVFS + ",seclabel" + } + validateMount(t, target, tc.expectedOpts, tc.expectedOptional, expectedVFS) }) } } From e38ecccceb5967db3750aa893b8c56bd8719e9c3 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Mon, 25 Sep 2017 13:58:51 +0200 Subject: [PATCH 21/34] Match not implemented error check to others Signed-off-by: Christopher Crone Upstream-commit: 7406088853b6cbcb8996c367062cee2e1ee6eaaa Component: engine --- components/engine/client/errors.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/client/errors.go b/components/engine/client/errors.go index 3f52dfe5f6..ba29bce679 100644 --- a/components/engine/client/errors.go +++ b/components/engine/client/errors.go @@ -171,10 +171,10 @@ func (e notImplementedError) NotImplemented() bool { return true } -// IsNotImplementedError returns true if the error is a NotImplemented error. +// IsErrNotImplemented returns true if the error is a NotImplemented error. // This is returned by the API when a requested feature has not been // implemented. -func IsNotImplementedError(err error) bool { +func IsErrNotImplemented(err error) bool { te, ok := err.(notImplementedError) return ok && te.NotImplemented() } From 0b2721b9c10315a072e4f28d56315a70c6db7248 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Mon, 25 Sep 2017 14:05:18 +0200 Subject: [PATCH 22/34] Protect environment for system integration tests Signed-off-by: Christopher Crone Upstream-commit: d43dac2202667a407f4c5ab061c04b0ea334aa20 Component: engine --- components/engine/integration/system/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/integration/system/main_test.go b/components/engine/integration/system/main_test.go index ad0d203753..575bbc166d 100644 --- a/components/engine/integration/system/main_test.go +++ b/components/engine/integration/system/main_test.go @@ -23,6 +23,6 @@ func TestMain(m *testing.M) { } func setupTest(t *testing.T) func() { - environment.ProtectImages(t, testEnv) + environment.ProtectAll(t, testEnv) return func() { testEnv.Clean(t) } } From 522e6cc8eceb0a3640b7af8cd33fcd1e959f2877 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Mon, 25 Sep 2017 14:08:03 +0200 Subject: [PATCH 23/34] Do not use deprecated call for APIClient Signed-off-by: Christopher Crone Upstream-commit: 82440a039f58b6a1487f8042486fe8dba675df54 Component: engine --- components/engine/integration/system/version_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/engine/integration/system/version_test.go b/components/engine/integration/system/version_test.go index 110bd9f56f..c1231fd383 100644 --- a/components/engine/integration/system/version_test.go +++ b/components/engine/integration/system/version_test.go @@ -3,15 +3,14 @@ package system import ( "testing" - "github.com/docker/docker/integration-cli/request" + "github.com/docker/docker/integration/util/request" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" ) func TestVersion(t *testing.T) { - client, err := request.NewClient() - require.NoError(t, err) + client := request.NewAPIClient(t) version, err := client.ServerVersion(context.Background()) require.NoError(t, err) From 0c54b3a04187ff8cc8d413e50787171b3b96e5b2 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 21 Sep 2017 16:03:59 +0200 Subject: [PATCH 24/34] Improve error message for COPY missing destination Signed-off-by: Sebastiaan van Stijn Upstream-commit: 5d05a8291314b8f727b04b504b8d7fc7ed7f42da Component: engine --- .../engine/builder/dockerfile/instructions/parse.go | 8 ++++---- .../engine/builder/dockerfile/instructions/parse_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/engine/builder/dockerfile/instructions/parse.go b/components/engine/builder/dockerfile/instructions/parse.go index e52ac47a11..86ddc57b7e 100644 --- a/components/engine/builder/dockerfile/instructions/parse.go +++ b/components/engine/builder/dockerfile/instructions/parse.go @@ -235,7 +235,7 @@ func parseLabel(req parseRequest) (*LabelCommand, error) { func parseAdd(req parseRequest) (*AddCommand, error) { if len(req.args) < 2 { - return nil, errAtLeastTwoArguments("ADD") + return nil, errNoDestinationArgument("ADD") } flChown := req.flags.AddString("chown", "") if err := req.flags.Parse(); err != nil { @@ -250,7 +250,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) { func parseCopy(req parseRequest) (*CopyCommand, error) { if len(req.args) < 2 { - return nil, errAtLeastTwoArguments("COPY") + return nil, errNoDestinationArgument("COPY") } flChown := req.flags.AddString("chown", "") flFrom := req.flags.AddString("from", "") @@ -622,8 +622,8 @@ func errExactlyOneArgument(command string) error { return errors.Errorf("%s requires exactly one argument", command) } -func errAtLeastTwoArguments(command string) error { - return errors.Errorf("%s requires at least two arguments", command) +func errNoDestinationArgument(command string) error { + return errors.Errorf("%s requires at least two arguments, but only one was provided. Destination could not be determined.", command) } func errBlankCommandNames(command string) error { diff --git a/components/engine/builder/dockerfile/instructions/parse_test.go b/components/engine/builder/dockerfile/instructions/parse_test.go index bf41b1a266..f15eaca1d6 100644 --- a/components/engine/builder/dockerfile/instructions/parse_test.go +++ b/components/engine/builder/dockerfile/instructions/parse_test.go @@ -45,7 +45,7 @@ func TestCommandsAtLeastOneArgument(t *testing.T) { } } -func TestCommandsAtLeastTwoArgument(t *testing.T) { +func TestCommandsNoDestinationArgument(t *testing.T) { commands := []string{ "ADD", "COPY", @@ -55,7 +55,7 @@ func TestCommandsAtLeastTwoArgument(t *testing.T) { ast, err := parser.Parse(strings.NewReader(command + " arg1")) require.NoError(t, err) _, err = ParseInstruction(ast.AST.Children[0]) - assert.EqualError(t, err, errAtLeastTwoArguments(command).Error()) + assert.EqualError(t, err, errNoDestinationArgument(command).Error()) } } From 0975184f91e465cb57d6a75ad46ebc2782bc5068 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Mon, 25 Sep 2017 14:09:17 +0200 Subject: [PATCH 25/34] Docker EE integration test fixes Signed-off-by: Christopher Crone Upstream-commit: 8c5f98c93e06de81b7fb6416372a3f42aa7aeb5d Component: engine --- .../engine/internal/test/environment/clean.go | 16 +++++++++++++++- .../engine/internal/test/environment/protect.go | 5 +++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/components/engine/internal/test/environment/clean.go b/components/engine/internal/test/environment/clean.go index 1bdd21080a..e9548d1cde 100644 --- a/components/engine/internal/test/environment/clean.go +++ b/components/engine/internal/test/environment/clean.go @@ -81,7 +81,7 @@ func deleteAllContainers(t assert.TestingT, apiclient client.ContainerAPIClient, Force: true, RemoveVolumes: true, }) - if err == nil || client.IsErrNotFound(err) || alreadyExists.MatchString(err.Error()) { + if err == nil || client.IsErrNotFound(err) || alreadyExists.MatchString(err.Error()) || isErrNotFoundSwarmClassic(err) { continue } assert.NoError(t, err, "failed to remove %s", container.ID) @@ -138,6 +138,10 @@ func deleteAllVolumes(t assert.TestingT, c client.VolumeAPIClient, protectedVolu continue } err := c.VolumeRemove(context.Background(), v.Name, true) + // Docker EE may list volumes that no longer exist. + if isErrNotFoundSwarmClassic(err) { + continue + } assert.NoError(t, err, "failed to remove volume %s", v.Name) } } @@ -164,6 +168,10 @@ func deleteAllNetworks(t assert.TestingT, c client.NetworkAPIClient, daemonPlatf func deleteAllPlugins(t assert.TestingT, c client.PluginAPIClient, protectedPlugins map[string]struct{}) { plugins, err := c.PluginList(context.Background(), filters.Args{}) + // Docker EE does not allow cluster-wide plugin management. + if client.IsErrNotImplemented(err) { + return + } assert.NoError(t, err, "failed to list plugins") for _, p := range plugins { @@ -174,3 +182,9 @@ func deleteAllPlugins(t assert.TestingT, c client.PluginAPIClient, protectedPlug assert.NoError(t, err, "failed to remove plugin %s", p.ID) } } + +// Swarm classic aggregates node errors and returns a 500 so we need to check +// the error string instead of just IsErrNotFound(). +func isErrNotFoundSwarmClassic(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), "no such") +} diff --git a/components/engine/internal/test/environment/protect.go b/components/engine/internal/test/environment/protect.go index 3c74fcf1bb..296ae73789 100644 --- a/components/engine/internal/test/environment/protect.go +++ b/components/engine/internal/test/environment/protect.go @@ -5,6 +5,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + dclient "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/fixtures/load" "github.com/stretchr/testify/require" ) @@ -172,6 +173,10 @@ func ProtectPlugins(t testingT, testEnv *Execution) { func getExistingPlugins(t require.TestingT, testEnv *Execution) []string { client := testEnv.APIClient() pluginList, err := client.PluginList(context.Background(), filters.Args{}) + // Docker EE does not allow cluster-wide plugin management. + if dclient.IsErrNotImplemented(err) { + return []string{} + } require.NoError(t, err, "failed to list plugins") plugins := []string{} From 658351133fff4ee890a165b716579507ffdfaa81 Mon Sep 17 00:00:00 2001 From: Darren Stahl Date: Mon, 25 Sep 2017 12:39:27 -0700 Subject: [PATCH 26/34] Fix error string about containers feature Signed-off-by: Darren Stahl Upstream-commit: 31405b556f155d8f56902086c7c24efe25dd8de0 Component: engine --- components/engine/daemon/daemon_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index c85a1483f2..bbf5383c14 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -235,7 +235,7 @@ func checkSystem() error { vmcompute := windows.NewLazySystemDLL("vmcompute.dll") if vmcompute.Load() != nil { - return fmt.Errorf("Failed to load vmcompute.dll. Ensure that the Containers role is installed.") + return fmt.Errorf("failed to load vmcompute.dll, ensure that the Containers feature is installed") } return nil From 101f740d40dfda01f3eadf1d16e6ec99965e997a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 22 Sep 2017 14:40:10 -0400 Subject: [PATCH 27/34] Move RFC3339NanoFixed to a more appropriate package. Signed-off-by: Daniel Nephin Upstream-commit: 27cfa68af16721c978803c3b695bcc7181ccc721 Component: engine --- .../api/server/httputils/write_log_stream.go | 8 ++------ components/engine/cmd/dockerd/daemon.go | 4 ++-- components/engine/daemon/logger/logger.go | 3 --- .../integration-cli/docker_cli_logs_test.go | 4 ++-- components/engine/pkg/jsonlog/time_marshalling.go | 15 ++++----------- components/engine/pkg/jsonmessage/jsonmessage.go | 9 ++++++--- .../engine/pkg/jsonmessage/jsonmessage_test.go | 13 ++++++------- 7 files changed, 22 insertions(+), 34 deletions(-) diff --git a/components/engine/api/server/httputils/write_log_stream.go b/components/engine/api/server/httputils/write_log_stream.go index e90e610da6..d7a49a1584 100644 --- a/components/engine/api/server/httputils/write_log_stream.go +++ b/components/engine/api/server/httputils/write_log_stream.go @@ -11,7 +11,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/jsonlog" + "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" ) @@ -49,11 +49,7 @@ func WriteLogStream(_ context.Context, w io.Writer, msgs <-chan *backend.LogMess logLine = append(logLine, msg.Line...) } if config.Timestamps { - // TODO(dperny) the format is defined in - // daemon/logger/logger.go as logger.TimeFormat. importing - // logger is verboten (not part of backend) so idk if just - // importing the same thing from jsonlog is good enough - logLine = append([]byte(msg.Timestamp.Format(jsonlog.RFC3339NanoFixed)+" "), logLine...) + logLine = append([]byte(msg.Timestamp.Format(jsonmessage.RFC3339NanoFixed)+" "), logLine...) } if msg.Source == "stdout" && config.ShowStdout { outStream.Write(logLine) diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index 2e8acf97da..c76886fd10 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -38,7 +38,7 @@ import ( "github.com/docker/docker/libcontainerd" dopts "github.com/docker/docker/opts" "github.com/docker/docker/pkg/authorization" - "github.com/docker/docker/pkg/jsonlog" + "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/pidfile" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/docker/pkg/signal" @@ -94,7 +94,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { } logrus.SetFormatter(&logrus.TextFormatter{ - TimestampFormat: jsonlog.RFC3339NanoFixed, + TimestampFormat: jsonmessage.RFC3339NanoFixed, DisableColors: cli.Config.RawLogs, FullTimestamp: true, }) diff --git a/components/engine/daemon/logger/logger.go b/components/engine/daemon/logger/logger.go index a9d1e7640b..ee91b79c98 100644 --- a/components/engine/daemon/logger/logger.go +++ b/components/engine/daemon/logger/logger.go @@ -12,7 +12,6 @@ import ( "time" "github.com/docker/docker/api/types/backend" - "github.com/docker/docker/pkg/jsonlog" ) // ErrReadLogsNotSupported is returned when the underlying log driver does not support reading @@ -26,8 +25,6 @@ func (ErrReadLogsNotSupported) Error() string { func (ErrReadLogsNotSupported) NotImplemented() {} const ( - // TimeFormat is the time format used for timestamps sent to log readers. - TimeFormat = jsonlog.RFC3339NanoFixed logWatcherBufferSize = 4096 ) diff --git a/components/engine/integration-cli/docker_cli_logs_test.go b/components/engine/integration-cli/docker_cli_logs_test.go index d9523bffcc..f75da1849c 100644 --- a/components/engine/integration-cli/docker_cli_logs_test.go +++ b/components/engine/integration-cli/docker_cli_logs_test.go @@ -10,7 +10,7 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/cli" - "github.com/docker/docker/pkg/jsonlog" + "github.com/docker/docker/pkg/jsonmessage" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/icmd" ) @@ -55,7 +55,7 @@ func (s *DockerSuite) TestLogsTimestamps(c *check.C) { for _, l := range lines { if l != "" { - _, err := time.Parse(jsonlog.RFC3339NanoFixed+" ", ts.FindString(l)) + _, err := time.Parse(jsonmessage.RFC3339NanoFixed+" ", ts.FindString(l)) c.Assert(err, checker.IsNil, check.Commentf("Failed to parse timestamp from %v", l)) // ensure we have padded 0's c.Assert(l[29], checker.Equals, uint8('Z')) diff --git a/components/engine/pkg/jsonlog/time_marshalling.go b/components/engine/pkg/jsonlog/time_marshalling.go index 2117338149..8c3a8d8282 100644 --- a/components/engine/pkg/jsonlog/time_marshalling.go +++ b/components/engine/pkg/jsonlog/time_marshalling.go @@ -1,19 +1,12 @@ -// Package jsonlog provides helper functions to parse and print time (time.Time) as JSON. package jsonlog import ( - "errors" "time" + + "github.com/pkg/errors" ) -const ( - // RFC3339NanoFixed is our own version of RFC339Nano because we want one - // that pads the nano seconds part with zeros to ensure - // the timestamps are aligned in the logs. - RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" - // JSONFormat is the format used by FastMarshalJSON - JSONFormat = `"` + time.RFC3339Nano + `"` -) +const jsonFormat = `"` + time.RFC3339Nano + `"` // FastTimeMarshalJSON avoids one of the extra allocations that // time.MarshalJSON is making. @@ -23,5 +16,5 @@ func FastTimeMarshalJSON(t time.Time) (string, error) { // See golang.org/issue/4556#c15 for more discussion. return "", errors.New("time.MarshalJSON: year outside of range [0,9999]") } - return t.Format(JSONFormat), nil + return t.Format(jsonFormat), nil } diff --git a/components/engine/pkg/jsonmessage/jsonmessage.go b/components/engine/pkg/jsonmessage/jsonmessage.go index 09fc4cc745..6cfa464830 100644 --- a/components/engine/pkg/jsonmessage/jsonmessage.go +++ b/components/engine/pkg/jsonmessage/jsonmessage.go @@ -9,11 +9,14 @@ import ( "time" gotty "github.com/Nvveen/Gotty" - "github.com/docker/docker/pkg/jsonlog" "github.com/docker/docker/pkg/term" units "github.com/docker/go-units" ) +// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to +// ensure the formatted time isalways the same number of characters. +const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + // JSONError wraps a concrete Code and Message, `Code` is // is an integer error code, `Message` is the error message. type JSONError struct { @@ -199,9 +202,9 @@ func (jm *JSONMessage) Display(out io.Writer, termInfo termInfo) error { return nil } if jm.TimeNano != 0 { - fmt.Fprintf(out, "%s ", time.Unix(0, jm.TimeNano).Format(jsonlog.RFC3339NanoFixed)) + fmt.Fprintf(out, "%s ", time.Unix(0, jm.TimeNano).Format(RFC3339NanoFixed)) } else if jm.Time != 0 { - fmt.Fprintf(out, "%s ", time.Unix(jm.Time, 0).Format(jsonlog.RFC3339NanoFixed)) + fmt.Fprintf(out, "%s ", time.Unix(jm.Time, 0).Format(RFC3339NanoFixed)) } if jm.ID != "" { fmt.Fprintf(out, "%s: ", jm.ID) diff --git a/components/engine/pkg/jsonmessage/jsonmessage_test.go b/components/engine/pkg/jsonmessage/jsonmessage_test.go index 5206789ac2..9740bcd98d 100644 --- a/components/engine/pkg/jsonmessage/jsonmessage_test.go +++ b/components/engine/pkg/jsonmessage/jsonmessage_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/docker/docker/pkg/jsonlog" "github.com/docker/docker/pkg/term" "github.com/stretchr/testify/assert" ) @@ -115,8 +114,8 @@ func TestJSONMessageDisplay(t *testing.T) { From: "From", Status: "status", }: { - fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(jsonlog.RFC3339NanoFixed)), - fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(jsonlog.RFC3339NanoFixed)), + fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(RFC3339NanoFixed)), + fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(RFC3339NanoFixed)), }, // General, with nano precision time { @@ -125,8 +124,8 @@ func TestJSONMessageDisplay(t *testing.T) { From: "From", Status: "status", }: { - fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)), - fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)), + fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)), + fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)), }, // General, with both times Nano is preferred { @@ -136,8 +135,8 @@ func TestJSONMessageDisplay(t *testing.T) { From: "From", Status: "status", }: { - fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)), - fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)), + fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)), + fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(RFC3339NanoFixed)), }, // Stream over status { From 0c6f1703459091735d25ca5281742819832f4556 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 22 Sep 2017 15:37:16 -0400 Subject: [PATCH 28/34] Unexport FastTimeMarshalJSON Signed-off-by: Daniel Nephin Upstream-commit: 7de92de636ef307d66b7b20b24f166a47f40f72b Component: engine --- .../daemon/logger/jsonfilelog/jsonfilelog.go | 8 +--- .../engine/pkg/jsonlog/jsonlog_marshalling.go | 2 +- .../pkg/jsonlog/jsonlog_marshalling_test.go | 16 +++---- components/engine/pkg/jsonlog/jsonlogbytes.go | 19 +++++--- .../engine/pkg/jsonlog/jsonlogbytes_test.go | 45 ++++++++--------- .../engine/pkg/jsonlog/time_marshalling.go | 4 +- .../pkg/jsonlog/time_marshalling_test.go | 48 +++++++------------ 7 files changed, 65 insertions(+), 77 deletions(-) diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index c5f12d0021..351d6eeed5 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -113,18 +113,14 @@ func writeMessageBuf(w io.Writer, m *logger.Message, extra json.RawMessage, buf } func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { - timestamp, err := jsonlog.FastTimeMarshalJSON(msg.Timestamp) - if err != nil { - return err - } logLine := msg.Line if !msg.Partial { logLine = append(msg.Line, '\n') } - err = (&jsonlog.JSONLogs{ + err := (&jsonlog.JSONLogs{ Log: logLine, Stream: msg.Source, - Created: timestamp, + Created: msg.Timestamp, RawAttrs: extra, }).MarshalJSONBuf(buf) if err != nil { diff --git a/components/engine/pkg/jsonlog/jsonlog_marshalling.go b/components/engine/pkg/jsonlog/jsonlog_marshalling.go index 83ce684a8e..8fae044b96 100644 --- a/components/engine/pkg/jsonlog/jsonlog_marshalling.go +++ b/components/engine/pkg/jsonlog/jsonlog_marshalling.go @@ -105,7 +105,7 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { buf.WriteString(`,`) } buf.WriteString(`"time":`) - timestamp, err = FastTimeMarshalJSON(mj.Created) + timestamp, err = fastTimeMarshalJSON(mj.Created) if err != nil { return err } diff --git a/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go b/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go index 8b0d072cd3..a6178a8c5b 100644 --- a/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go +++ b/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go @@ -3,6 +3,10 @@ package jsonlog import ( "regexp" "testing" + + "encoding/json" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestJSONLogMarshalJSON(t *testing.T) { @@ -21,14 +25,8 @@ func TestJSONLogMarshalJSON(t *testing.T) { } for jsonLog, expression := range logs { data, err := jsonLog.MarshalJSON() - if err != nil { - t.Fatal(err) - } - res := string(data) - t.Logf("Result of WriteLog: %q", res) - logRe := regexp.MustCompile(expression) - if !logRe.MatchString(res) { - t.Fatalf("Log line not in expected format [%v]: %q", expression, res) - } + require.NoError(t, err) + assert.Regexp(t, regexp.MustCompile(expression), string(data)) + assert.NoError(t, json.Unmarshal(data, &map[string]interface{}{})) } } diff --git a/components/engine/pkg/jsonlog/jsonlogbytes.go b/components/engine/pkg/jsonlog/jsonlogbytes.go index 0ba716f261..b6663b9289 100644 --- a/components/engine/pkg/jsonlog/jsonlogbytes.go +++ b/components/engine/pkg/jsonlog/jsonlogbytes.go @@ -3,16 +3,15 @@ package jsonlog import ( "bytes" "encoding/json" + "time" "unicode/utf8" ) -// JSONLogs is based on JSONLog. -// It allows marshalling JSONLog from Log as []byte -// and an already marshalled Created timestamp. +// JSONLogs marshals encoded JSONLog objects type JSONLogs struct { - Log []byte `json:"log,omitempty"` - Stream string `json:"stream,omitempty"` - Created string `json:"time"` + Log []byte `json:"log,omitempty"` + Stream string `json:"stream,omitempty"` + Created time.Time `json:"time"` // json-encoded bytes RawAttrs json.RawMessage `json:"attrs,omitempty"` @@ -50,8 +49,14 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { if !first { buf.WriteString(`,`) } + + created, err := fastTimeMarshalJSON(mj.Created) + if err != nil { + return err + } + buf.WriteString(`"time":`) - buf.WriteString(mj.Created) + buf.WriteString(created) buf.WriteString(`}`) return nil } diff --git a/components/engine/pkg/jsonlog/jsonlogbytes_test.go b/components/engine/pkg/jsonlog/jsonlogbytes_test.go index 41049aaea8..4645cd4faa 100644 --- a/components/engine/pkg/jsonlog/jsonlogbytes_test.go +++ b/components/engine/pkg/jsonlog/jsonlogbytes_test.go @@ -2,38 +2,39 @@ package jsonlog import ( "bytes" + "encoding/json" "regexp" "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestJSONLogsMarshalJSONBuf(t *testing.T) { logs := map[*JSONLogs]string{ - {Log: []byte(`"A log line with \\"`)}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":}$`, - {Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"time\":}$`, - {Log: []byte("A log line with \r")}: `^{\"log\":\"A log line with \\r\",\"time\":}$`, - {Log: []byte("A log line with & < >")}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":}$`, - {Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":}$`, - {Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":}$`, - {Stream: "stdout", Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":}$`, - {Created: "time"}: `^{\"time\":time}$`, - {}: `^{\"time\":}$`, + {Log: []byte(`"A log line with \\"`)}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":`, + {Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"time\":`, + {Log: []byte("A log line with \r")}: `^{\"log\":\"A log line with \\r\",\"time\":`, + {Log: []byte("A log line with & < >")}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":`, + {Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":`, + {Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":`, + {Stream: "stdout", Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":`, + {Created: time.Date(2017, 9, 1, 1, 1, 1, 1, time.UTC)}: `^{\"time\":"2017-09-01T01:01:01.000000001Z"}$`, + + {}: `^{\"time\":"0001-01-01T00:00:00Z"}$`, // These ones are a little weird - {Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`, - {Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":}$`, - {Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":}$`, + {Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":`, + {Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":`, + {Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":`, // with raw attributes - {Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`, + {Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`, } for jsonLog, expression := range logs { var buf bytes.Buffer - if err := jsonLog.MarshalJSONBuf(&buf); err != nil { - t.Fatal(err) - } - res := buf.String() - t.Logf("Result of WriteLog: %q", res) - logRe := regexp.MustCompile(expression) - if !logRe.MatchString(res) { - t.Fatalf("Log line not in expected format [%v]: %q", expression, res) - } + err := jsonLog.MarshalJSONBuf(&buf) + require.NoError(t, err) + assert.Regexp(t, regexp.MustCompile(expression), buf.String()) + assert.NoError(t, json.Unmarshal(buf.Bytes(), &map[string]interface{}{})) } } diff --git a/components/engine/pkg/jsonlog/time_marshalling.go b/components/engine/pkg/jsonlog/time_marshalling.go index 8c3a8d8282..5fd8023e7c 100644 --- a/components/engine/pkg/jsonlog/time_marshalling.go +++ b/components/engine/pkg/jsonlog/time_marshalling.go @@ -8,9 +8,9 @@ import ( const jsonFormat = `"` + time.RFC3339Nano + `"` -// FastTimeMarshalJSON avoids one of the extra allocations that +// fastTimeMarshalJSON avoids one of the extra allocations that // time.MarshalJSON is making. -func FastTimeMarshalJSON(t time.Time) (string, error) { +func fastTimeMarshalJSON(t time.Time) (string, error) { if y := t.Year(); y < 0 || y >= 10000 { // RFC 3339 is clear that years are 4 digits exactly. // See golang.org/issue/4556#c15 for more discussion. diff --git a/components/engine/pkg/jsonlog/time_marshalling_test.go b/components/engine/pkg/jsonlog/time_marshalling_test.go index 02d0302c4a..931a9d3d66 100644 --- a/components/engine/pkg/jsonlog/time_marshalling_test.go +++ b/components/engine/pkg/jsonlog/time_marshalling_test.go @@ -3,45 +3,33 @@ package jsonlog import ( "testing" "time" + + "github.com/docker/docker/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -// Testing to ensure 'year' fields is between 0 and 9999 -func TestFastTimeMarshalJSONWithInvalidDate(t *testing.T) { +func TestFastTimeMarshalJSONWithInvalidYear(t *testing.T) { aTime := time.Date(-1, 1, 1, 0, 0, 0, 0, time.Local) - json, err := FastTimeMarshalJSON(aTime) - if err == nil { - t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json) - } - anotherTime := time.Date(10000, 1, 1, 0, 0, 0, 0, time.Local) - json, err = FastTimeMarshalJSON(anotherTime) - if err == nil { - t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json) - } + _, err := fastTimeMarshalJSON(aTime) + testutil.ErrorContains(t, err, "year outside of range") + anotherTime := time.Date(10000, 1, 1, 0, 0, 0, 0, time.Local) + _, err = fastTimeMarshalJSON(anotherTime) + testutil.ErrorContains(t, err, "year outside of range") } func TestFastTimeMarshalJSON(t *testing.T) { aTime := time.Date(2015, 5, 29, 11, 1, 2, 3, time.UTC) - json, err := FastTimeMarshalJSON(aTime) - if err != nil { - t.Fatal(err) - } - expected := "\"2015-05-29T11:01:02.000000003Z\"" - if json != expected { - t.Fatalf("Expected %v, got %v", expected, json) - } + json, err := fastTimeMarshalJSON(aTime) + require.NoError(t, err) + assert.Equal(t, "\"2015-05-29T11:01:02.000000003Z\"", json) location, err := time.LoadLocation("Europe/Paris") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + aTime = time.Date(2015, 5, 29, 11, 1, 2, 3, location) - json, err = FastTimeMarshalJSON(aTime) - if err != nil { - t.Fatal(err) - } - expected = "\"2015-05-29T11:01:02.000000003+02:00\"" - if json != expected { - t.Fatalf("Expected %v, got %v", expected, json) - } + json, err = fastTimeMarshalJSON(aTime) + require.NoError(t, err) + assert.Equal(t, "\"2015-05-29T11:01:02.000000003+02:00\"", json) } From a001c9d5c77717ee2aeb6d0f182c84d87cd6e6f5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 22 Sep 2017 15:59:28 -0400 Subject: [PATCH 29/34] Remove unused Format Signed-off-by: Daniel Nephin Upstream-commit: 638d4cc7e4390ab217be711913017b22ce2cd5c2 Component: engine --- components/engine/pkg/jsonlog/jsonlog.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/components/engine/pkg/jsonlog/jsonlog.go b/components/engine/pkg/jsonlog/jsonlog.go index 4734c31119..7b9871ae87 100644 --- a/components/engine/pkg/jsonlog/jsonlog.go +++ b/components/engine/pkg/jsonlog/jsonlog.go @@ -1,8 +1,6 @@ package jsonlog import ( - "encoding/json" - "fmt" "time" ) @@ -19,24 +17,10 @@ type JSONLog struct { Attrs map[string]string `json:"attrs,omitempty"` } -// Format returns the log formatted according to format -// If format is nil, returns the log message -// If format is json, returns the log marshaled in json format -// By default, returns the log with the log time formatted according to format. -func (jl *JSONLog) Format(format string) (string, error) { - if format == "" { - return jl.Log, nil - } - if format == "json" { - m, err := json.Marshal(jl) - return string(m), err - } - return fmt.Sprintf("%s %s", jl.Created.Format(format), jl.Log), nil -} - -// Reset resets the log to nil. +// Reset all fields to their zero value. func (jl *JSONLog) Reset() { jl.Log = "" jl.Stream = "" jl.Created = time.Time{} + jl.Attrs = make(map[string]string) } From 4701b668895428904eae7444d040f74661f2f41c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 22 Sep 2017 16:20:50 -0400 Subject: [PATCH 30/34] Remove unused JSONLog marshaling Signed-off-by: Daniel Nephin Upstream-commit: 231c5cbd50e1870f31eb6a525b2df2ed7a716565 Component: engine --- .../engine/daemon/logger/jsonfilelog/read.go | 3 +- .../engine/pkg/jsonlog/jsonlog_marshalling.go | 178 ------------------ .../pkg/jsonlog/jsonlog_marshalling_test.go | 32 ---- components/engine/pkg/jsonlog/jsonlogbytes.go | 66 ++++++- 4 files changed, 66 insertions(+), 213 deletions(-) delete mode 100644 components/engine/pkg/jsonlog/jsonlog_marshalling.go delete mode 100644 components/engine/pkg/jsonlog/jsonlog_marshalling_test.go diff --git a/components/engine/daemon/logger/jsonfilelog/read.go b/components/engine/daemon/logger/jsonfilelog/read.go index 25fc99a984..7f22cc4703 100644 --- a/components/engine/daemon/logger/jsonfilelog/read.go +++ b/components/engine/daemon/logger/jsonfilelog/read.go @@ -147,9 +147,8 @@ func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since ti rdr = bytes.NewBuffer(bytes.Join(ls, []byte("\n"))) } dec := json.NewDecoder(rdr) - l := &jsonlog.JSONLog{} for { - msg, err := decodeLogLine(dec, l) + msg, err := decodeLogLine(dec, &jsonlog.JSONLog{}) if err != nil { if err != io.EOF { logWatcher.Err <- err diff --git a/components/engine/pkg/jsonlog/jsonlog_marshalling.go b/components/engine/pkg/jsonlog/jsonlog_marshalling.go deleted file mode 100644 index 8fae044b96..0000000000 --- a/components/engine/pkg/jsonlog/jsonlog_marshalling.go +++ /dev/null @@ -1,178 +0,0 @@ -// This code was initially generated by ffjson -// This code was generated via the following steps: -// $ go get -u github.com/pquerna/ffjson -// $ make BIND_DIR=. shell -// $ ffjson pkg/jsonlog/jsonlog.go -// $ mv pkg/jsonglog/jsonlog_ffjson.go pkg/jsonlog/jsonlog_marshalling.go -// -// It has been modified to improve the performance of time marshalling to JSON -// and to clean it up. -// Should this code need to be regenerated when the JSONLog struct is changed, -// the relevant changes which have been made are: -// import ( -// "bytes" -//- -// "unicode/utf8" -// ) -// -// func (mj *JSONLog) MarshalJSON() ([]byte, error) { -//@@ -20,13 +16,13 @@ func (mj *JSONLog) MarshalJSON() ([]byte, error) { -// } -// return buf.Bytes(), nil -// } -//+ -// func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { -//- var err error -//- var obj []byte -//- var first bool = true -//- _ = obj -//- _ = err -//- _ = first -//+ var ( -//+ err error -//+ timestamp string -//+ first bool = true -//+ ) -// buf.WriteString(`{`) -// if len(mj.Log) != 0 { -// if first == true { -//@@ -52,11 +48,11 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { -// buf.WriteString(`,`) -// } -// buf.WriteString(`"time":`) -//- obj, err = mj.Created.MarshalJSON() -//+ timestamp, err = FastTimeMarshalJSON(mj.Created) -// if err != nil { -// return err -// } -//- buf.Write(obj) -//+ buf.WriteString(timestamp) -// buf.WriteString(`}`) -// return nil -// } -// @@ -81,9 +81,10 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { -// if len(mj.Log) != 0 { -// - if first == true { -// - first = false -// - } else { -// - buf.WriteString(`,`) -// - } -// + first = false -// buf.WriteString(`"log":`) -// ffjsonWriteJSONString(buf, mj.Log) -// } - -package jsonlog - -import ( - "bytes" - "unicode/utf8" -) - -// MarshalJSON marshals the JSONLog. -func (mj *JSONLog) MarshalJSON() ([]byte, error) { - var buf bytes.Buffer - buf.Grow(1024) - if err := mj.MarshalJSONBuf(&buf); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// MarshalJSONBuf marshals the JSONLog and stores the result to a bytes.Buffer. -func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { - var ( - err error - timestamp string - first = true - ) - buf.WriteString(`{`) - if len(mj.Log) != 0 { - first = false - buf.WriteString(`"log":`) - ffjsonWriteJSONString(buf, mj.Log) - } - if len(mj.Stream) != 0 { - if first { - first = false - } else { - buf.WriteString(`,`) - } - buf.WriteString(`"stream":`) - ffjsonWriteJSONString(buf, mj.Stream) - } - if !first { - buf.WriteString(`,`) - } - buf.WriteString(`"time":`) - timestamp, err = fastTimeMarshalJSON(mj.Created) - if err != nil { - return err - } - buf.WriteString(timestamp) - buf.WriteString(`}`) - return nil -} - -func ffjsonWriteJSONString(buf *bytes.Buffer, s string) { - const hex = "0123456789abcdef" - - buf.WriteByte('"') - start := 0 - for i := 0; i < len(s); { - if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { - i++ - continue - } - if start < i { - buf.WriteString(s[start:i]) - } - switch b { - case '\\', '"': - buf.WriteByte('\\') - buf.WriteByte(b) - case '\n': - buf.WriteByte('\\') - buf.WriteByte('n') - case '\r': - buf.WriteByte('\\') - buf.WriteByte('r') - default: - - buf.WriteString(`\u00`) - buf.WriteByte(hex[b>>4]) - buf.WriteByte(hex[b&0xF]) - } - i++ - start = i - continue - } - c, size := utf8.DecodeRuneInString(s[i:]) - if c == utf8.RuneError && size == 1 { - if start < i { - buf.WriteString(s[start:i]) - } - buf.WriteString(`\ufffd`) - i += size - start = i - continue - } - - if c == '\u2028' || c == '\u2029' { - if start < i { - buf.WriteString(s[start:i]) - } - buf.WriteString(`\u202`) - buf.WriteByte(hex[c&0xF]) - i += size - start = i - continue - } - i += size - } - if start < len(s) { - buf.WriteString(s[start:]) - } - buf.WriteByte('"') -} diff --git a/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go b/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go deleted file mode 100644 index a6178a8c5b..0000000000 --- a/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package jsonlog - -import ( - "regexp" - "testing" - - "encoding/json" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestJSONLogMarshalJSON(t *testing.T) { - logs := map[*JSONLog]string{ - {Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`, - {Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`, - {Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`, - {Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`, - {Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`, - {Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`, - {}: `^{\"time\":\".{20,}\"}$`, - // These ones are a little weird - {Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`, - {Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`, - {Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`, - } - for jsonLog, expression := range logs { - data, err := jsonLog.MarshalJSON() - require.NoError(t, err) - assert.Regexp(t, regexp.MustCompile(expression), string(data)) - assert.NoError(t, json.Unmarshal(data, &map[string]interface{}{})) - } -} diff --git a/components/engine/pkg/jsonlog/jsonlogbytes.go b/components/engine/pkg/jsonlog/jsonlogbytes.go index b6663b9289..79941d4119 100644 --- a/components/engine/pkg/jsonlog/jsonlogbytes.go +++ b/components/engine/pkg/jsonlog/jsonlogbytes.go @@ -61,8 +61,72 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { return nil } -// This is based on ffjsonWriteJSONBytesAsString. It has been changed +func ffjsonWriteJSONString(buf *bytes.Buffer, s string) { + const hex = "0123456789abcdef" + + buf.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + if start < i { + buf.WriteString(s[start:i]) + } + switch b { + case '\\', '"': + buf.WriteByte('\\') + buf.WriteByte(b) + case '\n': + buf.WriteByte('\\') + buf.WriteByte('n') + case '\r': + buf.WriteByte('\\') + buf.WriteByte('r') + default: + + buf.WriteString(`\u00`) + buf.WriteByte(hex[b>>4]) + buf.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\ufffd`) + i += size + start = i + continue + } + + if c == '\u2028' || c == '\u2029' { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\u202`) + buf.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + buf.WriteString(s[start:]) + } + buf.WriteByte('"') +} + +// This is based on ffjsonWriteJSONString. It has been changed // to accept a string passed as a slice of bytes. +// TODO: remove duplication with ffjsonWriteJSONString func ffjsonWriteJSONBytesAsString(buf *bytes.Buffer, s []byte) { const hex = "0123456789abcdef" From 7d644fb209e462012479a6a0ca8991fa703084af Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 25 Sep 2017 15:52:42 -0400 Subject: [PATCH 31/34] Fix benchmarks and remove more unnecessary code. Signed-off-by: Daniel Nephin Upstream-commit: a06ad2792ab92d4f246e4b4cc4c3529eb060651e Component: engine --- .../daemon/logger/jsonfilelog/jsonfilelog.go | 8 +- .../logger/jsonfilelog/jsonfilelog_test.go | 104 ++++++------------ .../daemon/logger/jsonfilelog/read_test.go | 65 +++++++++++ components/engine/pkg/jsonlog/jsonlog.go | 3 +- components/engine/pkg/jsonlog/jsonlogbytes.go | 72 +----------- 5 files changed, 103 insertions(+), 149 deletions(-) create mode 100644 components/engine/daemon/logger/jsonfilelog/read_test.go diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 351d6eeed5..f9aac82fe5 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -14,7 +14,7 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/jsonlog" - "github.com/docker/go-units" + units "github.com/docker/go-units" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -106,10 +106,8 @@ func writeMessageBuf(w io.Writer, m *logger.Message, extra json.RawMessage, buf return err } logger.PutMessage(m) - if _, err := w.Write(buf.Bytes()); err != nil { - return errors.Wrap(err, "error writing log entry") - } - return nil + _, err := w.Write(buf.Bytes()) + return errors.Wrap(err, "error writing log entry") } func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go index d2d36e943b..e3ebc64d0e 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -1,6 +1,7 @@ package jsonfilelog import ( + "bytes" "encoding/json" "io/ioutil" "os" @@ -12,6 +13,8 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/pkg/jsonlog" + "github.com/gotestyourself/gotestyourself/fs" + "github.com/stretchr/testify/require" ) func TestJSONFileLogger(t *testing.T) { @@ -54,36 +57,38 @@ func TestJSONFileLogger(t *testing.T) { } } -func BenchmarkJSONFileLogger(b *testing.B) { - cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" - tmp, err := ioutil.TempDir("", "docker-logger-") - if err != nil { - b.Fatal(err) - } - defer os.RemoveAll(tmp) - filename := filepath.Join(tmp, "container.log") - l, err := New(logger.Info{ - ContainerID: cid, - LogPath: filename, - }) - if err != nil { - b.Fatal(err) - } - defer l.Close() +func BenchmarkJSONFileLoggerLog(b *testing.B) { + tmp := fs.NewDir(b, "bench-jsonfilelog") + defer tmp.Remove() - testLine := "Line that thinks that it is log line from docker\n" - msg := &logger.Message{Line: []byte(testLine), Source: "stderr", Timestamp: time.Now().UTC()} - jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON() - if err != nil { - b.Fatal(err) + jsonlogger, err := New(logger.Info{ + ContainerID: "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657", + LogPath: tmp.Join("container.log"), + Config: map[string]string{ + "labels": "first,second", + }, + ContainerLabels: map[string]string{ + "first": "label_value", + "second": "label_foo", + }, + }) + require.NoError(b, err) + defer jsonlogger.Close() + + msg := &logger.Message{ + Line: []byte("Line that thinks that it is log line from docker\n"), + Source: "stderr", + Timestamp: time.Now().UTC(), } - b.SetBytes(int64(len(jsonlog)+1) * 30) + + buf := bytes.NewBuffer(nil) + require.NoError(b, marshalMessage(msg, jsonlogger.(*JSONFileLogger).extra, buf)) + b.SetBytes(int64(buf.Len())) + b.ResetTimer() for i := 0; i < b.N; i++ { - for j := 0; j < 30; j++ { - if err := l.Log(msg); err != nil { - b.Fatal(err) - } + if err := jsonlogger.Log(msg); err != nil { + b.Fatal(err) } } } @@ -200,50 +205,3 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) { t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected) } } - -func BenchmarkJSONFileLoggerWithReader(b *testing.B) { - b.StopTimer() - b.ResetTimer() - cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" - dir, err := ioutil.TempDir("", "json-logger-bench") - if err != nil { - b.Fatal(err) - } - defer os.RemoveAll(dir) - - l, err := New(logger.Info{ - ContainerID: cid, - LogPath: filepath.Join(dir, "container.log"), - }) - if err != nil { - b.Fatal(err) - } - defer l.Close() - msg := &logger.Message{Line: []byte("line"), Source: "src1"} - jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON() - if err != nil { - b.Fatal(err) - } - b.SetBytes(int64(len(jsonlog)+1) * 30) - - b.StartTimer() - - go func() { - for i := 0; i < b.N; i++ { - for j := 0; j < 30; j++ { - l.Log(msg) - } - } - l.Close() - }() - - lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Follow: true}) - watchClose := lw.WatchClose() - for { - select { - case <-lw.Msg: - case <-watchClose: - return - } - } -} diff --git a/components/engine/daemon/logger/jsonfilelog/read_test.go b/components/engine/daemon/logger/jsonfilelog/read_test.go new file mode 100644 index 0000000000..ffae3b0ab5 --- /dev/null +++ b/components/engine/daemon/logger/jsonfilelog/read_test.go @@ -0,0 +1,65 @@ +package jsonfilelog + +import ( + "testing" + + "bytes" + "time" + + "github.com/docker/docker/daemon/logger" + "github.com/gotestyourself/gotestyourself/fs" + "github.com/stretchr/testify/require" +) + +func BenchmarkJSONFileLoggerReadLogs(b *testing.B) { + tmp := fs.NewDir(b, "bench-jsonfilelog") + defer tmp.Remove() + + jsonlogger, err := New(logger.Info{ + ContainerID: "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657", + LogPath: tmp.Join("container.log"), + Config: map[string]string{ + "labels": "first,second", + }, + ContainerLabels: map[string]string{ + "first": "label_value", + "second": "label_foo", + }, + }) + require.NoError(b, err) + defer jsonlogger.Close() + + msg := &logger.Message{ + Line: []byte("Line that thinks that it is log line from docker\n"), + Source: "stderr", + Timestamp: time.Now().UTC(), + } + + buf := bytes.NewBuffer(nil) + require.NoError(b, marshalMessage(msg, jsonlogger.(*JSONFileLogger).extra, buf)) + b.SetBytes(int64(buf.Len())) + + b.ResetTimer() + + chError := make(chan error, b.N+1) + go func() { + for i := 0; i < b.N; i++ { + chError <- jsonlogger.Log(msg) + } + chError <- jsonlogger.Close() + }() + + lw := jsonlogger.(*JSONFileLogger).ReadLogs(logger.ReadConfig{Follow: true}) + watchClose := lw.WatchClose() + for { + select { + case <-lw.Msg: + case <-watchClose: + return + case err := <-chError: + if err != nil { + b.Fatal(err) + } + } + } +} diff --git a/components/engine/pkg/jsonlog/jsonlog.go b/components/engine/pkg/jsonlog/jsonlog.go index 7b9871ae87..549e355855 100644 --- a/components/engine/pkg/jsonlog/jsonlog.go +++ b/components/engine/pkg/jsonlog/jsonlog.go @@ -4,8 +4,7 @@ import ( "time" ) -// JSONLog represents a log message, typically a single entry from a given log stream. -// JSONLogs can be easily serialized to and from JSON and support custom formatting. +// JSONLog is a log message, typically a single entry from a given log stream. type JSONLog struct { // Log is the log message Log string `json:"log,omitempty"` diff --git a/components/engine/pkg/jsonlog/jsonlogbytes.go b/components/engine/pkg/jsonlog/jsonlogbytes.go index 79941d4119..37604ae549 100644 --- a/components/engine/pkg/jsonlog/jsonlogbytes.go +++ b/components/engine/pkg/jsonlog/jsonlogbytes.go @@ -17,8 +17,8 @@ type JSONLogs struct { RawAttrs json.RawMessage `json:"attrs,omitempty"` } -// MarshalJSONBuf is based on the same method from JSONLog -// It has been modified to take into account the necessary changes. +// MarshalJSONBuf is an optimized JSON marshaller that avoids reflection +// and unnecessary allocation. func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { var first = true @@ -35,7 +35,7 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { buf.WriteString(`,`) } buf.WriteString(`"stream":`) - ffjsonWriteJSONString(buf, mj.Stream) + ffjsonWriteJSONBytesAsString(buf, []byte(mj.Stream)) } if len(mj.RawAttrs) > 0 { if first { @@ -61,72 +61,6 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { return nil } -func ffjsonWriteJSONString(buf *bytes.Buffer, s string) { - const hex = "0123456789abcdef" - - buf.WriteByte('"') - start := 0 - for i := 0; i < len(s); { - if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { - i++ - continue - } - if start < i { - buf.WriteString(s[start:i]) - } - switch b { - case '\\', '"': - buf.WriteByte('\\') - buf.WriteByte(b) - case '\n': - buf.WriteByte('\\') - buf.WriteByte('n') - case '\r': - buf.WriteByte('\\') - buf.WriteByte('r') - default: - - buf.WriteString(`\u00`) - buf.WriteByte(hex[b>>4]) - buf.WriteByte(hex[b&0xF]) - } - i++ - start = i - continue - } - c, size := utf8.DecodeRuneInString(s[i:]) - if c == utf8.RuneError && size == 1 { - if start < i { - buf.WriteString(s[start:i]) - } - buf.WriteString(`\ufffd`) - i += size - start = i - continue - } - - if c == '\u2028' || c == '\u2029' { - if start < i { - buf.WriteString(s[start:i]) - } - buf.WriteString(`\u202`) - buf.WriteByte(hex[c&0xF]) - i += size - start = i - continue - } - i += size - } - if start < len(s) { - buf.WriteString(s[start:]) - } - buf.WriteByte('"') -} - -// This is based on ffjsonWriteJSONString. It has been changed -// to accept a string passed as a slice of bytes. -// TODO: remove duplication with ffjsonWriteJSONString func ffjsonWriteJSONBytesAsString(buf *bytes.Buffer, s []byte) { const hex = "0123456789abcdef" From f367f50630c2e5b4dab5cda6675e6949384e7703 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 25 Sep 2017 15:57:45 -0400 Subject: [PATCH 32/34] Move jsonlog to a subpackage of jsonfilelog Signed-off-by: Daniel Nephin Upstream-commit: 035604cca6d6bd9a432268caf7515a35023908ed Component: engine --- components/engine/daemon/logger/jsonfilelog/jsonfilelog.go | 2 +- .../engine/daemon/logger/jsonfilelog/jsonfilelog_test.go | 2 +- .../{pkg => daemon/logger/jsonfilelog}/jsonlog/jsonlog.go | 0 .../{pkg => daemon/logger/jsonfilelog}/jsonlog/jsonlogbytes.go | 0 .../logger/jsonfilelog}/jsonlog/jsonlogbytes_test.go | 0 .../logger/jsonfilelog}/jsonlog/time_marshalling.go | 0 .../logger/jsonfilelog}/jsonlog/time_marshalling_test.go | 0 components/engine/daemon/logger/jsonfilelog/read.go | 2 +- components/engine/daemon/logger/jsonfilelog/read_test.go | 3 +-- 9 files changed, 4 insertions(+), 5 deletions(-) rename components/engine/{pkg => daemon/logger/jsonfilelog}/jsonlog/jsonlog.go (100%) rename components/engine/{pkg => daemon/logger/jsonfilelog}/jsonlog/jsonlogbytes.go (100%) rename components/engine/{pkg => daemon/logger/jsonfilelog}/jsonlog/jsonlogbytes_test.go (100%) rename components/engine/{pkg => daemon/logger/jsonfilelog}/jsonlog/time_marshalling.go (100%) rename components/engine/{pkg => daemon/logger/jsonfilelog}/jsonlog/time_marshalling_test.go (100%) diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index f9aac82fe5..177c070394 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -12,8 +12,8 @@ import ( "sync" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/docker/docker/daemon/logger/loggerutils" - "github.com/docker/docker/pkg/jsonlog" units "github.com/docker/go-units" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go index e3ebc64d0e..2b2b2b5229 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -12,7 +12,7 @@ import ( "time" "github.com/docker/docker/daemon/logger" - "github.com/docker/docker/pkg/jsonlog" + "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/gotestyourself/gotestyourself/fs" "github.com/stretchr/testify/require" ) diff --git a/components/engine/pkg/jsonlog/jsonlog.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go similarity index 100% rename from components/engine/pkg/jsonlog/jsonlog.go rename to components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go diff --git a/components/engine/pkg/jsonlog/jsonlogbytes.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go similarity index 100% rename from components/engine/pkg/jsonlog/jsonlogbytes.go rename to components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go diff --git a/components/engine/pkg/jsonlog/jsonlogbytes_test.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go similarity index 100% rename from components/engine/pkg/jsonlog/jsonlogbytes_test.go rename to components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go diff --git a/components/engine/pkg/jsonlog/time_marshalling.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/time_marshalling.go similarity index 100% rename from components/engine/pkg/jsonlog/time_marshalling.go rename to components/engine/daemon/logger/jsonfilelog/jsonlog/time_marshalling.go diff --git a/components/engine/pkg/jsonlog/time_marshalling_test.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/time_marshalling_test.go similarity index 100% rename from components/engine/pkg/jsonlog/time_marshalling_test.go rename to components/engine/daemon/logger/jsonfilelog/jsonlog/time_marshalling_test.go diff --git a/components/engine/daemon/logger/jsonfilelog/read.go b/components/engine/daemon/logger/jsonfilelog/read.go index 7f22cc4703..2586c7d7f7 100644 --- a/components/engine/daemon/logger/jsonfilelog/read.go +++ b/components/engine/daemon/logger/jsonfilelog/read.go @@ -13,9 +13,9 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/docker/docker/daemon/logger/jsonfilelog/multireader" "github.com/docker/docker/pkg/filenotify" - "github.com/docker/docker/pkg/jsonlog" "github.com/docker/docker/pkg/tailfile" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/components/engine/daemon/logger/jsonfilelog/read_test.go b/components/engine/daemon/logger/jsonfilelog/read_test.go index ffae3b0ab5..01a05c4b78 100644 --- a/components/engine/daemon/logger/jsonfilelog/read_test.go +++ b/components/engine/daemon/logger/jsonfilelog/read_test.go @@ -1,9 +1,8 @@ package jsonfilelog import ( - "testing" - "bytes" + "testing" "time" "github.com/docker/docker/daemon/logger" From ec4868350dc2ace4abc877d39cedd03c07297a0e Mon Sep 17 00:00:00 2001 From: Boaz Shuster Date: Wed, 2 Aug 2017 00:30:18 +0300 Subject: [PATCH 33/34] Add an integration test for bug #31392 regression This verifies that bug #31392 won't surface again. To reproduce the bug: 1) docker run -dit --name a0 busybox sh 2) docker run -dit --name b0 --link a0 busybox sh 3) docker rename a0 a1 4) docker run -dit --name a0 busybox sh 5) docker rm -f b0 6) docker run -dit --name b0 --link a0 busybox sh Signed-off-by: Boaz Shuster Upstream-commit: 48a26ba9e42f25ebc1ad732b4c0d31e77a7aaa55 Component: engine --- .../integration/container/rename_test.go | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 components/engine/integration/container/rename_test.go diff --git a/components/engine/integration/container/rename_test.go b/components/engine/integration/container/rename_test.go new file mode 100644 index 0000000000..cf3675734a --- /dev/null +++ b/components/engine/integration/container/rename_test.go @@ -0,0 +1,88 @@ +package container + +import ( + "context" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/client" + "github.com/docker/docker/integration/util/request" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func runContainer(ctx context.Context, t *testing.T, client client.APIClient, cntCfg *container.Config, hstCfg *container.HostConfig, nwkCfg *network.NetworkingConfig, cntName string) string { + cnt, err := client.ContainerCreate(ctx, cntCfg, hstCfg, nwkCfg, cntName) + require.NoError(t, err) + + err = client.ContainerStart(ctx, cnt.ID, types.ContainerStartOptions{}) + require.NoError(t, err) + return cnt.ID +} + +// This test simulates the scenario mentioned in #31392: +// Having two linked container, renaming the target and bringing a replacement +// and then deleting and recreating the source container linked to the new target. +// This checks that "rename" updates source container correctly and doesn't set it to null. +func TestRenameLinkedContainer(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + cntConfig := &container.Config{ + Image: "busybox", + Tty: true, + Cmd: strslice.StrSlice([]string{"top"}), + } + + var ( + aID, bID string + cntJSON types.ContainerJSON + err error + ) + + aID = runContainer(ctx, t, client, + cntConfig, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "a0", + ) + + bID = runContainer(ctx, t, client, + cntConfig, + &container.HostConfig{ + Links: []string{"a0"}, + }, + &network.NetworkingConfig{}, + "b0", + ) + + err = client.ContainerRename(ctx, aID, "a1") + require.NoError(t, err) + + runContainer(ctx, t, client, + cntConfig, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "a0", + ) + + err = client.ContainerRemove(ctx, bID, types.ContainerRemoveOptions{Force: true}) + require.NoError(t, err) + + bID = runContainer(ctx, t, client, + cntConfig, + &container.HostConfig{ + Links: []string{"a0"}, + }, + &network.NetworkingConfig{}, + "b0", + ) + + cntJSON, err = client.ContainerInspect(ctx, bID) + require.NoError(t, err) + assert.Equal(t, []string{"/a0:/b0/a0"}, cntJSON.HostConfig.Links) +} From cbf8cc6faf436e0257a10e0c195fcfeca2dcd1e5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 26 Sep 2017 11:45:54 +0200 Subject: [PATCH 34/34] Suppress warning for renaming missing tmp directory When starting `dockerd` on a host that has no `/var/lib/docker/tmp` directory, a warning was printed in the logs: $ dockerd --data-root=/no-such-directory ... WARN[2017-09-26T09:37:00.045153377Z] failed to rename /no-such-directory/tmp for background deletion: rename /no-such-directory/tmp /no-such-directory/tmp-old: no such file or directory. Deleting synchronously Although harmless, the warning does not show any useful information, so can be skipped. This patch checks thetype of error, so that warning is not printed. Other errors will still show up: $ touch /i-am-a-file $ dockerd --data-root=/i-am-a-file Unable to get the full path to root (/i-am-a-file): canonical path points to a file '/i-am-a-file' Signed-off-by: Sebastiaan van Stijn Upstream-commit: 2b50b14aebc12722f81db8d8f66415e1fa7b954a Component: engine --- components/engine/daemon/daemon.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 19d22bf702..c187f29ceb 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -1044,7 +1044,7 @@ func prepareTempDir(rootDir string, rootIDs idtools.IDPair) (string, error) { logrus.Warnf("failed to delete old tmp directory: %s", newName) } }() - } else { + } else if !os.IsNotExist(err) { logrus.Warnf("failed to rename %s for background deletion: %s. Deleting synchronously", tmpDir, err) if err := os.RemoveAll(tmpDir); err != nil { logrus.Warnf("failed to delete old tmp directory: %s", tmpDir)