From 1c2b78478c0c39c30add316454ec634952948cf2 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 26 Jun 2017 12:54:09 +0100 Subject: [PATCH 01/22] builder: Emit a BuildResult after squashing. This ensures that the final BuildResult is the actual tagged image. Fixes #33822. Signed-off-by: Ian Campbell Upstream-commit: 9777ec3be0e3056d0bedcf36052704f832e45759 Component: engine --- .../api/server/backend/build/backend.go | 5 ++++ .../integration-cli/docker_cli_build_test.go | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/components/engine/api/server/backend/build/backend.go b/components/engine/api/server/backend/build/backend.go index fe98c087b8..b5a758c387 100644 --- a/components/engine/api/server/backend/build/backend.go +++ b/components/engine/api/server/backend/build/backend.go @@ -55,6 +55,11 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string if imageID, err = squashBuild(build, b.imageComponent); err != nil { return "", err } + if config.ProgressWriter.AuxFormatter != nil { + if err = config.ProgressWriter.AuxFormatter.Emit(types.BuildResult{ID: imageID}); err != nil { + return "", err + } + } } stdout := config.ProgressWriter.StdoutFormatter diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index b9a5dff789..77b151e89f 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -6459,3 +6459,32 @@ func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { c.Assert(err, check.NotNil) c.Assert(os.IsNotExist(err), check.Equals, true) } + +func (s *DockerSuite) TestBuildIidFileSquash(c *check.C) { + testRequires(c, ExperimentalDaemon) + tmpDir, err := ioutil.TempDir("", "TestBuildIidFileSquash") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpDir) + tmpIidFile := filepath.Join(tmpDir, "iidsquash") + + name := "testbuildiidfilesquash" + // Use a Dockerfile with multiple stages to ensure we get the last one + cli.BuildCmd(c, name, + // This could be minimalBaseImage except + // https://github.com/moby/moby/issues/33823 requires + // `touch` to workaround. + build.WithDockerfile(`FROM busybox +ENV FOO FOO +ENV BAR BAR +RUN touch /foop +`), + cli.WithFlags("--iidfile", tmpIidFile, "--squash")) + + id, err := ioutil.ReadFile(tmpIidFile) + c.Assert(err, check.IsNil) + d, err := digest.Parse(string(id)) + c.Assert(err, check.IsNil) + c.Assert(d.String(), checker.Equals, getIDByName(c, name)) +} From 5a815104490f05bb3b1d085fac90c7ebf2c70333 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 26 Jun 2017 17:29:24 -0400 Subject: [PATCH 02/22] Do not error on relabel when relabel not supported Signed-off-by: Brian Goff Upstream-commit: ebfdfc5768b74e0a52875cf76a0576bfcd66445c Component: engine --- components/engine/volume/volume.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/engine/volume/volume.go b/components/engine/volume/volume.go index 2abfcc7b58..07176b89ce 100644 --- a/components/engine/volume/volume.go +++ b/components/engine/volume/volume.go @@ -156,6 +156,10 @@ func (m *MountPoint) Setup(mountLabel string, rootIDs idtools.IDPair, checkFun f if err == nil { if label.RelabelNeeded(m.Mode) { if err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode)); err != nil { + if err == syscall.ENOTSUP { + err = nil + return + } path = "" err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source) return From 836d4632cb46bae3cb529c6180f592ce890f6bdc Mon Sep 17 00:00:00 2001 From: John Stephens Date: Mon, 12 Jun 2017 18:18:14 -0700 Subject: [PATCH 03/22] Stop trying to load images on an incompatible OS Signed-off-by: John Stephens Upstream-commit: b9255e4a531d2ad0239481eba2a635f7d48718fb Component: engine --- components/engine/image/tarexport/load.go | 29 ++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/components/engine/image/tarexport/load.go b/components/engine/image/tarexport/load.go index adf4574da8..af8cefc6ad 100644 --- a/components/engine/image/tarexport/load.go +++ b/components/engine/image/tarexport/load.go @@ -79,6 +79,9 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) if err != nil { return err } + if err := checkCompatibleOS(img.OS); err != nil { + return err + } var rootFS image.RootFS rootFS = *img.RootFS rootFS.DiffIDs = nil @@ -292,11 +295,18 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str return err } - var img struct{ Parent string } + var img struct { + OS string + Parent string + } if err := json.Unmarshal(imageJSON, &img); err != nil { return err } + if err := checkCompatibleOS(img.OS); err != nil { + return err + } + var parentID image.ID if img.Parent != "" { for { @@ -402,3 +412,20 @@ func checkValidParent(img, parent *image.Image) bool { } return true } + +func checkCompatibleOS(os string) error { + // TODO @jhowardmsft LCOW - revisit for simultaneous platforms + platform := runtime.GOOS + if system.LCOWSupported() { + platform = "linux" + } + // always compatible if the OS matches; also match an empty OS + if os == platform || os == "" { + return nil + } + // for compatibility, only fail if the image or runtime OS is Windows + if os == "windows" || platform == "windows" { + return fmt.Errorf("cannot load %s image on %s", os, platform) + } + return nil +} From 6fc578b05fd3f502dea34500b659ef03f447e782 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 26 Jun 2017 16:10:03 -0700 Subject: [PATCH 04/22] Add builder dev report for 2017-06-26 Signed-off-by: Tonis Tiigi Upstream-commit: 294095d07e9d524a3add4afab3a9005a79a47015 Component: engine --- .../engine/reports/builder/2017-06-26.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 components/engine/reports/builder/2017-06-26.md diff --git a/components/engine/reports/builder/2017-06-26.md b/components/engine/reports/builder/2017-06-26.md new file mode 100644 index 0000000000..e0ba95a7a5 --- /dev/null +++ b/components/engine/reports/builder/2017-06-26.md @@ -0,0 +1,78 @@ +# Development Report for June 26, 2017 + + +### BuildKit + +[Repo](https://github.com/moby/buildkit) +[Proposal](https://github.com/moby/moby/issues/32925) + +New development repo is open at https://github.com/moby/buildkit + +The readme file provides examples how to get started. You can see an example of building BuildKit with BuildKit. + +There are lots of new issues opened as well to track the missing functionality. You are welcomed to help on any of them or discuss the design there. + +Last week most of the work was done on improving the `llb` client library for more complicated use cases and providing traces and interactive progress of executed build jobs. + +The `llb` client package is a go library that helps you to generate the build definition graph. It uses chained methods to make it easy to describe what steps need to be running. Mounts can be added to the execution steps for defining multiple inputs or outputs. To prepare the graph, you just have to call `Marshal()` on a leaf node that will generate the protobuf definition for everything required to build that node. + +### Typed Dockerfile parsing + +[PR](https://github.com/moby/moby/pull/33492) + +This PR that enables parsing Dockerfiles into typed structures so they can be preprocessed to eliminate unnecessary build stages and reused with different kinds of dispatchers(eg. BuildKit). + +The PR had some review and updates in last week. Should be ready to code review soon. + +### Merged: Long running session & incremental file sending + +[PR](https://github.com/moby/moby/pull/32677) + +Incremental context sending PR was merged and is expected to land in `v17.07`. + +This feature experimental feature lets you skip sending the build context to the daemon on repeated builder invocations during development. Currently, this feature requires a CLI flag `--stream=true`. If this flag is used, one first builder invocation full build context is sent to the daemon. On a second attempt, only the changed files are transferred. + +Previous build context is saved in the build cache, and you can see how much space it takes form `docker system df`. Build cache will be automatically garbage collected and can also be manually cleared with `docker prune`. + +### Quality: Dependency interface switch + +[Move file copying from the daemon to the builder](https://github.com/moby/moby/pull/33454) PR was merged. + + +### Proposals for new Dockerfile features that need design feedback: + +[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100) + +[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487) + +[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507) + +[DAG image builder](https://github.com/moby/moby/issues/32550) + +[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new) + +[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new) + +[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989) + +If you are interested in implementing any of them, leave a comment on the specific issues. + +### Other builder PRs merged last week + +[Warn/deprecate continuing on empty lines in `Dockerfile`](https://github.com/moby/moby/pull/29161) + +[Fix behavior of absolute paths in .dockerignore](https://github.com/moby/moby/pull/32088) + +[fix copy —from conflict with force pull](https://github.com/moby/moby/pull/33735) + +### Builder features currently in code-review: + +[Fix handling of remote "git@" notation](https://github.com/moby/moby/pull/33696) + +[builder: Emit a BuildResult after squashing.](https://github.com/moby/moby/pull/33824) + +[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704) + +### Backlog + +[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard. \ No newline at end of file From 359031100e3bc4e4598166fee485dc4671a73ead Mon Sep 17 00:00:00 2001 From: Dhawal Yogesh Bhanushali Date: Wed, 10 May 2017 04:24:35 -0700 Subject: [PATCH 05/22] Add overlay2.size daemon storage-opt This commit adds the overlay2.size option to the daemon daemon storage opts. The user can override this option by the "docker run --storage-opt" options. Signed-off-by: Dhawal Yogesh Bhanushali Upstream-commit: a63d5bc03513755015827d0fe93563240429f1e0 Component: engine --- .../daemon/graphdriver/overlay2/overlay.go | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index ccb2c3669d..deea3e0176 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -194,6 +194,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), supportsDType: supportsDType, locker: locker.New(), + options: *opts, } d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps) @@ -202,7 +203,12 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap // Try to enable project quota support over xfs. if d.quotaCtl, err = quota.NewControl(home); err == nil { projectQuotaSupported = true + } else if opts.quota.Size > 0 { + return nil, fmt.Errorf("Storage option overlay2.size not supported. Filesystem does not support Project Quota: %v", err) } + } else if opts.quota.Size > 0 { + // if xfs is not the backing fs then error out if the storage-opt overlay2.size is used. + return nil, fmt.Errorf("Storage Option overlay2.size only supported for backingFS XFS. Found %v", backingFs) } logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) @@ -224,9 +230,14 @@ func parseOptions(options []string) (*overlayOptions, error) { if err != nil { return nil, err } - + case "overlay2.size": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + o.quota.Size = uint64(size) default: - return nil, fmt.Errorf("overlay2: Unknown option %s\n", key) + return nil, fmt.Errorf("overlay2: unknown option %s", key) } } return o, nil @@ -312,17 +323,38 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { - return d.Create(id, parent, opts) + if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported { + return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option") + } + + if opts == nil { + opts = &graphdriver.CreateOpts{ + StorageOpt: map[string]string{}, + } + } + + if _, ok := opts.StorageOpt["size"]; !ok { + if opts.StorageOpt == nil { + opts.StorageOpt = map[string]string{} + } + opts.StorageOpt["size"] = strconv.FormatUint(d.options.quota.Size, 10) + } + + return d.create(id, parent, opts) } // Create is used to create the upper, lower, and merge directories required for overlay fs for a given id. // The parent filesystem is used to configure these directories for the overlay. func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { - - if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported { - return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option") + if opts != nil && len(opts.StorageOpt) != 0 { + if _, ok := opts.StorageOpt["size"]; ok { + return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers") + } } + return d.create(id, parent, opts) +} +func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { dir := d.dir(id) rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) From 4c52cfc0fb1727761ca3fe428c343ff8876d2017 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 27 Jun 2017 10:38:01 -0700 Subject: [PATCH 06/22] build: fix add from remote url Signed-off-by: Tonis Tiigi Upstream-commit: 2981667e11e24e9e1e61252a40ceb11fd7cb9811 Component: engine --- components/engine/builder/dockerfile/copy.go | 13 +++--- .../integration-cli/docker_api_build_test.go | 44 +++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/components/engine/builder/dockerfile/copy.go b/components/engine/builder/dockerfile/copy.go index 0026bdc158..c7db943f5c 100644 --- a/components/engine/builder/dockerfile/copy.go +++ b/components/engine/builder/dockerfile/copy.go @@ -32,9 +32,10 @@ type pathCache interface { // copyInfo is a data object which stores the metadata about each source file in // a copyInstruction type copyInfo struct { - root string - path string - hash string + root string + path string + hash string + noDecompress bool } func (c copyInfo) fullPath() (string, error) { @@ -124,7 +125,9 @@ func (o *copier) getCopyInfoForSourcePath(orig string) ([]copyInfo, error) { o.tmpPaths = append(o.tmpPaths, remote.Root()) hash, err := remote.Hash(path) - return newCopyInfos(newCopyInfoFromSource(remote, path, hash)), err + ci := newCopyInfoFromSource(remote, path, hash) + ci.noDecompress = true // data from http shouldn't be extracted even on ADD + return newCopyInfos(ci), err } // Cleanup removes any temporary directories created as part of downloading @@ -387,7 +390,7 @@ func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) if src.IsDir() { return copyDirectory(archiver, srcPath, destPath) } - if options.decompress && archive.IsArchivePath(srcPath) { + if options.decompress && archive.IsArchivePath(srcPath) && !source.noDecompress { return archiver.UntarPath(srcPath, destPath) } diff --git a/components/engine/integration-cli/docker_api_build_test.go b/components/engine/integration-cli/docker_api_build_test.go index a5e19f658b..c1ab7661e0 100644 --- a/components/engine/integration-cli/docker_api_build_test.go +++ b/components/engine/integration-cli/docker_api_build_test.go @@ -366,6 +366,50 @@ func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) { assert.Contains(c, string(out), "Successfully built") } +func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) { + buffer := new(bytes.Buffer) + tw := tar.NewWriter(buffer) + dt := []byte("contents") + err := tw.WriteHeader(&tar.Header{ + Name: "foo", + Size: int64(len(dt)), + Mode: 0600, + Typeflag: tar.TypeReg, + }) + require.NoError(c, err) + _, err = tw.Write(dt) + require.NoError(c, err) + err = tw.Close() + require.NoError(c, err) + + server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ + "test.tar": buffer, + })) + defer server.Close() + + dockerfile := fmt.Sprintf(` + FROM busybox + ADD %s/test.tar / + RUN [ -f test.tar ] + `, server.URL()) + + ctx := fakecontext.New(c, "", + fakecontext.WithDockerfile(dockerfile), + ) + defer ctx.Close() + + res, body, err := request.Post( + "/build", + request.RawContent(ctx.AsTarReader(c)), + request.ContentType("application/x-tar")) + require.NoError(c, err) + assert.Equal(c, http.StatusOK, res.StatusCode) + + out, err := testutil.ReadBody(body) + require.NoError(c, err) + assert.Contains(c, string(out), "Successfully built") +} + func (s *DockerSuite) TestBuildWithSession(c *check.C) { testRequires(c, ExperimentalDaemon) From 92d764e7fbe8182395c7614dfe8b11ba52d69fb1 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 23 Jun 2017 14:34:50 -0700 Subject: [PATCH 07/22] LCOW: Dedicated scratch space for global service VM Signed-off-by: John Howard Upstream-commit: d1d649c891c8b06567afba7542ea9bf518cdaa02 Component: engine --- .../engine/daemon/graphdriver/lcow/lcow.go | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index 6bf491574a..2adbf12c2b 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -16,6 +16,7 @@ import ( "sync" "time" + "github.com/Microsoft/hcsshim" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" @@ -32,6 +33,9 @@ func init() { const ( // sandboxFilename is the name of the file containing a layers sandbox (read-write layer) sandboxFilename = "sandbox.vhdx" + + // svmScratchFilename is the name of the scratch-space used by an SVM to avoid running out of memory + svmScratchFilename = "scratch.vhdx" ) // cacheType is our internal structure representing an item in our local cache @@ -51,8 +55,13 @@ type Driver struct { cachedSandboxFile string // options are the graphdriver options we are initialised with options []string - // JJH LIFETIME TODO - Remove this and move up to daemon. For now, a global service utility-VM + // config is the representation of the SVM. + // @jhowardmsft LIFETIME TODO - For now, a global service utility-VM config client.Config + // svmScratchSpaceFile is a host location for a dedicated scratch space + // that the SVM utilities can use as a scratch-space to avoid OOMs + // @jhowardmsft LIFETIME TODO - For now, a global service utility-VM + svmScratchSpaceFile string // it is safe for windows to use a cache here because it does not support // restoring containers when the daemon dies. @@ -69,10 +78,11 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) ( logrus.Debugf("%s %s", title, home) d := &Driver{ - homeDir: home, - options: options, - cachedSandboxFile: filepath.Join(home, "cache", sandboxFilename), - cache: make(map[string]cacheType), + homeDir: home, + options: options, + cachedSandboxFile: filepath.Join(home, "cache", sandboxFilename), + svmScratchSpaceFile: filepath.Join(home, "svmscratch", svmScratchFilename), + cache: make(map[string]cacheType), } if err := idtools.MkdirAllAs(home, 0700, 0, 0); err != nil { @@ -84,6 +94,11 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) ( return nil, fmt.Errorf("%s failed to create '%s': %v", title, home, err) } + // Location for the SVM scratch + if err := idtools.MkdirAllAs(filepath.Dir(d.svmScratchSpaceFile), 0700, 0, 0); err != nil { + return nil, fmt.Errorf("%s failed to create '%s': %v", title, home, err) + } + return d, nil } @@ -92,6 +107,8 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) ( // service VM globally to a service VM per container (or offline operation). However, // for the initial bring-up of LCOW, this is acceptable. func (d *Driver) startUvm(context string) error { + const toolsScratchPath = "/mnt/gcs/LinuxServiceVM/scratch" + // Nothing to do if it's already running if d.config.Uvm != nil { return nil @@ -102,10 +119,36 @@ func (d *Driver) startUvm(context string) error { return fmt.Errorf("failed to generate default gogcs configuration (%s): %s", context, err) } + scratchAttached := false + if _, err := os.Stat(d.svmScratchSpaceFile); err == nil { + // We have a scratch space already, so just attach it as a mapped virtual disk + logrus.Debugf("lcowdriver: startuvm: (%s) attaching pre-existing scratch", context) + mvd := hcsshim.MappedVirtualDisk{ + HostPath: d.svmScratchSpaceFile, + ContainerPath: toolsScratchPath, + CreateInUtilityVM: true, + } + d.config.MappedVirtualDisks = append(d.config.MappedVirtualDisks, mvd) + scratchAttached = true + } + d.config.Name = "LinuxServiceVM" // TODO @jhowardmsft - This requires an in-flight platform change. Can't hard code it to this longer term if err := d.config.Create(); err != nil { return fmt.Errorf("failed to start utility VM (%s): %s", context, err) } + + // Hot-add the scratch-space if not already attached + if !scratchAttached { + logrus.Debugf("lcowdriver: startuvm: (%s) creating an SVM scratch", context) + if err := d.config.CreateSandbox(d.svmScratchSpaceFile, client.DefaultSandboxSizeMB, d.cachedSandboxFile); err != nil { + return fmt.Errorf("failed to create SVM scratch VHDX (%s): %s", context, err) + } + logrus.Debugf("lcowdriver: startuvm: (%s) hot-adding an SVM scratch", context) + if err := d.config.HotAddVhd(d.svmScratchSpaceFile, toolsScratchPath); err != nil { + return fmt.Errorf("failed to hot-add %s failed: %s", d.svmScratchSpaceFile, err) + } + } + logrus.Debugf("lcowdriver: startuvm: (%s) successful", context) return nil } From 9a5747b3f93b5d8e175c42daae0d5b8751fc5bcf Mon Sep 17 00:00:00 2001 From: John Howard Date: Mon, 26 Jun 2017 09:11:54 -0700 Subject: [PATCH 08/22] LCOW: Fix nits from 33241 Signed-off-by: John Howard Upstream-commit: 4ec9766a27ffb964cec6b8b2745725965b2644aa Component: engine --- components/engine/api/server/backend/build/tag.go | 2 +- .../engine/api/server/router/image/image_routes.go | 4 ++-- components/engine/builder/dockerfile/evaluator.go | 2 +- components/engine/builder/dockerfile/parser/parser.go | 10 +++++----- components/engine/container/container.go | 7 +++---- .../daemon/cluster/executor/container/adapter.go | 2 +- components/engine/daemon/create.go | 4 ++-- components/engine/daemon/graphdriver/lcow/lcow.go | 3 +-- .../engine/daemon/graphdriver/windows/windows.go | 2 +- components/engine/daemon/image_exporter.go | 4 ++-- components/engine/daemon/image_push.go | 2 +- components/engine/daemon/images.go | 4 ++-- components/engine/daemon/info.go | 2 +- components/engine/daemon/prune.go | 2 +- components/engine/distribution/pull_v2.go | 2 +- components/engine/image/store.go | 4 ++-- components/engine/layer/layer_store.go | 3 +-- components/engine/plugin/manager.go | 2 +- 18 files changed, 29 insertions(+), 32 deletions(-) diff --git a/components/engine/api/server/backend/build/tag.go b/components/engine/api/server/backend/build/tag.go index e2de4c13bb..7bd5dcdeb2 100644 --- a/components/engine/api/server/backend/build/tag.go +++ b/components/engine/api/server/backend/build/tag.go @@ -37,7 +37,7 @@ func (bt *Tagger) TagImages(imageID image.ID) error { for _, rt := range bt.repoAndTags { // TODO @jhowardmsft LCOW support. Will need revisiting. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } if err := bt.imageComponent.TagImageWithReference(imageID, platform, rt); err != nil { diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index 232d795683..9b99a585f3 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -101,7 +101,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite // } // valid := []string{runtime.GOOS} // - // if runtime.GOOS == "windows" && system.LCOWSupported() { + // if system.LCOWSupported() { // valid = append(valid, "linux") // } // @@ -118,7 +118,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite // return err // } platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/builder/dockerfile/evaluator.go b/components/engine/builder/dockerfile/evaluator.go index 69e3f6d0f2..ba4315940a 100644 --- a/components/engine/builder/dockerfile/evaluator.go +++ b/components/engine/builder/dockerfile/evaluator.go @@ -233,7 +233,7 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) { func (s *dispatchState) setDefaultPath() { // TODO @jhowardmsft LCOW Support - This will need revisiting later platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } if system.DefaultPathEnv(platform) == "" { diff --git a/components/engine/builder/dockerfile/parser/parser.go b/components/engine/builder/dockerfile/parser/parser.go index 9766fbfdfe..7f07ff2150 100644 --- a/components/engine/builder/dockerfile/parser/parser.go +++ b/components/engine/builder/dockerfile/parser/parser.go @@ -91,8 +91,8 @@ var ( // DefaultEscapeToken is the default escape token const DefaultEscapeToken = '\\' -// DefaultPlatformToken is the platform assumed for the build if not explicitly provided -var DefaultPlatformToken = runtime.GOOS +// defaultPlatformToken is the platform assumed for the build if not explicitly provided +var defaultPlatformToken = runtime.GOOS // Directive is the structure used during a build run to hold the state of // parsing directives. @@ -119,7 +119,7 @@ func (d *Directive) setEscapeToken(s string) error { func (d *Directive) setPlatformToken(s string) error { s = strings.ToLower(s) valid := []string{runtime.GOOS} - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { valid = append(valid, "linux") } for _, item := range valid { @@ -154,7 +154,7 @@ func (d *Directive) possibleParserDirective(line string) error { // TODO @jhowardmsft LCOW Support: Eventually this check can be removed, // but only recognise a platform token if running in LCOW mode. - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line)) if len(tpcMatch) != 0 { for i, n := range tokenPlatformCommand.SubexpNames() { @@ -177,7 +177,7 @@ func (d *Directive) possibleParserDirective(line string) error { func NewDefaultDirective() *Directive { directive := Directive{} directive.setEscapeToken(string(DefaultEscapeToken)) - directive.setPlatformToken(runtime.GOOS) + directive.setPlatformToken(defaultPlatformToken) return &directive } diff --git a/components/engine/container/container.go b/components/engine/container/container.go index fe050bcd17..854850f4fb 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -1038,7 +1038,7 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string platform = runtime.GOOS } env := []string{} - if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && system.LCOWSupported() && platform == "linux") { + if runtime.GOOS != "windows" || (system.LCOWSupported() && platform == "linux") { env = []string{ "PATH=" + system.DefaultPathEnv(platform), "HOSTNAME=" + container.Config.Hostname, @@ -1052,7 +1052,6 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string // because the env on the container can override certain default values // we need to replace the 'env' keys where they match and append anything // else. - //return ReplaceOrAppendEnvValues(linkedEnv, container.Config.Env) - foo := ReplaceOrAppendEnvValues(env, container.Config.Env) - return foo + env = ReplaceOrAppendEnvValues(env, container.Config.Env) + return env } diff --git a/components/engine/daemon/cluster/executor/container/adapter.go b/components/engine/daemon/cluster/executor/container/adapter.go index 75a085ef1d..92e4947e65 100644 --- a/components/engine/daemon/cluster/executor/container/adapter.go +++ b/components/engine/daemon/cluster/executor/container/adapter.go @@ -93,7 +93,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { // TODO @jhowardmsft LCOW Support: This will need revisiting as // the stack is built up to include LCOW support for swarm. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } err := c.backend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index d47de31fa8..78070fd29d 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -82,7 +82,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( if params.Platform == "" { params.Platform = runtime.GOOS } - if params.Platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { params.Platform = "linux" } @@ -106,7 +106,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( if img != nil { if params.Platform != img.Platform() { // Ignore this in LCOW mode. @jhowardmsft TODO - This will need revisiting later. - if !(runtime.GOOS == "windows" && system.LCOWSupported()) { + if !system.LCOWSupported() { return nil, fmt.Errorf("cannot create a %s container from a %s image", params.Platform, img.Platform()) } } diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index 6bf491574a..379b7c37ce 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -118,7 +118,7 @@ func (d *Driver) terminateUvm(context string) error { // FIXME: @jhowardmsft // This isn't thread-safe yet, but will change anyway with the lifetime - // changes and multiple instances. Defering that work for now. + // changes and multiple instances. Deferring that work for now. uvm := d.config.Uvm d.config.Uvm = nil @@ -190,7 +190,6 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { // Make sure layers are created with the correct ACL so that VMs can access them. layerPath := d.dir(id) logrus.Debugf("lcowdriver: create: id %s: creating layerPath %s", id, layerPath) - // Make sure the layers are created with the correct ACL so that VMs can access them. if err := system.MkdirAllWithACL(layerPath, 755, system.SddlNtvmAdministratorsLocalSystem); err != nil { return err } diff --git a/components/engine/daemon/graphdriver/windows/windows.go b/components/engine/daemon/graphdriver/windows/windows.go index a722078e35..514650aa7c 100644 --- a/components/engine/daemon/graphdriver/windows/windows.go +++ b/components/engine/daemon/graphdriver/windows/windows.go @@ -154,7 +154,7 @@ func (d *Driver) Status() [][2]string { } // panicIfUsedByLcow does exactly what it says. -// TODO @jhowardmsft - this is an temporary measure for the bring-up of +// TODO @jhowardmsft - this is a temporary measure for the bring-up of // Linux containers on Windows. It is a failsafe to ensure that the right // graphdriver is used. func panicIfUsedByLcow() { diff --git a/components/engine/daemon/image_exporter.go b/components/engine/daemon/image_exporter.go index edac775d43..a7b0be64c2 100644 --- a/components/engine/daemon/image_exporter.go +++ b/components/engine/daemon/image_exporter.go @@ -16,7 +16,7 @@ import ( func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { // TODO @jhowardmsft LCOW. This will need revisiting later. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) @@ -29,7 +29,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { // TODO @jhowardmsft LCOW. This will need revisiting later. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) diff --git a/components/engine/daemon/image_push.go b/components/engine/daemon/image_push.go index b1d15454fb..c2e5967b19 100644 --- a/components/engine/daemon/image_push.go +++ b/components/engine/daemon/image_push.go @@ -43,7 +43,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead // TODO @jhowardmsft LCOW Support. This will require revisiting. For now, hard-code. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/daemon/images.go b/components/engine/daemon/images.go index 6e29cae4d0..4baf703715 100644 --- a/components/engine/daemon/images.go +++ b/components/engine/daemon/images.go @@ -38,7 +38,7 @@ func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } func (daemon *Daemon) Map() map[image.ID]*image.Image { // TODO @jhowardmsft LCOW. This will need work to enumerate the stores for all platforms. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } return daemon.stores[platform].imageStore.Map() @@ -53,7 +53,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs // TODO @jhowardmsft LCOW. This will need work to enumerate the stores for all platforms. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index a11775c2e9..2cb3e84798 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -90,7 +90,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { // TODO @jhowardmsft LCOW support. For now, hard-code the platform shown for the driver status p := runtime.GOOS - if p == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { p = "linux" } diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index 561ac1e01e..1d8686b36f 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -161,7 +161,7 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { // TODO @jhowardmsft LCOW Support: This will need revisiting later. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/distribution/pull_v2.go b/components/engine/distribution/pull_v2.go index 0f3f31cf52..50257f5cb4 100644 --- a/components/engine/distribution/pull_v2.go +++ b/components/engine/distribution/pull_v2.go @@ -491,7 +491,7 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif // the history does, but unfortunately that's a string, so search through // all the history until hopefully we find one which indicates the os. platform := runtime.GOOS - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { type config struct { Os string `json:"os,omitempty"` } diff --git a/components/engine/image/store.go b/components/engine/image/store.go index 541b07bba1..97eb360011 100644 --- a/components/engine/image/store.go +++ b/components/engine/image/store.go @@ -3,7 +3,6 @@ package image import ( "encoding/json" "fmt" - "runtime" "strings" "sync" @@ -116,8 +115,9 @@ func (is *store) Create(config []byte) (ID, error) { return "", err } + // TODO @jhowardmsft - LCOW Support. This will need revisiting. // Integrity check - ensure we are creating something for the correct platform - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { if strings.ToLower(img.Platform()) != strings.ToLower(is.platform) { return "", fmt.Errorf("cannot create entry for platform %q in image store for platform %q", img.Platform(), is.platform) } diff --git a/components/engine/layer/layer_store.go b/components/engine/layer/layer_store.go index e7c424aa92..75ac1e4f48 100644 --- a/components/engine/layer/layer_store.go +++ b/components/engine/layer/layer_store.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "runtime" "strings" "sync" @@ -273,7 +272,7 @@ func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, platf var p *roLayer // Integrity check - ensure we are creating something for the correct platform - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { if strings.ToLower(ls.platform) != strings.ToLower(string(platform)) { return nil, fmt.Errorf("cannot create entry for platform %q in layer store for platform %q", platform, ls.platform) } diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index aa090f5a90..02d372741f 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -353,7 +353,7 @@ func isEqualPrivilege(a, b types.PluginPrivilege) bool { func configToRootFS(c []byte) (*image.RootFS, layer.Platform, error) { // TODO @jhowardmsft LCOW - Will need to revisit this. For now, calculate the platform. platform := layer.Platform(runtime.GOOS) - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } var pluginConfig types.PluginConfig From 7a07cf4db407af949e560b8db8a7181b05b1e4f2 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 28 Jun 2017 03:18:55 +0000 Subject: [PATCH 09/22] Fix build error for `make test-unit` This fix tries to address the issue raised by #33856 where `make test-unit` will result in the failure: ``` ... ... dockerversion/useragent.go:20: undefined: Version dockerversion/useragent.go:22: undefined: GitCommit ok github.com/docker/docker/api 0.026s coverage: 68.0% of statements ok github.com/docker/docker/api/errors 0.003s coverage: 100.0% of statements FAIL github.com/docker/docker/api/server [build failed] make: *** [test-unit] Error 2 ``` The issue is because in case of `make test-unit`, `source "${MAKEDIR}/.go-autogen"` is missing. This caused the `make test-unit` failure. This fix adds `source "${MAKEDIR}/.go-autogen"` in `hack/make/test-unit` This fix fixes #33856. Signed-off-by: Yong Tang Upstream-commit: fac486d0a02d6af2759f6d0e81bd8a23d82d432c Component: engine --- components/engine/hack/make/test-unit | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/hack/make/test-unit b/components/engine/hack/make/test-unit index d3f182c094..85eef5b5b6 100644 --- a/components/engine/hack/make/test-unit +++ b/components/engine/hack/make/test-unit @@ -26,6 +26,8 @@ bundle_test_unit() { TEST_PATH=./${TESTDIRS} fi + source "${MAKEDIR}/.go-autogen" + if [ "$(go env GOHOSTOS)" = 'solaris' ]; then pkg_list=$(go list -e \ -f '{{if ne .Name "github.com/docker/docker"}} From 30ba67aee013b62128e5281dff03c7bc97168626 Mon Sep 17 00:00:00 2001 From: Nishant Totla Date: Wed, 28 Jun 2017 15:02:54 -0700 Subject: [PATCH 10/22] Do not add duplicate platform information to service spec Signed-off-by: Nishant Totla Upstream-commit: da85b62001ec390bdc30b321c4a5cd23c17eeb81 Component: engine --- components/engine/client/service_create.go | 12 +++++++----- components/engine/client/service_update.go | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/components/engine/client/service_create.go b/components/engine/client/service_create.go index 71a7583e86..c2fc2776a8 100644 --- a/components/engine/client/service_create.go +++ b/components/engine/client/service_create.go @@ -39,7 +39,7 @@ func (cli *Client) ServiceCreate(ctx context.Context, service swarm.ServiceSpec, service.TaskTemplate.ContainerSpec.Image = img } // add platforms that are compatible with the service - service.TaskTemplate.Placement = updateServicePlatforms(service.TaskTemplate.Placement, distributionInspect) + service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect) } } var response types.ServiceCreateResponse @@ -86,13 +86,15 @@ func imageWithTagString(image string) string { return "" } -// updateServicePlatforms updates the Platforms in swarm.Placement to list -// all compatible platforms for the service, as found in distributionInspect -// and returns a pointer to the new or updated swarm.Placement struct -func updateServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement { +// setServicePlatforms sets Platforms in swarm.Placement to list all +// compatible platforms for the service, as found in distributionInspect +// and returns a pointer to the new or updated swarm.Placement struct. +func setServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement { if placement == nil { placement = &swarm.Placement{} } + // reset any existing listed platforms + placement.Platforms = []swarm.Platform{} for _, p := range distributionInspect.Platforms { placement.Platforms = append(placement.Platforms, swarm.Platform{ Architecture: p.Architecture, diff --git a/components/engine/client/service_update.go b/components/engine/client/service_update.go index 412790b5dd..a72adc997c 100644 --- a/components/engine/client/service_update.go +++ b/components/engine/client/service_update.go @@ -51,7 +51,7 @@ func (cli *Client) ServiceUpdate(ctx context.Context, serviceID string, version service.TaskTemplate.ContainerSpec.Image = img } // add platforms that are compatible with the service - service.TaskTemplate.Placement = updateServicePlatforms(service.TaskTemplate.Placement, distributionInspect) + service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect) } } From d76f67a9f8795b32365e4626e45d616317c005ae Mon Sep 17 00:00:00 2001 From: lixiaobing10051267 Date: Wed, 28 Jun 2017 14:49:12 +0800 Subject: [PATCH 11/22] add f.Close() after dockerignore.ReadAll(f) before return err Signed-off-by: lixiaobing10051267 Signed-off-by: Sebastiaan van Stijn Upstream-commit: 3072fce0671c5529a42b9e40a5344cce99318312 Component: engine --- components/engine/builder/remotecontext/detect.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/builder/remotecontext/detect.go b/components/engine/builder/remotecontext/detect.go index 22ad5b7c9f..4345736805 100644 --- a/components/engine/builder/remotecontext/detect.go +++ b/components/engine/builder/remotecontext/detect.go @@ -131,6 +131,7 @@ func removeDockerfile(c modifiableContext, filesToRemove ...string) error { } excludes, err := dockerignore.ReadAll(f) if err != nil { + f.Close() return err } f.Close() From 53df97e8813e5c270181450da476da031af637e5 Mon Sep 17 00:00:00 2001 From: lixiaobing10051267 Date: Thu, 29 Jun 2017 17:10:02 +0800 Subject: [PATCH 12/22] update url for issue Signed-off-by: lixiaobing10051267 Upstream-commit: 6fcae83036970dd2aad73127da6cbd0d2becb333 Component: engine --- components/engine/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index abf6c45a3c..917214cd1b 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -39,7 +39,7 @@ A great way to contribute to the project is to send a detailed report when you encounter an issue. We always appreciate a well-written, thorough bug report, and will thank you for it! -Check that [our issue database](https://github.com/docker/docker/issues) +Check that [our issue database](https://github.com/moby/moby/issues) doesn't already include that problem or suggestion before submitting an issue. If you find a match, you can use the "subscribe" button to get notified on updates. Do *not* leave random "+1" or "I have this too" comments, as they @@ -66,7 +66,7 @@ This section gives the experienced contributor some tips and guidelines. Not sure if that typo is worth a pull request? Found a bug and know how to fix it? Do it! We will appreciate it. Any significant improvement should be -documented as [a GitHub issue](https://github.com/docker/docker/issues) before +documented as [a GitHub issue](https://github.com/moby/moby/issues) before anybody starts working on it. We are always thrilled to receive pull requests. We do our best to process them From 0b353a620424d3e7e3275aa973effd0bb6e9d7eb Mon Sep 17 00:00:00 2001 From: Vitaly Ostrosablin Date: Thu, 29 Jun 2017 12:38:13 +0300 Subject: [PATCH 13/22] Change order of arguments in assertion to be more logical Because of argument order to assert, in case of fail, we'll get message like: ... obtained string = "uid=100(test) gid=101(test)\n" ... expected string = "uid=100(test) gid=101(test) groups=101(test)\n" But obtained string should be second one, not the first one. And expected string should be first. Here's sample output from test above: ... Output: before restart 'uid=100(test) gid=101(test) ', after restart 'uid=100(test) gid=101(test) groups=101(test) It's confusing, because expected order of strings in assertion is reversed. What goes before restart - should be "expected string" and what goes after - should be "obtained string". Signed-off-by: Vitaly Ostrosablin Upstream-commit: 07cc701947533089fac5fea42e7d5a9a6ea1cfcc Component: engine --- components/engine/integration-cli/docker_cli_daemon_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/integration-cli/docker_cli_daemon_test.go b/components/engine/integration-cli/docker_cli_daemon_test.go index b7f8f0a542..525ff4b767 100644 --- a/components/engine/integration-cli/docker_cli_daemon_test.go +++ b/components/engine/integration-cli/docker_cli_daemon_test.go @@ -2797,7 +2797,7 @@ func (s *DockerDaemonSuite) TestExecWithUserAfterLiveRestore(c *check.C) { out2, err := s.d.Cmd("exec", "-u", "test", "top", "id") c.Assert(err, check.IsNil, check.Commentf("Output: %s", out2)) - c.Assert(out1, check.Equals, out2, check.Commentf("Output: before restart '%s', after restart '%s'", out1, out2)) + c.Assert(out2, check.Equals, out1, check.Commentf("Output: before restart '%s', after restart '%s'", out1, out2)) out, err = s.d.Cmd("stop", "top") c.Assert(err, check.IsNil, check.Commentf("Output: %s", out)) From 85421deb82d1a602249e01cfaaa9e7adf4737c5d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 26 Jun 2017 12:06:34 -0400 Subject: [PATCH 14/22] Set ping version even on error In some cases a server may return an error on the ping response but still provide version details. The client should use these values when available. Signed-off-by: Brian Goff Upstream-commit: 27ef09a46ffeb8ba42548de937b68351009f30ea Component: engine --- components/engine/client/ping.go | 14 +++-- components/engine/client/ping_test.go | 82 +++++++++++++++++++++++++++ components/engine/client/request.go | 77 +++++++++++++++---------- 3 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 components/engine/client/ping_test.go diff --git a/components/engine/client/ping.go b/components/engine/client/ping.go index 631ed02005..a4c2e2c4dd 100644 --- a/components/engine/client/ping.go +++ b/components/engine/client/ping.go @@ -18,13 +18,15 @@ func (cli *Client) Ping(ctx context.Context) (types.Ping, error) { } defer ensureReaderClosed(serverResp) - ping.APIVersion = serverResp.header.Get("API-Version") + if serverResp.header != nil { + ping.APIVersion = serverResp.header.Get("API-Version") - if serverResp.header.Get("Docker-Experimental") == "true" { - ping.Experimental = true + if serverResp.header.Get("Docker-Experimental") == "true" { + ping.Experimental = true + } + ping.OSType = serverResp.header.Get("OSType") } - ping.OSType = serverResp.header.Get("OSType") - - return ping, nil + err = cli.checkResponseErr(serverResp) + return ping, err } diff --git a/components/engine/client/ping_test.go b/components/engine/client/ping_test.go new file mode 100644 index 0000000000..7a4a1a9024 --- /dev/null +++ b/components/engine/client/ping_test.go @@ -0,0 +1,82 @@ +package client + +import ( + "errors" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +// TestPingFail tests that when a server sends a non-successful response that we +// can still grab API details, when set. +// Some of this is just excercising the code paths to make sure there are no +// panics. +func TestPingFail(t *testing.T) { + var withHeader bool + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + if withHeader { + resp.Header = http.Header{} + resp.Header.Set("API-Version", "awesome") + resp.Header.Set("Docker-Experimental", "true") + } + resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server")) + return resp, nil + }), + } + + ping, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, false, ping.Experimental) + assert.Equal(t, "", ping.APIVersion) + + withHeader = true + ping2, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, true, ping2.Experimental) + assert.Equal(t, "awesome", ping2.APIVersion) +} + +// TestPingWithError tests the case where there is a protocol error in the ping. +// This test is mostly just testing that there are no panics in this code path. +func TestPingWithError(t *testing.T) { + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + resp.Header = http.Header{} + resp.Header.Set("API-Version", "awesome") + resp.Header.Set("Docker-Experimental", "true") + resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server")) + return resp, errors.New("some error") + }), + } + + ping, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, false, ping.Experimental) + assert.Equal(t, "", ping.APIVersion) +} + +// TestPingSuccess tests that we are able to get the expected API headers/ping +// details on success. +func TestPingSuccess(t *testing.T) { + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + resp.Header = http.Header{} + resp.Header.Set("API-Version", "awesome") + resp.Header.Set("Docker-Experimental", "true") + resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server")) + return resp, nil + }), + } + ping, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, true, ping.Experimental) + assert.Equal(t, "awesome", ping.APIVersion) +} diff --git a/components/engine/client/request.go b/components/engine/client/request.go index 6457b316a3..3e7d43feac 100644 --- a/components/engine/client/request.go +++ b/components/engine/client/request.go @@ -24,6 +24,7 @@ type serverResponse struct { body io.ReadCloser header http.Header statusCode int + reqURL *url.URL } // head sends an http request to the docker API using the method HEAD. @@ -118,11 +119,18 @@ func (cli *Client) sendRequest(ctx context.Context, method, path string, query u if err != nil { return serverResponse{}, err } - return cli.doRequest(ctx, req) + resp, err := cli.doRequest(ctx, req) + if err != nil { + return resp, err + } + if err := cli.checkResponseErr(resp); err != nil { + return resp, err + } + return resp, nil } func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResponse, error) { - serverResp := serverResponse{statusCode: -1} + serverResp := serverResponse{statusCode: -1, reqURL: req.URL} resp, err := ctxhttp.Do(ctx, cli.client, req) if err != nil { @@ -179,37 +187,44 @@ func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResp if resp != nil { serverResp.statusCode = resp.StatusCode + serverResp.body = resp.Body + serverResp.header = resp.Header } - - if serverResp.statusCode < 200 || serverResp.statusCode >= 400 { - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return serverResp, err - } - if len(body) == 0 { - return serverResp, fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), req.URL) - } - - var errorMessage string - if (cli.version == "" || versions.GreaterThan(cli.version, "1.23")) && - resp.Header.Get("Content-Type") == "application/json" { - var errorResponse types.ErrorResponse - if err := json.Unmarshal(body, &errorResponse); err != nil { - return serverResp, fmt.Errorf("Error reading JSON: %v", err) - } - errorMessage = errorResponse.Message - } else { - errorMessage = string(body) - } - - return serverResp, fmt.Errorf("Error response from daemon: %s", strings.TrimSpace(errorMessage)) - } - - serverResp.body = resp.Body - serverResp.header = resp.Header return serverResp, nil } +func (cli *Client) checkResponseErr(serverResp serverResponse) error { + if serverResp.statusCode >= 200 && serverResp.statusCode < 400 { + return nil + } + + body, err := ioutil.ReadAll(serverResp.body) + if err != nil { + return err + } + if len(body) == 0 { + return fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), serverResp.reqURL) + } + + var ct string + if serverResp.header != nil { + ct = serverResp.header.Get("Content-Type") + } + + var errorMessage string + if (cli.version == "" || versions.GreaterThan(cli.version, "1.23")) && ct == "application/json" { + var errorResponse types.ErrorResponse + if err := json.Unmarshal(body, &errorResponse); err != nil { + return fmt.Errorf("Error reading JSON: %v", err) + } + errorMessage = errorResponse.Message + } else { + errorMessage = string(body) + } + + return fmt.Errorf("Error response from daemon: %s", strings.TrimSpace(errorMessage)) +} + func (cli *Client) addHeaders(req *http.Request, headers headers) *http.Request { // Add CLI Config's HTTP Headers BEFORE we set the Docker headers // then the user can't change OUR headers @@ -239,9 +254,9 @@ func encodeData(data interface{}) (*bytes.Buffer, error) { } func ensureReaderClosed(response serverResponse) { - if body := response.body; body != nil { + if response.body != nil { // Drain up to 512 bytes and close the body to let the Transport reuse the connection - io.CopyN(ioutil.Discard, body, 512) + io.CopyN(ioutil.Discard, response.body, 512) response.body.Close() } } From 60185c5eb39da0443c642f7d021f04a5db734405 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 29 Jun 2017 15:02:46 -0700 Subject: [PATCH 15/22] container: Handle failed memdb lookups If a container doesn't exist in the memdb, First will return nil, not an error. This should be checked for before using the result. Signed-off-by: Aaron Lehmann Upstream-commit: c26b0cdfd1a026af88fcfbed9d3c3acdd6d171a0 Component: engine --- components/engine/container/view.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/components/engine/container/view.go b/components/engine/container/view.go index 13c7161852..f605b4f483 100644 --- a/components/engine/container/view.go +++ b/components/engine/container/view.go @@ -73,6 +73,17 @@ type memDB struct { store *memdb.MemDB } +// NoSuchContainerError indicates that the container wasn't found in the +// database. +type NoSuchContainerError struct { + id string +} + +// Error satisfies the error interface. +func (e NoSuchContainerError) Error() string { + return "no such container " + e.id +} + // NewViewDB provides the default implementation, with the default schema func NewViewDB() (ViewDB, error) { store, err := memdb.NewMemDB(schema) @@ -134,6 +145,9 @@ func (v *memdbView) Get(id string) (*Snapshot, error) { if err != nil { return nil, err } + if s == nil { + return nil, NoSuchContainerError{id: id} + } return v.transform(s.(*Container)), nil } From 1a311235bab8cd31f8b3a63b03c34097f1d52818 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Jun 2017 22:24:49 -0700 Subject: [PATCH 16/22] Always perform version-negotiation If a client is initialized without a specific version set, version negotiation would not be functional. This patch changes the behavior to always perform version negotation (if called), in which case the "current" (maximum supported API version) is used as a default. Signed-off-by: Sebastiaan van Stijn Upstream-commit: 5975dc4b4b8a6fa2e0edd1f1b113e7f66dad732b Component: engine --- components/engine/client/client.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/engine/client/client.go b/components/engine/client/client.go index 222aa50f11..62a3db360a 100644 --- a/components/engine/client/client.go +++ b/components/engine/client/client.go @@ -260,8 +260,13 @@ func (cli *Client) NegotiateAPIVersionPing(p types.Ping) { p.APIVersion = "1.24" } - // if server version is lower than the current cli, downgrade - if versions.LessThan(p.APIVersion, cli.ClientVersion()) { + // if the client is not initialized with a version, start with the latest supported version + if cli.version == "" { + cli.version = api.DefaultVersion + } + + // if server version is lower than the maximum version supported by the Client, downgrade + if versions.LessThan(p.APIVersion, api.DefaultVersion) { cli.version = p.APIVersion } } From 8d232fdb6812e5b53f57df506e42b6808e50d3c8 Mon Sep 17 00:00:00 2001 From: Boaz Shuster Date: Fri, 30 Jun 2017 11:04:35 +0300 Subject: [PATCH 17/22] Refactor MountPoint Setup function in volume.go Signed-off-by: Boaz Shuster Upstream-commit: fb8b27cd41c92c9a739be70dee11f6e6ffffafdf Component: engine --- components/engine/volume/volume.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/components/engine/volume/volume.go b/components/engine/volume/volume.go index 07176b89ce..2de248aa5f 100644 --- a/components/engine/volume/volume.go +++ b/components/engine/volume/volume.go @@ -153,20 +153,18 @@ func (m *MountPoint) Cleanup() error { // before creating the source directory on the host. func (m *MountPoint) Setup(mountLabel string, rootIDs idtools.IDPair, checkFun func(m *MountPoint) error) (path string, err error) { defer func() { - if err == nil { - if label.RelabelNeeded(m.Mode) { - if err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode)); err != nil { - if err == syscall.ENOTSUP { - err = nil - return - } - path = "" - err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source) - return - } - } + if err != nil || !label.RelabelNeeded(m.Mode) { + return + } + + err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode)) + if err == syscall.ENOTSUP { + err = nil + } + if err != nil { + path = "" + err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source) } - return }() if m.Volume != nil { From cccb3a210888cbfdbe91461e109591c507999075 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 27 Jun 2017 17:20:21 -0700 Subject: [PATCH 18/22] add dev report 2017-06-26 Signed-off-by: Victor Vieux Upstream-commit: 8b0384fd8918e82459aab037d50bca6394bbf781 Component: engine --- components/engine/reports/2017-06-26.md | 120 ++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 components/engine/reports/2017-06-26.md diff --git a/components/engine/reports/2017-06-26.md b/components/engine/reports/2017-06-26.md new file mode 100644 index 0000000000..e12533ae46 --- /dev/null +++ b/components/engine/reports/2017-06-26.md @@ -0,0 +1,120 @@ +# Development Report for June 26, 2017 + +## Moby Summit + +The Moby Summit held in San Francisco was very active and well attended ([blog](http://mobyproject.org/blog/2017/06/26/moby-summit-recap/) / [linuxkit table notes](https://github.com/linuxkit/linuxkit/blob/master/reports/2017-06-19-summit.md) [#2090](https://github.com/linuxkit/linuxkit/pull/2090) [#2033](https://github.com/linuxkit/linuxkit/pull/2033) [@mgoelzer] [@justincormack]). + +## Container Engine + +Thanks to @fabiokung there is no container locks anymore on `docker ps` [#31273](https://github.com/moby/moby/pull/31273) + +## BuildKit + +[Repo](https://github.com/moby/buildkit) +[Proposal](https://github.com/moby/moby/issues/32925) + +New development repo is open at https://github.com/moby/buildkit + +The readme file provides examples how to get started. You can see an example of building BuildKit with BuildKit. + +There are lots of new issues opened as well to track the missing functionality. You are welcomed to help on any of them or discuss the design there. + +Last week most of the work was done on improving the `llb` client library for more complicated use cases and providing traces and interactive progress of executed build jobs. + +The `llb` client package is a go library that helps you to generate the build definition graph. It uses chained methods to make it easy to describe what steps need to be running. Mounts can be added to the execution steps for defining multiple inputs or outputs. To prepare the graph, you just have to call `Marshal()` on a leaf node that will generate the protobuf definition for everything required to build that node. + +### Typed Dockerfile parsing + +[PR](https://github.com/moby/moby/pull/33492) + +This PR that enables parsing Dockerfiles into typed structures so they can be preprocessed to eliminate unnecessary build stages and reused with different kinds of dispatchers(eg. BuildKit). + +The PR had some review and updates in last week. Should be ready to code review soon. + +### Merged: Long running session & incremental file sending + +[PR](https://github.com/moby/moby/pull/32677) + +Incremental context sending PR was merged and is expected to land in `v17.07`. + +This feature experimental feature lets you skip sending the build context to the daemon on repeated builder invocations during development. Currently, this feature requires a CLI flag `--stream=true`. If this flag is used, one first builder invocation full build context is sent to the daemon. On a second attempt, only the changed files are transferred. + +Previous build context is saved in the build cache, and you can see how much space it takes form `docker system df`. Build cache will be automatically garbage collected and can also be manually cleared with `docker prune`. + +### Quality: Dependency interface switch + +[Move file copying from the daemon to the builder](https://github.com/moby/moby/pull/33454) PR was merged. + + +### Proposals for new Dockerfile features that need design feedback: + +[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100) + +[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487) + +[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507) + +[DAG image builder](https://github.com/moby/moby/issues/32550) + +[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new) + +[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new) + +[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989) + +If you are interested in implementing any of them, leave a comment on the specific issues. + +### Other builder PRs merged last week + +[Warn/deprecate continuing on empty lines in `Dockerfile`](https://github.com/moby/moby/pull/29161) + +[Fix behavior of absolute paths in .dockerignore](https://github.com/moby/moby/pull/32088) + +[fix copy —from conflict with force pull](https://github.com/moby/moby/pull/33735) + +### Builder features currently in code-review: + +[Fix handling of remote "git@" notation](https://github.com/moby/moby/pull/33696) + +[builder: Emit a BuildResult after squashing.](https://github.com/moby/moby/pull/33824) + +[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704) + +### Backlog + +[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard. + +## LinuxKit + +* **Kernel GPG verification:** The kernel compilation containers now verify the GPG and SHA256 + checksums before building the binaries. ([#2062](https://github.com/linuxkit/linuxkit/issues/2062) [#2083](https://github.com/linuxkit/linuxkit/issues/2083) [@mscribe] [@justincormack] [@rn] [@riyazdf]). + The base Alpine build image now includes `gnupg` to support this feature ([#2091](https://github.com/linuxkit/linuxkit/issues/2091) [@riyazdf] [@rn]). + +* **Security SIG on Landlock:** The third Moby Security SIG focussed on the [Landlock](https://github.com/landlock-lsm) security module that provides unprivileged fine-grained sandboxing to applications. There are videos and forum links ([#2087](https://github.com/linuxkit/linuxkit/issues/2087) [#2089](https://github.com/linuxkit/linuxkit/issues/2089) [#2073](https://github.com/linuxkit/linuxkit/issues/2073) [@riyazdf]). + +* **Networking drivers now modules:** The kernels have been updated to 4.11.6/4.9.33/4.4.73, and many drivers are now loaded as modules to speed up boot-time ([#2095](https://github.com/linuxkit/linuxkit/issues/2095) [#2061](https://github.com/linuxkit/linuxkit/issues/2061) [@rn] [@justincormack] [@tych0]) + +- **Whaley important update:** The ASCII logo was updated and we fondly wave goodbye to the waves. ([#2084](https://github.com/linuxkit/linuxkit/issues/2084) [@thaJeztah] [@rn]) + +- **Containerised getty and sshd:** The login services now run in their own mount namespace, which was confusing people since they were expecting it to be on the host filesystem. This is now being addressed via a reminder in the `motd` upon login ([#2078](https://github.com/linuxkit/linuxkit/issues/2078) [#2097](https://github.com/linuxkit/linuxkit/issues/2097) [@deitch] [@ijc] [@justincormack] [@riyazdf] [@rn]) + +- **Hardened user copying:** The RFC on ensuring that we use a hardened kernel/userspace copying system was closed, as it is enabled by default on all our modern kernels and a regression test is included by default ([#2086](https://github.com/linuxkit/linuxkit/issues/2086) [@fntlnz] [@riyazdf]). + +- **Vultr provider:** There is an ongoing effort to add a metadata provider for [Vultr](http://vultr.com) ([#2101](https://github.com/linuxkit/linuxkit/issues/2101) [@furious-luke] [@justincormack]). + +### Packages and Projects + +- Simplified Makefiles for packages ([#2080](https://github.com/linuxkit/linuxkit/issues/2080) [@justincormack] [@rn]) +- The MirageOS SDK is integrating many upstream changes from dependent libraries, for the DHCP client ([#2070](https://github.com/linuxkit/linuxkit/issues/2070) [#2072](https://github.com/linuxkit/linuxkit/issues/2072) [@samoht] [@talex5] [@avsm]). + +### Documentation and Tests + +- A comprehensive test suite for containerd is now integrated into LinuxKit tests ([#2062](https://github.com/linuxkit/linuxkit/issues/2062) [@AkihiroSuda] [@justincormack] [@rn]) +- Fix documentation links ([#2074](https://github.com/linuxkit/linuxkit/issues/2074) [@ndauten] [@justincormack]) +- Update RTF version ([#2077](https://github.com/linuxkit/linuxkit/issues/2077) [@justincormack]) +- tests: add build test for Docker for Mac blueprint ([#2093](https://github.com/linuxkit/linuxkit/issues/2093) [@riyazdf] [@MagnusS]) +- Disable Qemu EFI ISO test for now ([#2100](https://github.com/linuxkit/linuxkit/issues/2100) [@justincormack]) +- The CI whitelists and ACLs were updated ([linuxkit-ci#11](https://github.com/linuxkit/linuxkit-ce/issues/11) [linuxkit-ci#15](https://github.com/linuxkit/linuxkit-ce/issues/15) [linuxkit/linuxkit-ci#10](https://github.com/linuxkit/linuxkit-ce/issues/10) [@rn] [@justincormack]) +- Fix spelling errors ([#2079](https://github.com/linuxkit/linuxkit/issues/2079) [@ndauten]) +- Fix typo in dev report ([#2094](https://github.com/linuxkit/linuxkit/issues/2094) [@justincormack]) +- Fix dead Link to VMWare File ([#2082](https://github.com/linuxkit/linuxkit/issues/2082) [@davefreitag]) \ No newline at end of file From 27d204d88889d5255911fd657ffe734c520e6904 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Sat, 1 Jul 2017 04:54:51 +1000 Subject: [PATCH 19/22] make: add dynbinary target Recently, it's become necessary to build dynbinaries on certain distributions, so make it easier for users to build dynbinaries. It looks like this was just an oversight when we added dynbinary builds. Signed-off-by: Aleksa Sarai Upstream-commit: c81255c9c0314b81ab27f6afd7646cdd259e6495 Component: engine --- components/engine/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 526d6405ea..0d99606cc2 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win +.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win # set the graph driver as the current graphdriver if not set DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) @@ -106,6 +106,9 @@ all: build ## validate all checks, build linux binaries, run all tests\ncross bu binary: build ## build the linux binaries $(DOCKER_RUN_DOCKER) hack/make.sh binary +dynbinary: build ## build the linux dynbinaries + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary + build: bundles init-go-pkg-cache $(warning The docker client CLI has moved to github.com/docker/cli. By default, it is built from the git sha specified in hack/dockerfile/binaries-commits. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . From c5ade3e304b250eced8f32bd47c744f177974742 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 29 Jun 2017 15:25:17 -0700 Subject: [PATCH 20/22] daemon: Ignore nonexistent containers when listing containers The name/ID relationships are maintained separately from the memdb and can be out of sync from any particular memdb snapshot. If a container does not exist in the memdb, we must accept this as normal and not fail the listing. This is consistent with what the code used to do before memdb was introduced. Signed-off-by: Aaron Lehmann Signed-off-by: Sebastiaan van Stijn Upstream-commit: d257a63fb6a2ad0c2369981914544add389917bf Component: engine --- components/engine/daemon/list.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index 870a7bda2d..b854be7549 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -162,11 +162,13 @@ func (daemon *Daemon) filterByNameIDMatches(view container.View, ctx *listContex cntrs := make([]container.Snapshot, 0, len(matches)) for id := range matches { c, err := view.Get(id) - if err != nil { - return nil, err - } - if c != nil { + switch err.(type) { + case nil: cntrs = append(cntrs, *c) + case container.NoSuchContainerError: + // ignore error + default: + return nil, err } } From 9787d41c94338ecaebaa8ca7b42b84cb99a3aa3d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2017 00:19:15 -0700 Subject: [PATCH 21/22] Remove comment about forked golang.org/x/net This dependency was temporarily forked, but the fork was removed in db37a86d37431a1d82505cf6adc91a5d91dad146. This patch removes the comment Signed-off-by: Sebastiaan van Stijn Upstream-commit: fe877c2b959b60d953ad4b2d3eeda377add575a5 Component: engine --- components/engine/vendor.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index bad511c4d4..20460d5cc9 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -13,7 +13,6 @@ github.com/kr/pty 5cf931ef8f github.com/mattn/go-shellwords v1.0.3 github.com/tchap/go-patricia v2.2.6 github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3 -# forked golang.org/x/net package includes a patch for lazy loading trace templates golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1 From 3a59ac4d9f0ad42658e2d2ee115f7290392c19a0 Mon Sep 17 00:00:00 2001 From: Lei Jitang Date: Mon, 3 Jul 2017 03:31:34 -0400 Subject: [PATCH 22/22] Fix wrong filemode for rotate log files the filemode should be 0640 but not 06400 Signed-off-by: Lei Jitang Upstream-commit: 96ea8eaa1592856e0829c3465e64f93911d3e389 Component: engine --- components/engine/daemon/logger/loggerutils/rotatefilewriter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/logger/loggerutils/rotatefilewriter.go b/components/engine/daemon/logger/loggerutils/rotatefilewriter.go index ecb461fcec..457a39b5a3 100644 --- a/components/engine/daemon/logger/loggerutils/rotatefilewriter.go +++ b/components/engine/daemon/logger/loggerutils/rotatefilewriter.go @@ -74,7 +74,7 @@ func (w *RotateFileWriter) checkCapacityAndRotate() error { if err := rotate(name, w.maxFiles); err != nil { return err } - file, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 06400) + file, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0640) if err != nil { return err }