From e56d0513c658907ad3777a1677f3edfa68820ba2 Mon Sep 17 00:00:00 2001 From: Vlastimil Zeman Date: Tue, 21 Nov 2017 15:20:07 +0000 Subject: [PATCH 1/8] Add journald tag as SYSLOG_IDENTIFIER Signed-off-by: Vlastimil Zeman Upstream-commit: ae557f9d85f41e009229d1a33844f06f7ad1fb99 Component: engine --- components/engine/daemon/logger/journald/journald.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/daemon/logger/journald/journald.go b/components/engine/daemon/logger/journald/journald.go index 46aac8328b..ee6e5b68f7 100644 --- a/components/engine/daemon/logger/journald/journald.go +++ b/components/engine/daemon/logger/journald/journald.go @@ -75,6 +75,7 @@ func New(info logger.Info) (logger.Logger, error) { "CONTAINER_ID_FULL": info.ContainerID, "CONTAINER_NAME": info.Name(), "CONTAINER_TAG": tag, + "SYSLOG_IDENTIFIER": tag, } extraAttrs, err := info.ExtraAttributes(sanitizeKeyMod) if err != nil { From b8eda642a795cd25930c53a978f2cb677d10aa65 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 15 Nov 2017 09:46:32 -0500 Subject: [PATCH 2/8] Splunk: limit the reader size on error responses Before this patch, if a splunk endpoint returned a non-OK status, the splunk driver would read the entire response body. This could lead to excessive memory allocations. Instead put a cap on the amount the driver will read from the response body. Signed-off-by: Brian Goff Upstream-commit: 7f14542ddd1401de734be3ac0331c0ada941c959 Component: engine --- .../engine/daemon/logger/splunk/splunk.go | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/components/engine/daemon/logger/splunk/splunk.go b/components/engine/daemon/logger/splunk/splunk.go index eb80a0b526..30ab7b7414 100644 --- a/components/engine/daemon/logger/splunk/splunk.go +++ b/components/engine/daemon/logger/splunk/splunk.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" + "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/urlutil" "github.com/sirupsen/logrus" ) @@ -55,6 +56,8 @@ const ( defaultBufferMaximum = 10 * defaultPostMessagesBatchSize // Number of messages allowed to be queued in the channel defaultStreamChannelSize = 4 * defaultPostMessagesBatchSize + // maxResponseSize is the max amount that will be read from an http response + maxResponseSize = 1024 ) const ( @@ -500,20 +503,22 @@ func (l *splunkLogger) tryPostMessages(ctx context.Context, messages []*splunkMe if l.gzipCompression { req.Header.Set("Content-Encoding", "gzip") } - res, err := l.client.Do(req) + resp, err := l.client.Do(req) if err != nil { return err } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - var body []byte - body, err = ioutil.ReadAll(res.Body) + defer func() { + pools.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + if resp.StatusCode != http.StatusOK { + rdr := io.LimitReader(resp.Body, maxResponseSize) + body, err := ioutil.ReadAll(rdr) if err != nil { return err } - return fmt.Errorf("%s: failed to send event - %s - %s", driverName, res.Status, body) + return fmt.Errorf("%s: failed to send event - %s - %s", driverName, resp.Status, string(body)) } - io.Copy(ioutil.Discard, res.Body) return nil } @@ -596,20 +601,22 @@ func verifySplunkConnection(l *splunkLogger) error { if err != nil { return err } - res, err := l.client.Do(req) + resp, err := l.client.Do(req) if err != nil { return err } - if res.Body != nil { - defer res.Body.Close() - } - if res.StatusCode != http.StatusOK { - var body []byte - body, err = ioutil.ReadAll(res.Body) + defer func() { + pools.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + rdr := io.LimitReader(resp.Body, maxResponseSize) + body, err := ioutil.ReadAll(rdr) if err != nil { return err } - return fmt.Errorf("%s: failed to verify connection - %s - %s", driverName, res.Status, body) + return fmt.Errorf("%s: failed to verify connection - %s - %s", driverName, resp.Status, string(body)) } return nil } From 94fbfde561830b4f541214f80112e5cfd9b5395d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 7 Dec 2017 13:44:08 -0800 Subject: [PATCH 3/8] gitutils: fix checking out submodules Signed-off-by: Tonis Tiigi Upstream-commit: a6025255246aab21426a51986b7d138a14d9ac90 Component: engine --- .../builder/remotecontext/git/gitutils.go | 20 ++++- .../remotecontext/git/gitutils_test.go | 79 ++++++++++++++----- 2 files changed, 79 insertions(+), 20 deletions(-) diff --git a/components/engine/builder/remotecontext/git/gitutils.go b/components/engine/builder/remotecontext/git/gitutils.go index 7bc2268155..b579db9765 100644 --- a/components/engine/builder/remotecontext/git/gitutils.go +++ b/components/engine/builder/remotecontext/git/gitutils.go @@ -29,6 +29,10 @@ func Clone(remoteURL string) (string, error) { return "", err } + return cloneGitRepo(repo) +} + +func cloneGitRepo(repo gitRepo) (string, error) { fetch := fetchArgs(repo.remote, repo.ref) root, err := ioutil.TempDir("", "docker-build-git") @@ -50,7 +54,19 @@ func Clone(remoteURL string) (string, error) { return "", errors.Wrapf(err, "error fetching: %s", output) } - return checkoutGit(root, repo.ref, repo.subdir) + root, err = checkoutGit(root, repo.ref, repo.subdir) + if err != nil { + return "", err + } + + cmd := exec.Command("git", "submodule", "update", "--init", "--recursive", "--depth=1") + cmd.Dir = root + output, err := cmd.CombinedOutput() + if err != nil { + return "", errors.Wrapf(err, "error initializing submodules: %s", output) + } + + return root, nil } func parseRemoteURL(remoteURL string) (gitRepo, error) { @@ -96,7 +112,7 @@ func getRefAndSubdir(fragment string) (ref string, subdir string) { } func fetchArgs(remoteURL string, ref string) []string { - args := []string{"fetch", "--recurse-submodules=yes"} + args := []string{"fetch"} if supportsShallowClone(remoteURL) { args = append(args, "--depth", "1") diff --git a/components/engine/builder/remotecontext/git/gitutils_test.go b/components/engine/builder/remotecontext/git/gitutils_test.go index c638a498f2..b4c49195f0 100644 --- a/components/engine/builder/remotecontext/git/gitutils_test.go +++ b/components/engine/builder/remotecontext/git/gitutils_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -61,7 +62,7 @@ func TestCloneArgsSmartHttp(t *testing.T) { }) args := fetchArgs(serverURL.String(), "master") - exp := []string{"fetch", "--recurse-submodules=yes", "--depth", "1", "origin", "master"} + exp := []string{"fetch", "--depth", "1", "origin", "master"} assert.Equal(t, exp, args) } @@ -77,13 +78,13 @@ func TestCloneArgsDumbHttp(t *testing.T) { }) args := fetchArgs(serverURL.String(), "master") - exp := []string{"fetch", "--recurse-submodules=yes", "origin", "master"} + exp := []string{"fetch", "origin", "master"} assert.Equal(t, exp, args) } func TestCloneArgsGit(t *testing.T) { args := fetchArgs("git://github.com/docker/docker", "master") - exp := []string{"fetch", "--recurse-submodules=yes", "--depth", "1", "origin", "master"} + exp := []string{"fetch", "--depth", "1", "origin", "master"} assert.Equal(t, exp, args) } @@ -165,24 +166,55 @@ func TestCheckoutGit(t *testing.T) { _, err = gitWithinDir(gitDir, "checkout", "master") require.NoError(t, err) + // set up submodule + subrepoDir := filepath.Join(root, "subrepo") + _, err = git("init", subrepoDir) + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "config", "user.email", "test@docker.com") + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "config", "user.name", "Docker test") + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(subrepoDir, "subfile"), []byte("subcontents"), 0644) + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "add", "-A") + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "commit", "-am", "Subrepo initial") + require.NoError(t, err) + + cmd := exec.Command("git", "submodule", "add", subrepoDir, "sub") // this command doesn't work with --work-tree + cmd.Dir = gitDir + require.NoError(t, cmd.Run()) + + _, err = gitWithinDir(gitDir, "add", "-A") + require.NoError(t, err) + + _, err = gitWithinDir(gitDir, "commit", "-am", "With submodule") + require.NoError(t, err) + type singleCase struct { - frag string - exp string - fail bool + frag string + exp string + fail bool + submodule bool } cases := []singleCase{ - {"", "FROM scratch", false}, - {"master", "FROM scratch", false}, - {":subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, - {":nosubdir", "", true}, // missing directory error - {":Dockerfile", "", true}, // not a directory error - {"master:nosubdir", "", true}, - {"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, - {"master:../subdir", "", true}, - {"test", "FROM scratch" + eol + "EXPOSE 3000", false}, - {"test:", "FROM scratch" + eol + "EXPOSE 3000", false}, - {"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false}, + {"", "FROM scratch", false, true}, + {"master", "FROM scratch", false, true}, + {":subdir", "FROM scratch" + eol + "EXPOSE 5000", false, false}, + {":nosubdir", "", true, false}, // missing directory error + {":Dockerfile", "", true, false}, // not a directory error + {"master:nosubdir", "", true, false}, + {"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false, false}, + {"master:../subdir", "", true, false}, + {"test", "FROM scratch" + eol + "EXPOSE 3000", false, false}, + {"test:", "FROM scratch" + eol + "EXPOSE 3000", false, false}, + {"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false, false}, } if runtime.GOOS != "windows" { @@ -197,11 +229,22 @@ func TestCheckoutGit(t *testing.T) { for _, c := range cases { ref, subdir := getRefAndSubdir(c.frag) - r, err := checkoutGit(gitDir, ref, subdir) + r, err := cloneGitRepo(gitRepo{remote: gitDir, ref: ref, subdir: subdir}) if c.fail { assert.Error(t, err) continue + } else { + require.NoError(t, err) + if c.submodule { + b, err := ioutil.ReadFile(filepath.Join(r, "sub/subfile")) + require.NoError(t, err) + assert.Equal(t, "subcontents", string(b)) + } else { + _, err := os.Stat(filepath.Join(r, "sub/subfile")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + } } b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile")) From 5b48c4106b1d85e0271f8c3b3ec81ba7bd4cbcac Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 8 Dec 2017 11:51:10 -0800 Subject: [PATCH 4/8] gitutils: remove checkout directory on error Signed-off-by: Tonis Tiigi Upstream-commit: 4ea320cb8ef71c2fc9ee391e2e8be915ba99b17e Component: engine --- .../builder/remotecontext/git/gitutils.go | 12 ++++++++--- .../remotecontext/git/gitutils_test.go | 20 +++++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/components/engine/builder/remotecontext/git/gitutils.go b/components/engine/builder/remotecontext/git/gitutils.go index b579db9765..67cff594ab 100644 --- a/components/engine/builder/remotecontext/git/gitutils.go +++ b/components/engine/builder/remotecontext/git/gitutils.go @@ -32,7 +32,7 @@ func Clone(remoteURL string) (string, error) { return cloneGitRepo(repo) } -func cloneGitRepo(repo gitRepo) (string, error) { +func cloneGitRepo(repo gitRepo) (checkoutDir string, err error) { fetch := fetchArgs(repo.remote, repo.ref) root, err := ioutil.TempDir("", "docker-build-git") @@ -40,6 +40,12 @@ func cloneGitRepo(repo gitRepo) (string, error) { return "", err } + defer func() { + if err != nil { + os.RemoveAll(root) + } + }() + if out, err := gitWithinDir(root, "init"); err != nil { return "", errors.Wrapf(err, "failed to init repo at %s: %s", root, out) } @@ -54,7 +60,7 @@ func cloneGitRepo(repo gitRepo) (string, error) { return "", errors.Wrapf(err, "error fetching: %s", output) } - root, err = checkoutGit(root, repo.ref, repo.subdir) + checkoutDir, err = checkoutGit(root, repo.ref, repo.subdir) if err != nil { return "", err } @@ -66,7 +72,7 @@ func cloneGitRepo(repo gitRepo) (string, error) { return "", errors.Wrapf(err, "error initializing submodules: %s", output) } - return root, nil + return checkoutDir, nil } func parseRemoteURL(remoteURL string) (gitRepo, error) { diff --git a/components/engine/builder/remotecontext/git/gitutils_test.go b/components/engine/builder/remotecontext/git/gitutils_test.go index b4c49195f0..fd58d6bd98 100644 --- a/components/engine/builder/remotecontext/git/gitutils_test.go +++ b/components/engine/builder/remotecontext/git/gitutils_test.go @@ -234,17 +234,17 @@ func TestCheckoutGit(t *testing.T) { if c.fail { assert.Error(t, err) continue - } else { + } + require.NoError(t, err) + defer os.RemoveAll(r) + if c.submodule { + b, err := ioutil.ReadFile(filepath.Join(r, "sub/subfile")) require.NoError(t, err) - if c.submodule { - b, err := ioutil.ReadFile(filepath.Join(r, "sub/subfile")) - require.NoError(t, err) - assert.Equal(t, "subcontents", string(b)) - } else { - _, err := os.Stat(filepath.Join(r, "sub/subfile")) - require.Error(t, err) - require.True(t, os.IsNotExist(err)) - } + assert.Equal(t, "subcontents", string(b)) + } else { + _, err := os.Stat(filepath.Join(r, "sub/subfile")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) } b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile")) From 6b042c3e6d405bc55e6cdd7628f898434d02d5af Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Thu, 7 Dec 2017 17:49:40 -0800 Subject: [PATCH 5/8] install-containerd-static: use netgo buildtag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When compiling containerd binaries statically, linker rightfully complains: + make BUILDTAGS=static_build 'EXTRA_FLAGS=-buildmode pie' 'EXTRA_LDFLAGS=-extldflags "-fno-PIC -static"' 🇩 bin/ctr # github.com/containerd/containerd/cmd/ctr /tmp/go-link-343047789/000000.o: In function `_cgo_b0c710f30cfd_C2func_getaddrinfo': /tmp/go-build/net/_obj/cgo-gcc-prolog:46: warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking The same error appears for ctr, containerd, and containerd-stress binaries. The fix is to use Go's own DNS resolver functions, rather than glibc's getaddrinfo() -- this option is turned on by `netgo` build tag. See https://golang.org/pkg/net/ (look for "Name Resolution") for more details. Signed-off-by: Kir Kolyshkin Upstream-commit: 7368ef96c89fd4f6879addf5214c4a09889a05be Component: engine --- components/engine/hack/dockerfile/install-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/install-binaries.sh b/components/engine/hack/dockerfile/install-binaries.sh index 160ff1a1da..ed97ce58b8 100755 --- a/components/engine/hack/dockerfile/install-binaries.sh +++ b/components/engine/hack/dockerfile/install-binaries.sh @@ -48,7 +48,7 @@ install_containerd_static() { git checkout -q "$CONTAINERD_COMMIT" ( export GOPATH - make BUILDTAGS='static_build' EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' + make BUILDTAGS='static_build netgo' EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' ) cp bin/containerd /usr/local/bin/docker-containerd cp bin/containerd-shim /usr/local/bin/docker-containerd-shim From fa87c342b2a3beb6b6f104d0d81aac7c48631e78 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 3 Jan 2018 09:54:05 -0800 Subject: [PATCH 6/8] Vendor Microsoft/hcsshim @ v0.6.8 Signed-off-by: John Howard Upstream-commit: 172a442c27ed35778662980809824fdf15a722a6 Component: engine --- components/engine/vendor.conf | 2 +- .../vendor/github.com/Microsoft/hcsshim/legacy.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index cf6bea58f7..184ac03777 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -1,6 +1,6 @@ # the following lines are in sorted order, FYI github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/Microsoft/hcsshim v0.6.7 +github.com/Microsoft/hcsshim v0.6.8 github.com/Microsoft/go-winio v0.4.5 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go b/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go index a0a97d7c72..673a4f8795 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go @@ -121,6 +121,16 @@ func (r *legacyLayerReader) walkUntilCancelled() error { if err != nil { return err } + + // Indirect fix for https://github.com/moby/moby/issues/32838#issuecomment-343610048. + // Handle failure from what may be a golang bug in the conversion of + // UTF16 to UTF8 in files which are left in the recycle bin. Os.Lstat + // which is called by filepath.Walk will fail when a filename contains + // unicode characters. Skip the recycle bin regardless which is goodness. + if strings.HasPrefix(path, filepath.Join(r.root, `Files\$Recycle.Bin`)) { + return filepath.SkipDir + } + if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") { return nil } From fb52c2003b51a88318fa8abf542c6fabf819da08 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 3 Jan 2018 15:59:06 -0800 Subject: [PATCH 7/8] Windows: Bump RW layer size Signed-off-by: John Howard Upstream-commit: deb335d04fd47e96fdb7b342dece513174b574bd Component: engine --- components/engine/builder/dockerfile/internals.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index c38f48afc0..adce7b9709 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strconv" "strings" @@ -534,7 +535,7 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf Ulimits: options.Ulimits, } - return &container.HostConfig{ + hc := &container.HostConfig{ SecurityOpt: options.SecurityOpt, Isolation: options.Isolation, ShmSize: options.ShmSize, @@ -544,6 +545,17 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf LogConfig: defaultLogConfig, ExtraHosts: options.ExtraHosts, } + + // For WCOW, the default of 20GB hard-coded in the platform + // is too small for builder scenarios where many users are + // using RUN statements to install large amounts of data. + // Use 127GB as that's the default size of a VHD in Hyper-V. + if runtime.GOOS == "windows" && options.Platform == "windows" { + hc.StorageOpt = make(map[string]string) + hc.StorageOpt["size"] = "127GB" + } + + return hc } // fromSlash works like filepath.FromSlash but with a given OS platform field From 656c1163c8a8e468181e58657f15887f579f0bc9 Mon Sep 17 00:00:00 2001 From: chchliang Date: Fri, 29 Sep 2017 15:23:54 +0800 Subject: [PATCH 8/8] add namespace method conver Signed-off-by: chchliang Upstream-commit: 0446b6a04edaae1bddeb1dfdb82fd0d931fe1198 Component: engine --- .../engine/distribution/metadata/v1_id_service_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/engine/distribution/metadata/v1_id_service_test.go b/components/engine/distribution/metadata/v1_id_service_test.go index 385901ec46..957ed5d63a 100644 --- a/components/engine/distribution/metadata/v1_id_service_test.go +++ b/components/engine/distribution/metadata/v1_id_service_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/docker/layer" + "github.com/stretchr/testify/require" ) func TestV1IDService(t *testing.T) { @@ -22,6 +23,10 @@ func TestV1IDService(t *testing.T) { } v1IDService := NewV1IDService(metadataStore) + ns := v1IDService.namespace() + + require.Equal(t, "v1id", ns) + testVectors := []struct { registry string v1ID string