diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index c38f48afc0..adce7b9709 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strconv" "strings" @@ -534,7 +535,7 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf Ulimits: options.Ulimits, } - return &container.HostConfig{ + hc := &container.HostConfig{ SecurityOpt: options.SecurityOpt, Isolation: options.Isolation, ShmSize: options.ShmSize, @@ -544,6 +545,17 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf LogConfig: defaultLogConfig, ExtraHosts: options.ExtraHosts, } + + // For WCOW, the default of 20GB hard-coded in the platform + // is too small for builder scenarios where many users are + // using RUN statements to install large amounts of data. + // Use 127GB as that's the default size of a VHD in Hyper-V. + if runtime.GOOS == "windows" && options.Platform == "windows" { + hc.StorageOpt = make(map[string]string) + hc.StorageOpt["size"] = "127GB" + } + + return hc } // fromSlash works like filepath.FromSlash but with a given OS platform field diff --git a/components/engine/builder/remotecontext/git/gitutils.go b/components/engine/builder/remotecontext/git/gitutils.go index 7bc2268155..67cff594ab 100644 --- a/components/engine/builder/remotecontext/git/gitutils.go +++ b/components/engine/builder/remotecontext/git/gitutils.go @@ -29,6 +29,10 @@ func Clone(remoteURL string) (string, error) { return "", err } + return cloneGitRepo(repo) +} + +func cloneGitRepo(repo gitRepo) (checkoutDir string, err error) { fetch := fetchArgs(repo.remote, repo.ref) root, err := ioutil.TempDir("", "docker-build-git") @@ -36,6 +40,12 @@ func Clone(remoteURL string) (string, error) { return "", err } + defer func() { + if err != nil { + os.RemoveAll(root) + } + }() + if out, err := gitWithinDir(root, "init"); err != nil { return "", errors.Wrapf(err, "failed to init repo at %s: %s", root, out) } @@ -50,7 +60,19 @@ func Clone(remoteURL string) (string, error) { return "", errors.Wrapf(err, "error fetching: %s", output) } - return checkoutGit(root, repo.ref, repo.subdir) + checkoutDir, err = checkoutGit(root, repo.ref, repo.subdir) + if err != nil { + return "", err + } + + cmd := exec.Command("git", "submodule", "update", "--init", "--recursive", "--depth=1") + cmd.Dir = root + output, err := cmd.CombinedOutput() + if err != nil { + return "", errors.Wrapf(err, "error initializing submodules: %s", output) + } + + return checkoutDir, nil } func parseRemoteURL(remoteURL string) (gitRepo, error) { @@ -96,7 +118,7 @@ func getRefAndSubdir(fragment string) (ref string, subdir string) { } func fetchArgs(remoteURL string, ref string) []string { - args := []string{"fetch", "--recurse-submodules=yes"} + args := []string{"fetch"} if supportsShallowClone(remoteURL) { args = append(args, "--depth", "1") diff --git a/components/engine/builder/remotecontext/git/gitutils_test.go b/components/engine/builder/remotecontext/git/gitutils_test.go index c638a498f2..fd58d6bd98 100644 --- a/components/engine/builder/remotecontext/git/gitutils_test.go +++ b/components/engine/builder/remotecontext/git/gitutils_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -61,7 +62,7 @@ func TestCloneArgsSmartHttp(t *testing.T) { }) args := fetchArgs(serverURL.String(), "master") - exp := []string{"fetch", "--recurse-submodules=yes", "--depth", "1", "origin", "master"} + exp := []string{"fetch", "--depth", "1", "origin", "master"} assert.Equal(t, exp, args) } @@ -77,13 +78,13 @@ func TestCloneArgsDumbHttp(t *testing.T) { }) args := fetchArgs(serverURL.String(), "master") - exp := []string{"fetch", "--recurse-submodules=yes", "origin", "master"} + exp := []string{"fetch", "origin", "master"} assert.Equal(t, exp, args) } func TestCloneArgsGit(t *testing.T) { args := fetchArgs("git://github.com/docker/docker", "master") - exp := []string{"fetch", "--recurse-submodules=yes", "--depth", "1", "origin", "master"} + exp := []string{"fetch", "--depth", "1", "origin", "master"} assert.Equal(t, exp, args) } @@ -165,24 +166,55 @@ func TestCheckoutGit(t *testing.T) { _, err = gitWithinDir(gitDir, "checkout", "master") require.NoError(t, err) + // set up submodule + subrepoDir := filepath.Join(root, "subrepo") + _, err = git("init", subrepoDir) + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "config", "user.email", "test@docker.com") + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "config", "user.name", "Docker test") + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(subrepoDir, "subfile"), []byte("subcontents"), 0644) + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "add", "-A") + require.NoError(t, err) + + _, err = gitWithinDir(subrepoDir, "commit", "-am", "Subrepo initial") + require.NoError(t, err) + + cmd := exec.Command("git", "submodule", "add", subrepoDir, "sub") // this command doesn't work with --work-tree + cmd.Dir = gitDir + require.NoError(t, cmd.Run()) + + _, err = gitWithinDir(gitDir, "add", "-A") + require.NoError(t, err) + + _, err = gitWithinDir(gitDir, "commit", "-am", "With submodule") + require.NoError(t, err) + type singleCase struct { - frag string - exp string - fail bool + frag string + exp string + fail bool + submodule bool } cases := []singleCase{ - {"", "FROM scratch", false}, - {"master", "FROM scratch", false}, - {":subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, - {":nosubdir", "", true}, // missing directory error - {":Dockerfile", "", true}, // not a directory error - {"master:nosubdir", "", true}, - {"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, - {"master:../subdir", "", true}, - {"test", "FROM scratch" + eol + "EXPOSE 3000", false}, - {"test:", "FROM scratch" + eol + "EXPOSE 3000", false}, - {"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false}, + {"", "FROM scratch", false, true}, + {"master", "FROM scratch", false, true}, + {":subdir", "FROM scratch" + eol + "EXPOSE 5000", false, false}, + {":nosubdir", "", true, false}, // missing directory error + {":Dockerfile", "", true, false}, // not a directory error + {"master:nosubdir", "", true, false}, + {"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false, false}, + {"master:../subdir", "", true, false}, + {"test", "FROM scratch" + eol + "EXPOSE 3000", false, false}, + {"test:", "FROM scratch" + eol + "EXPOSE 3000", false, false}, + {"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false, false}, } if runtime.GOOS != "windows" { @@ -197,12 +229,23 @@ func TestCheckoutGit(t *testing.T) { for _, c := range cases { ref, subdir := getRefAndSubdir(c.frag) - r, err := checkoutGit(gitDir, ref, subdir) + r, err := cloneGitRepo(gitRepo{remote: gitDir, ref: ref, subdir: subdir}) if c.fail { assert.Error(t, err) continue } + require.NoError(t, err) + defer os.RemoveAll(r) + if c.submodule { + b, err := ioutil.ReadFile(filepath.Join(r, "sub/subfile")) + require.NoError(t, err) + assert.Equal(t, "subcontents", string(b)) + } else { + _, err := os.Stat(filepath.Join(r, "sub/subfile")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + } b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile")) require.NoError(t, err) diff --git a/components/engine/daemon/logger/journald/journald.go b/components/engine/daemon/logger/journald/journald.go index 46aac8328b..ee6e5b68f7 100644 --- a/components/engine/daemon/logger/journald/journald.go +++ b/components/engine/daemon/logger/journald/journald.go @@ -75,6 +75,7 @@ func New(info logger.Info) (logger.Logger, error) { "CONTAINER_ID_FULL": info.ContainerID, "CONTAINER_NAME": info.Name(), "CONTAINER_TAG": tag, + "SYSLOG_IDENTIFIER": tag, } extraAttrs, err := info.ExtraAttributes(sanitizeKeyMod) if err != nil { diff --git a/components/engine/daemon/logger/splunk/splunk.go b/components/engine/daemon/logger/splunk/splunk.go index eb80a0b526..30ab7b7414 100644 --- a/components/engine/daemon/logger/splunk/splunk.go +++ b/components/engine/daemon/logger/splunk/splunk.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" + "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/urlutil" "github.com/sirupsen/logrus" ) @@ -55,6 +56,8 @@ const ( defaultBufferMaximum = 10 * defaultPostMessagesBatchSize // Number of messages allowed to be queued in the channel defaultStreamChannelSize = 4 * defaultPostMessagesBatchSize + // maxResponseSize is the max amount that will be read from an http response + maxResponseSize = 1024 ) const ( @@ -500,20 +503,22 @@ func (l *splunkLogger) tryPostMessages(ctx context.Context, messages []*splunkMe if l.gzipCompression { req.Header.Set("Content-Encoding", "gzip") } - res, err := l.client.Do(req) + resp, err := l.client.Do(req) if err != nil { return err } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - var body []byte - body, err = ioutil.ReadAll(res.Body) + defer func() { + pools.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + if resp.StatusCode != http.StatusOK { + rdr := io.LimitReader(resp.Body, maxResponseSize) + body, err := ioutil.ReadAll(rdr) if err != nil { return err } - return fmt.Errorf("%s: failed to send event - %s - %s", driverName, res.Status, body) + return fmt.Errorf("%s: failed to send event - %s - %s", driverName, resp.Status, string(body)) } - io.Copy(ioutil.Discard, res.Body) return nil } @@ -596,20 +601,22 @@ func verifySplunkConnection(l *splunkLogger) error { if err != nil { return err } - res, err := l.client.Do(req) + resp, err := l.client.Do(req) if err != nil { return err } - if res.Body != nil { - defer res.Body.Close() - } - if res.StatusCode != http.StatusOK { - var body []byte - body, err = ioutil.ReadAll(res.Body) + defer func() { + pools.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + rdr := io.LimitReader(resp.Body, maxResponseSize) + body, err := ioutil.ReadAll(rdr) if err != nil { return err } - return fmt.Errorf("%s: failed to verify connection - %s - %s", driverName, res.Status, body) + return fmt.Errorf("%s: failed to verify connection - %s - %s", driverName, resp.Status, string(body)) } return nil } diff --git a/components/engine/distribution/metadata/v1_id_service_test.go b/components/engine/distribution/metadata/v1_id_service_test.go index 385901ec46..957ed5d63a 100644 --- a/components/engine/distribution/metadata/v1_id_service_test.go +++ b/components/engine/distribution/metadata/v1_id_service_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/docker/layer" + "github.com/stretchr/testify/require" ) func TestV1IDService(t *testing.T) { @@ -22,6 +23,10 @@ func TestV1IDService(t *testing.T) { } v1IDService := NewV1IDService(metadataStore) + ns := v1IDService.namespace() + + require.Equal(t, "v1id", ns) + testVectors := []struct { registry string v1ID string diff --git a/components/engine/hack/dockerfile/install-binaries.sh b/components/engine/hack/dockerfile/install-binaries.sh index 160ff1a1da..ed97ce58b8 100755 --- a/components/engine/hack/dockerfile/install-binaries.sh +++ b/components/engine/hack/dockerfile/install-binaries.sh @@ -48,7 +48,7 @@ install_containerd_static() { git checkout -q "$CONTAINERD_COMMIT" ( export GOPATH - make BUILDTAGS='static_build' EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' + make BUILDTAGS='static_build netgo' EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' ) cp bin/containerd /usr/local/bin/docker-containerd cp bin/containerd-shim /usr/local/bin/docker-containerd-shim diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index cf6bea58f7..184ac03777 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -1,6 +1,6 @@ # the following lines are in sorted order, FYI github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/Microsoft/hcsshim v0.6.7 +github.com/Microsoft/hcsshim v0.6.8 github.com/Microsoft/go-winio v0.4.5 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go b/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go index a0a97d7c72..673a4f8795 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go @@ -121,6 +121,16 @@ func (r *legacyLayerReader) walkUntilCancelled() error { if err != nil { return err } + + // Indirect fix for https://github.com/moby/moby/issues/32838#issuecomment-343610048. + // Handle failure from what may be a golang bug in the conversion of + // UTF16 to UTF8 in files which are left in the recycle bin. Os.Lstat + // which is called by filepath.Walk will fail when a filename contains + // unicode characters. Skip the recycle bin regardless which is goodness. + if strings.HasPrefix(path, filepath.Join(r.root, `Files\$Recycle.Bin`)) { + return filepath.SkipDir + } + if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") { return nil }