From 7e985fdbccb650b41c605cf7692e12175b9895f0 Mon Sep 17 00:00:00 2001 From: Dani Louca Date: Tue, 26 Feb 2019 11:09:25 -0500 Subject: [PATCH 1/3] set bigger grpc limit for GetConfigs api Signed-off-by: Dani Louca (cherry picked from commit 3fbbeb703c1d04e9eb723459960fbfc7f3bbfc40) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 5f40e17cfd2474776088a97552fce7d5a9abe549 Component: engine --- components/engine/daemon/cluster/configs.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/engine/daemon/cluster/configs.go b/components/engine/daemon/cluster/configs.go index 6b373e618b..f44adb284e 100644 --- a/components/engine/daemon/cluster/configs.go +++ b/components/engine/daemon/cluster/configs.go @@ -7,6 +7,7 @@ import ( types "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/daemon/cluster/convert" swarmapi "github.com/docker/swarmkit/api" + "google.golang.org/grpc" ) // GetConfig returns a config from a managed swarm cluster @@ -44,7 +45,8 @@ func (c *Cluster) GetConfigs(options apitypes.ConfigListOptions) ([]types.Config defer cancel() r, err := state.controlClient.ListConfigs(ctx, - &swarmapi.ListConfigsRequest{Filters: filters}) + &swarmapi.ListConfigsRequest{Filters: filters}, + grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse)) if err != nil { return nil, err } From 78d7be4aebf7eced0e96618f0279f0b6f845c6f5 Mon Sep 17 00:00:00 2001 From: Sergio Lopez Date: Fri, 21 Dec 2018 09:30:09 +0100 Subject: [PATCH 2/3] layer/layer_store: ensure NewInputTarStream resources are released In applyTar, if the driver's ApplyDiff returns an error, the function returns early without calling io.Copy. As a consequence, the resources (a goroutine and some buffers holding the uncompressed image, the digest, etc...) allocated or referenced by NewInputTarStream above aren't released, as the worker goroutine only finishes when it finds EOF or a closed pipe. Signed-off-by: Sergio Lopez (cherry picked from commit 5846db10af9fb37061ab92a07c3d82fbea92b2e0) Signed-off-by: Sebastiaan van Stijn Upstream-commit: f660ef2c25590f21fdd10357c08a542a4876d6c3 Component: engine --- components/engine/layer/layer_store.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/engine/layer/layer_store.go b/components/engine/layer/layer_store.go index bc3e8719fc..1601465c04 100644 --- a/components/engine/layer/layer_store.go +++ b/components/engine/layer/layer_store.go @@ -253,13 +253,14 @@ func (ls *layerStore) applyTar(tx *fileMetadataTransaction, ts io.Reader, parent } applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, rdr) + // discard trailing data but ensure metadata is picked up to reconstruct stream + // unconditionally call io.Copy here before checking err to ensure the resources + // allocated by NewInputTarStream above are always released + io.Copy(ioutil.Discard, rdr) // ignore error as reader may be closed if err != nil { return err } - // Discard trailing data but ensure metadata is picked up to reconstruct stream - io.Copy(ioutil.Discard, rdr) // ignore error as reader may be closed - layer.size = applySize layer.diffID = DiffID(digester.Digest()) From ddb553c9c7855dc97ba5361b20c895e3d59731fc Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Thu, 24 Jan 2019 17:43:34 -0800 Subject: [PATCH 3/3] pkg/archive:CopyTo(): fix for long dest filename As reported in docker/for-linux/issues/484, since Docker 18.06 docker cp with a destination file name fails with the following error: > archive/tar: cannot encode header: Format specifies USTAR; and USTAR cannot encode Name="a_very_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_long_filename_that_is_101_characters" The problem is caused by changes in Go 1.10 archive/tar, which mis-guesses the tar stream format as USTAR (rather than PAX), which, in turn, leads to inability to specify file names longer than 100 characters. This tar stream is sent by TarWithOptions() (which, since we switched to Go 1.10, explicitly sets format=PAX for every file, see FileInfoHeader(), and before Go 1.10 it was PAX by default). Unfortunately, the receiving side, RebaseArchiveEntries(), which calls tar.Next(), mistakenly guesses header format as USTAR, which leads to the above error. The fix is easy: set the format to PAX in RebaseArchiveEntries() where we read the tar stream and change the file name. A unit test is added to prevent future regressions. NOTE this code is not used by dockerd, but rather but docker cli (also possibly other clients), so this needs to be re-vendored to cli in order to take effect. Signed-off-by: Kir Kolyshkin (cherry picked from commit f55a4176febbd0dffd6e5eb65beb70bc32912d0b) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 989e7f5d3a3f40ebb936376245b770f766ea42e9 Component: engine --- components/engine/pkg/archive/copy.go | 8 +++++++ .../engine/pkg/archive/copy_unix_test.go | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/components/engine/pkg/archive/copy.go b/components/engine/pkg/archive/copy.go index d0f13ca79b..57fddac078 100644 --- a/components/engine/pkg/archive/copy.go +++ b/components/engine/pkg/archive/copy.go @@ -336,6 +336,14 @@ func RebaseArchiveEntries(srcContent io.Reader, oldBase, newBase string) io.Read return } + // srcContent tar stream, as served by TarWithOptions(), is + // definitely in PAX format, but tar.Next() mistakenly guesses it + // as USTAR, which creates a problem: if the newBase is >100 + // characters long, WriteHeader() returns an error like + // "archive/tar: cannot encode header: Format specifies USTAR; and USTAR cannot encode Name=...". + // + // To fix, set the format to PAX here. See docker/for-linux issue #484. + hdr.Format = tar.FormatPAX hdr.Name = strings.Replace(hdr.Name, oldBase, newBase, 1) if hdr.Typeflag == tar.TypeLink { hdr.Linkname = strings.Replace(hdr.Linkname, oldBase, newBase, 1) diff --git a/components/engine/pkg/archive/copy_unix_test.go b/components/engine/pkg/archive/copy_unix_test.go index 739ad0e3ef..ca0f4653af 100644 --- a/components/engine/pkg/archive/copy_unix_test.go +++ b/components/engine/pkg/archive/copy_unix_test.go @@ -257,6 +257,30 @@ func TestCopyErrDstNotDir(t *testing.T) { } } +// Test to check if CopyTo works with a long (>100 characters) destination file name. +// This is a regression (see https://github.com/docker/for-linux/issues/484). +func TestCopyLongDstFilename(t *testing.T) { + const longName = "a_very_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_long_filename_that_is_101_characters" + tmpDirA, tmpDirB := getTestTempDirs(t) + defer removeAllPaths(tmpDirA, tmpDirB) + + // Load A with some sample files and directories. + createSampleDir(t, tmpDirA) + + srcInfo := CopyInfo{Path: filepath.Join(tmpDirA, "file1"), Exists: true, IsDir: false} + + content, err := TarResource(srcInfo) + if err != nil { + t.Fatalf("unexpected error %T: %s", err, err) + } + defer content.Close() + + err = CopyTo(content, srcInfo, filepath.Join(tmpDirB, longName)) + if err != nil { + t.Fatalf("unexpected error %T: %s", err, err) + } +} + // Possibilities are reduced to the remaining 10 cases: // // case | srcIsDir | onlyDirContents | dstExists | dstIsDir | dstTrSep | action