From c052f895a805b1e1faac6107d81b3691663a5996 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 10 Oct 2017 09:23:33 +0000 Subject: [PATCH 1/6] image/spec: add historical information about v1 spec Signed-off-by: Akihiro Suda Upstream-commit: 1d17542f80d1a961224c762cec7628293f81465e Component: engine --- components/engine/image/spec/README.md | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 components/engine/image/spec/README.md diff --git a/components/engine/image/spec/README.md b/components/engine/image/spec/README.md new file mode 100644 index 0000000000..9769af781a --- /dev/null +++ b/components/engine/image/spec/README.md @@ -0,0 +1,46 @@ +# Docker Image Specification v1. + +This directory contains documents about Docker Image Specification v1.X. + +The v1 file layout and manifests are no longer used in Moby and Docker, except in `docker save` and `docker load`. + +However, v1 Image JSON (`application/vnd.docker.container.image.v1+json`) has been still widely +used and officially adopted in [V2 manifest](https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md) +and in [OCI Image Format Specification](https://github.com/opencontainers/image-spec). + +## v1.X rough Changelog + +All 1.X versions are compatible with older ones. + +### [v1.2](v1.2.md) + +* Implemented in Docker v1.12 (July, 2016) +* The official spec document was written in August 2016 ([#25750](https://github.com/moby/moby/pull/25750)) + +Changes: + +* `Healthcheck` struct was added to Image JSON + +### [v1.1](v1.1.md) + +* Implemented in Docker v1.10 (February, 2016) +* The official spec document was written in April 2016 ([#22264](https://github.com/moby/moby/pull/22264)) + +Changes: + +* IDs were made into SHA256 digest values rather than random values +* Layer directory names were made into deterministic values rather than random ID values +* `manifest.json` was added + +### [v1](v1.md) + +* The initial revision +* The official spec document was written in late 2014 ([#9560](https://github.com/moby/moby/pull/9560)), but actual implementations had existed even earlier + + +## Related specifications + +* [Open Containers Initiative (OCI) Image Format Specification v1.0.0](https://github.com/opencontainers/image-spec/tree/v1.0.0) +* [Docker Image Manifest Version 2, Schema 2](https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md) +* [Docker Image Manifest Version 2, Schema 1](https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md) (*DEPRECATED*) +* [Docker Registry HTTP API V2](https://docs.docker.com/registry/spec/api/) From dd2d0c4792cf0bce15c82f04c85ca05473d4f391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=85=95=E9=99=B6?= Date: Wed, 7 Mar 2018 19:30:17 +0800 Subject: [PATCH 2/6] fix(distribution): digest cache should not be moved if it was an auth error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit local digest cache will be removed when error occured on push image but it should not be removed if it is an auth error while on auth was provided https://github.com/moby/moby/issues/36309 Signed-off-by: 慕陶 Upstream-commit: 8b387b165ab2eaab3f9fdac25caa186d05d236a0 Component: engine --- components/engine/distribution/push_v2.go | 20 ++- .../engine/distribution/push_v2_test.go | 157 ++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/components/engine/distribution/push_v2.go b/components/engine/distribution/push_v2.go index 7b7155169d..f7b9a6d657 100644 --- a/components/engine/distribution/push_v2.go +++ b/components/engine/distribution/push_v2.go @@ -15,6 +15,7 @@ import ( "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" + "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/client" apitypes "github.com/docker/docker/api/types" "github.com/docker/docker/distribution/metadata" @@ -55,12 +56,14 @@ type pushState struct { // confirmedV2 is set to true if we confirm we're talking to a v2 // registry. This is used to limit fallbacks to the v1 protocol. confirmedV2 bool + hasAuthInfo bool } func (p *v2Pusher) Push(ctx context.Context) (err error) { p.pushState.remoteLayers = make(map[layer.DiffID]distribution.Descriptor) p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull") + p.pushState.hasAuthInfo = p.config.AuthConfig.RegistryToken != "" || (p.config.AuthConfig.Username != "" && p.config.AuthConfig.Password != "") if err != nil { logrus.Debugf("Error getting v2 registry: %v", err) return err @@ -308,6 +311,7 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata) + isUnauthorizedError := false for _, mountCandidate := range candidates { logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository) createOpts := []distribution.BlobCreateOption{} @@ -360,11 +364,26 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } return err.Descriptor, nil + case errcode.Errors: + for _, e := range err { + switch e := e.(type) { + case errcode.Error: + if e.Code == errcode.ErrorCodeUnauthorized { + // when unauthorized error that indicate user don't has right to push layer to register + logrus.Debugln("failed to push layer to registry because unauthorized error") + isUnauthorizedError = true + } + default: + } + } default: logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err) } + // when error is unauthorizedError and user don't hasAuthInfo that's the case user don't has right to push layer to register + // and he hasn't login either, in this case candidate cache should be removed if len(mountCandidate.SourceRepository) > 0 && + !(isUnauthorizedError && !pd.pushState.hasAuthInfo) && (metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) || len(mountCandidate.HMAC) == 0) { cause := "blob mount failure" @@ -398,7 +417,6 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. } } defer layerUpload.Close() - // upload the blob return pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload) } diff --git a/components/engine/distribution/push_v2_test.go b/components/engine/distribution/push_v2_test.go index ac68470b64..c3616b936d 100644 --- a/components/engine/distribution/push_v2_test.go +++ b/components/engine/distribution/push_v2_test.go @@ -2,6 +2,7 @@ package distribution // import "github.com/docker/docker/distribution" import ( "net/http" + "net/url" "reflect" "testing" @@ -9,9 +10,13 @@ import ( "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/docker/api/types" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/progress" + refstore "github.com/docker/docker/reference" + "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" ) @@ -461,6 +466,158 @@ func TestLayerAlreadyExists(t *testing.T) { } } +type mockReferenceStore struct { +} + +func (s *mockReferenceStore) References(id digest.Digest) []reference.Named { + return []reference.Named{} +} +func (s *mockReferenceStore) ReferencesByName(ref reference.Named) []refstore.Association { + return []refstore.Association{} +} +func (s *mockReferenceStore) AddTag(ref reference.Named, id digest.Digest, force bool) error { + return nil +} +func (s *mockReferenceStore) AddDigest(ref reference.Canonical, id digest.Digest, force bool) error { + return nil +} +func (s *mockReferenceStore) Delete(ref reference.Named) (bool, error) { + return true, nil +} +func (s *mockReferenceStore) Get(ref reference.Named) (digest.Digest, error) { + return "", nil +} + +func TestWhenEmptyAuthConfig(t *testing.T) { + for _, authInfo := range []struct { + username string + password string + registryToken string + expected bool + }{ + { + username: "", + password: "", + registryToken: "", + expected: false, + }, + { + username: "username", + password: "password", + registryToken: "", + expected: true, + }, + { + username: "", + password: "", + registryToken: "token", + expected: true, + }, + } { + imagePushConfig := &ImagePushConfig{} + imagePushConfig.AuthConfig = &types.AuthConfig{ + Username: authInfo.username, + Password: authInfo.password, + RegistryToken: authInfo.registryToken, + } + imagePushConfig.ReferenceStore = &mockReferenceStore{} + repoInfo, _ := reference.ParseNormalizedNamed("xujihui1985/test.img") + pusher := &v2Pusher{ + config: imagePushConfig, + repoInfo: ®istry.RepositoryInfo{ + Name: repoInfo, + }, + endpoint: registry.APIEndpoint{ + URL: &url.URL{ + Scheme: "https", + Host: "index.docker.io", + }, + Version: registry.APIVersion1, + TrimHostname: true, + }, + } + pusher.Push(context.Background()) + if pusher.pushState.hasAuthInfo != authInfo.expected { + t.Errorf("hasAuthInfo does not match expected: %t != %t", authInfo.expected, pusher.pushState.hasAuthInfo) + } + } +} + +type mockBlobStoreWithCreate struct { + mockBlobStore + repo *mockRepoWithBlob +} + +func (blob *mockBlobStoreWithCreate) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + return nil, errcode.Errors(append([]error{errcode.ErrorCodeUnauthorized.WithMessage("unauthorized")})) +} + +type mockRepoWithBlob struct { + mockRepo +} + +func (m *mockRepoWithBlob) Blobs(ctx context.Context) distribution.BlobStore { + blob := &mockBlobStoreWithCreate{} + blob.mockBlobStore.repo = &m.mockRepo + blob.repo = m + return blob +} + +type mockMetadataService struct { + mockV2MetadataService +} + +func (m *mockMetadataService) GetMetadata(diffID layer.DiffID) ([]metadata.V2Metadata, error) { + return []metadata.V2Metadata{ + taggedMetadata("abcd", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e28", "docker.io/user/app1"), + taggedMetadata("abcd", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e22", "docker.io/user/app/base"), + taggedMetadata("hash", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e23", "docker.io/user/app"), + taggedMetadata("abcd", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e24", "127.0.0.1/user/app"), + taggedMetadata("hash", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e25", "docker.io/user/foo"), + taggedMetadata("hash", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e26", "docker.io/app/bar"), + }, nil +} + +var removeMetadata bool + +func (m *mockMetadataService) Remove(metadata metadata.V2Metadata) error { + removeMetadata = true + return nil +} + +func TestPushRegistryWhenAuthInfoEmpty(t *testing.T) { + repoInfo, _ := reference.ParseNormalizedNamed("user/app") + ms := &mockMetadataService{} + remoteErrors := map[digest.Digest]error{digest.Digest("sha256:apple"): distribution.ErrAccessDenied} + remoteBlobs := map[digest.Digest]distribution.Descriptor{digest.Digest("sha256:apple"): {Digest: digest.Digest("shar256:apple")}} + repo := &mockRepoWithBlob{ + mockRepo: mockRepo{ + t: t, + errors: remoteErrors, + blobs: remoteBlobs, + requests: []string{}, + }, + } + pd := &v2PushDescriptor{ + hmacKey: []byte("abcd"), + repoInfo: repoInfo, + layer: &storeLayer{ + Layer: layer.EmptyLayer, + }, + repo: repo, + v2MetadataService: ms, + pushState: &pushState{ + remoteLayers: make(map[layer.DiffID]distribution.Descriptor), + hasAuthInfo: false, + }, + checkedDigests: make(map[digest.Digest]struct{}), + } + pd.Upload(context.Background(), &progressSink{t}) + if removeMetadata { + t.Fatalf("expect remove not be called but called") + } +} + func taggedMetadata(key string, dgst string, sourceRepo string) metadata.V2Metadata { meta := metadata.V2Metadata{ Digest: digest.Digest(dgst), From 809b43730dc0ad5ee16526d2db94d3786df6a0f4 Mon Sep 17 00:00:00 2001 From: Arash Deshmeh Date: Wed, 21 Mar 2018 17:47:49 -0400 Subject: [PATCH 3/6] use unique names for resources used by integration tests container/inspect_test, container/ps_test, container/stop_test Signed-off-by: Arash Deshmeh Upstream-commit: 78e4be91332e2237c0fa14eb3ba0fb5b915c3256 Component: engine --- .../engine/integration/container/inspect_test.go | 2 +- components/engine/integration/container/ps_test.go | 11 ++++++----- components/engine/integration/container/stop_test.go | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/components/engine/integration/container/inspect_test.go b/components/engine/integration/container/inspect_test.go index 03b9e45319..0433522e68 100644 --- a/components/engine/integration/container/inspect_test.go +++ b/components/engine/integration/container/inspect_test.go @@ -22,7 +22,7 @@ func TestInspectCpusetInConfigPre120(t *testing.T) { client := request.NewAPIClient(t, client.WithVersion("1.19")) ctx := context.Background() - name := "cpusetinconfig-pre120" + name := "cpusetinconfig-pre120-" + t.Name() // Create container with up to-date-API container.Run(t, ctx, request.NewAPIClient(t), container.WithName(name), container.WithCmd("true"), diff --git a/components/engine/integration/container/ps_test.go b/components/engine/integration/container/ps_test.go index 45bcaca239..4dacef1652 100644 --- a/components/engine/integration/container/ps_test.go +++ b/components/engine/integration/container/ps_test.go @@ -17,9 +17,10 @@ func TestPsFilter(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - prev := container.Create(t, ctx, client, container.WithName("prev")) - container.Create(t, ctx, client, container.WithName("top")) - next := container.Create(t, ctx, client, container.WithName("next")) + prev := container.Create(t, ctx, client, container.WithName("prev-"+t.Name())) + topContainerName := "top-" + t.Name() + container.Create(t, ctx, client, container.WithName(topContainerName)) + next := container.Create(t, ctx, client, container.WithName("next-"+t.Name())) containerIDs := func(containers []types.Container) []string { entries := []string{} @@ -30,7 +31,7 @@ func TestPsFilter(t *testing.T) { } f1 := filters.NewArgs() - f1.Add("since", "top") + f1.Add("since", topContainerName) q1, err := client.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: f1, @@ -39,7 +40,7 @@ func TestPsFilter(t *testing.T) { assert.Check(t, is.Contains(containerIDs(q1), next)) f2 := filters.NewArgs() - f2.Add("before", "top") + f2.Add("before", topContainerName) q2, err := client.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: f2, diff --git a/components/engine/integration/container/stop_test.go b/components/engine/integration/container/stop_test.go index 2cc9b82512..04aec21594 100644 --- a/components/engine/integration/container/stop_test.go +++ b/components/engine/integration/container/stop_test.go @@ -21,7 +21,7 @@ func TestStopContainerWithRestartPolicyAlways(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - names := []string{"verifyRestart1", "verifyRestart2"} + names := []string{"verifyRestart1-" + t.Name(), "verifyRestart2-" + t.Name()} for _, name := range names { container.Run(t, ctx, client, container.WithName(name), container.WithCmd("false"), func(c *container.TestContainerConfig) { c.HostConfig.RestartPolicy.Name = "always" @@ -49,7 +49,7 @@ func TestDeleteDevicemapper(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - id := container.Run(t, ctx, client, container.WithName("foo"), container.WithCmd("echo")) + id := container.Run(t, ctx, client, container.WithName("foo-"+t.Name()), container.WithCmd("echo")) poll.WaitOn(t, container.IsStopped(ctx, client, id), poll.WithDelay(100*time.Millisecond)) From f358bd4b474d8beefea0c7062c4e7e60c0a73481 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Thu, 22 Mar 2018 09:38:59 -0700 Subject: [PATCH 4/6] daemon: use context error rather than inventing new one Signed-off-by: Stephen J Day Upstream-commit: d84da75f01e0a0d20fbddb8b051a325e3b21eded Component: engine --- components/engine/daemon/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/exec.go b/components/engine/daemon/exec.go index 89c3c8969a..6a94aca417 100644 --- a/components/engine/daemon/exec.go +++ b/components/engine/daemon/exec.go @@ -270,7 +270,7 @@ func (d *Daemon) ContainerExecStart(ctx context.Context, name string, stdin io.R case <-attachErr: // TERM signal worked } - return fmt.Errorf("context cancelled") + return ctx.Err() case err := <-attachErr: if err != nil { if _, ok := err.(term.EscapeError); !ok { From 985dd23e4a327e403b2eb97d15bf8ea352beaa4c Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Tue, 13 Mar 2018 03:18:09 +0000 Subject: [PATCH 5/6] Enable CRIU on non-amd64 architectures Since the recent release of CRIU has already supported other arches such as AArch64, ppc64le, and s390x, so we can enable it now. Signed-off-by: Dennis Chen Upstream-commit: 7fd54a7a48f9f1bb2b28144e38c886d1fe813a04 Component: engine --- components/engine/Dockerfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index c2e279505f..7c4a8608db 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -44,9 +44,7 @@ FROM base AS criu # Install CRIU for checkpoint/restore support ENV CRIU_VERSION 3.6 # Install dependancy packages specific to criu -RUN case $(uname -m) in \ - x86_64) \ - apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y \ libnet-dev \ libprotobuf-c0-dev \ libprotobuf-dev \ @@ -59,13 +57,7 @@ RUN case $(uname -m) in \ && curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \ && cd /usr/src/criu \ && make \ - && make PREFIX=/opt/criu install-criu ;\ - ;; \ - armv7l|aarch64|ppc64le|s390x) \ - mkdir -p /opt/criu; \ - ;; \ - esac - + && make PREFIX=/opt/criu install-criu FROM base AS registry # Install two versions of the registry. The first is an older version that From 2bc1fa895ff21e1c3d32ffef0f5b51ee49101563 Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Fri, 23 Mar 2018 06:01:48 +0000 Subject: [PATCH 6/6] Remove the `uname -m` in Dockerfile Using `dpkg --print-architecture` instead of the `uname -m` to abstract the architecture value from the container images, which the build process is running inside, to match exactly the behavior specified by the following Docker file while not 'passthru' to the host. Signed-off-by: Dennis Chen Upstream-commit: 803a756941f5e4b68429a3642d52585c8ea6dbaa Component: engine --- components/engine/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 7c4a8608db..8bc546bd9a 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -72,8 +72,8 @@ RUN set -x \ && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \ && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \ go build -buildmode=pie -o /usr/local/bin/registry-v2 github.com/docker/distribution/cmd/registry \ - && case $(uname -m) in \ - x86_64|ppc64le|s390x) \ + && case $(dpkg --print-architecture) in \ + amd64|ppc64*|s390x) \ (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1"); \ GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH"; \ go build -buildmode=pie -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry; \