diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index c2e279505f..8bc546bd9a 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -44,9 +44,7 @@ FROM base AS criu # Install CRIU for checkpoint/restore support ENV CRIU_VERSION 3.6 # Install dependancy packages specific to criu -RUN case $(uname -m) in \ - x86_64) \ - apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y \ libnet-dev \ libprotobuf-c0-dev \ libprotobuf-dev \ @@ -59,13 +57,7 @@ RUN case $(uname -m) in \ && curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \ && cd /usr/src/criu \ && make \ - && make PREFIX=/opt/criu install-criu ;\ - ;; \ - armv7l|aarch64|ppc64le|s390x) \ - mkdir -p /opt/criu; \ - ;; \ - esac - + && make PREFIX=/opt/criu install-criu FROM base AS registry # Install two versions of the registry. The first is an older version that @@ -80,8 +72,8 @@ RUN set -x \ && (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \ && GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \ go build -buildmode=pie -o /usr/local/bin/registry-v2 github.com/docker/distribution/cmd/registry \ - && case $(uname -m) in \ - x86_64|ppc64le|s390x) \ + && case $(dpkg --print-architecture) in \ + amd64|ppc64*|s390x) \ (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1"); \ GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH"; \ go build -buildmode=pie -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry; \ diff --git a/components/engine/daemon/exec.go b/components/engine/daemon/exec.go index 89c3c8969a..6a94aca417 100644 --- a/components/engine/daemon/exec.go +++ b/components/engine/daemon/exec.go @@ -270,7 +270,7 @@ func (d *Daemon) ContainerExecStart(ctx context.Context, name string, stdin io.R case <-attachErr: // TERM signal worked } - return fmt.Errorf("context cancelled") + return ctx.Err() case err := <-attachErr: if err != nil { if _, ok := err.(term.EscapeError); !ok { diff --git a/components/engine/distribution/push_v2.go b/components/engine/distribution/push_v2.go index 7b7155169d..f7b9a6d657 100644 --- a/components/engine/distribution/push_v2.go +++ b/components/engine/distribution/push_v2.go @@ -15,6 +15,7 @@ import ( "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" + "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/client" apitypes "github.com/docker/docker/api/types" "github.com/docker/docker/distribution/metadata" @@ -55,12 +56,14 @@ type pushState struct { // confirmedV2 is set to true if we confirm we're talking to a v2 // registry. This is used to limit fallbacks to the v1 protocol. confirmedV2 bool + hasAuthInfo bool } func (p *v2Pusher) Push(ctx context.Context) (err error) { p.pushState.remoteLayers = make(map[layer.DiffID]distribution.Descriptor) p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull") + p.pushState.hasAuthInfo = p.config.AuthConfig.RegistryToken != "" || (p.config.AuthConfig.Username != "" && p.config.AuthConfig.Password != "") if err != nil { logrus.Debugf("Error getting v2 registry: %v", err) return err @@ -308,6 +311,7 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata) + isUnauthorizedError := false for _, mountCandidate := range candidates { logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository) createOpts := []distribution.BlobCreateOption{} @@ -360,11 +364,26 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } return err.Descriptor, nil + case errcode.Errors: + for _, e := range err { + switch e := e.(type) { + case errcode.Error: + if e.Code == errcode.ErrorCodeUnauthorized { + // when unauthorized error that indicate user don't has right to push layer to register + logrus.Debugln("failed to push layer to registry because unauthorized error") + isUnauthorizedError = true + } + default: + } + } default: logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err) } + // when error is unauthorizedError and user don't hasAuthInfo that's the case user don't has right to push layer to register + // and he hasn't login either, in this case candidate cache should be removed if len(mountCandidate.SourceRepository) > 0 && + !(isUnauthorizedError && !pd.pushState.hasAuthInfo) && (metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) || len(mountCandidate.HMAC) == 0) { cause := "blob mount failure" @@ -398,7 +417,6 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. } } defer layerUpload.Close() - // upload the blob return pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload) } diff --git a/components/engine/distribution/push_v2_test.go b/components/engine/distribution/push_v2_test.go index ac68470b64..c3616b936d 100644 --- a/components/engine/distribution/push_v2_test.go +++ b/components/engine/distribution/push_v2_test.go @@ -2,6 +2,7 @@ package distribution // import "github.com/docker/docker/distribution" import ( "net/http" + "net/url" "reflect" "testing" @@ -9,9 +10,13 @@ import ( "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/docker/api/types" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/progress" + refstore "github.com/docker/docker/reference" + "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" ) @@ -461,6 +466,158 @@ func TestLayerAlreadyExists(t *testing.T) { } } +type mockReferenceStore struct { +} + +func (s *mockReferenceStore) References(id digest.Digest) []reference.Named { + return []reference.Named{} +} +func (s *mockReferenceStore) ReferencesByName(ref reference.Named) []refstore.Association { + return []refstore.Association{} +} +func (s *mockReferenceStore) AddTag(ref reference.Named, id digest.Digest, force bool) error { + return nil +} +func (s *mockReferenceStore) AddDigest(ref reference.Canonical, id digest.Digest, force bool) error { + return nil +} +func (s *mockReferenceStore) Delete(ref reference.Named) (bool, error) { + return true, nil +} +func (s *mockReferenceStore) Get(ref reference.Named) (digest.Digest, error) { + return "", nil +} + +func TestWhenEmptyAuthConfig(t *testing.T) { + for _, authInfo := range []struct { + username string + password string + registryToken string + expected bool + }{ + { + username: "", + password: "", + registryToken: "", + expected: false, + }, + { + username: "username", + password: "password", + registryToken: "", + expected: true, + }, + { + username: "", + password: "", + registryToken: "token", + expected: true, + }, + } { + imagePushConfig := &ImagePushConfig{} + imagePushConfig.AuthConfig = &types.AuthConfig{ + Username: authInfo.username, + Password: authInfo.password, + RegistryToken: authInfo.registryToken, + } + imagePushConfig.ReferenceStore = &mockReferenceStore{} + repoInfo, _ := reference.ParseNormalizedNamed("xujihui1985/test.img") + pusher := &v2Pusher{ + config: imagePushConfig, + repoInfo: ®istry.RepositoryInfo{ + Name: repoInfo, + }, + endpoint: registry.APIEndpoint{ + URL: &url.URL{ + Scheme: "https", + Host: "index.docker.io", + }, + Version: registry.APIVersion1, + TrimHostname: true, + }, + } + pusher.Push(context.Background()) + if pusher.pushState.hasAuthInfo != authInfo.expected { + t.Errorf("hasAuthInfo does not match expected: %t != %t", authInfo.expected, pusher.pushState.hasAuthInfo) + } + } +} + +type mockBlobStoreWithCreate struct { + mockBlobStore + repo *mockRepoWithBlob +} + +func (blob *mockBlobStoreWithCreate) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + return nil, errcode.Errors(append([]error{errcode.ErrorCodeUnauthorized.WithMessage("unauthorized")})) +} + +type mockRepoWithBlob struct { + mockRepo +} + +func (m *mockRepoWithBlob) Blobs(ctx context.Context) distribution.BlobStore { + blob := &mockBlobStoreWithCreate{} + blob.mockBlobStore.repo = &m.mockRepo + blob.repo = m + return blob +} + +type mockMetadataService struct { + mockV2MetadataService +} + +func (m *mockMetadataService) GetMetadata(diffID layer.DiffID) ([]metadata.V2Metadata, error) { + return []metadata.V2Metadata{ + taggedMetadata("abcd", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e28", "docker.io/user/app1"), + taggedMetadata("abcd", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e22", "docker.io/user/app/base"), + taggedMetadata("hash", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e23", "docker.io/user/app"), + taggedMetadata("abcd", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e24", "127.0.0.1/user/app"), + taggedMetadata("hash", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e25", "docker.io/user/foo"), + taggedMetadata("hash", "sha256:ff3a5c916c92643ff77519ffa742d3ec61b7f591b6b7504599d95a4a41134e26", "docker.io/app/bar"), + }, nil +} + +var removeMetadata bool + +func (m *mockMetadataService) Remove(metadata metadata.V2Metadata) error { + removeMetadata = true + return nil +} + +func TestPushRegistryWhenAuthInfoEmpty(t *testing.T) { + repoInfo, _ := reference.ParseNormalizedNamed("user/app") + ms := &mockMetadataService{} + remoteErrors := map[digest.Digest]error{digest.Digest("sha256:apple"): distribution.ErrAccessDenied} + remoteBlobs := map[digest.Digest]distribution.Descriptor{digest.Digest("sha256:apple"): {Digest: digest.Digest("shar256:apple")}} + repo := &mockRepoWithBlob{ + mockRepo: mockRepo{ + t: t, + errors: remoteErrors, + blobs: remoteBlobs, + requests: []string{}, + }, + } + pd := &v2PushDescriptor{ + hmacKey: []byte("abcd"), + repoInfo: repoInfo, + layer: &storeLayer{ + Layer: layer.EmptyLayer, + }, + repo: repo, + v2MetadataService: ms, + pushState: &pushState{ + remoteLayers: make(map[layer.DiffID]distribution.Descriptor), + hasAuthInfo: false, + }, + checkedDigests: make(map[digest.Digest]struct{}), + } + pd.Upload(context.Background(), &progressSink{t}) + if removeMetadata { + t.Fatalf("expect remove not be called but called") + } +} + func taggedMetadata(key string, dgst string, sourceRepo string) metadata.V2Metadata { meta := metadata.V2Metadata{ Digest: digest.Digest(dgst), diff --git a/components/engine/image/spec/README.md b/components/engine/image/spec/README.md new file mode 100644 index 0000000000..9769af781a --- /dev/null +++ b/components/engine/image/spec/README.md @@ -0,0 +1,46 @@ +# Docker Image Specification v1. + +This directory contains documents about Docker Image Specification v1.X. + +The v1 file layout and manifests are no longer used in Moby and Docker, except in `docker save` and `docker load`. + +However, v1 Image JSON (`application/vnd.docker.container.image.v1+json`) has been still widely +used and officially adopted in [V2 manifest](https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md) +and in [OCI Image Format Specification](https://github.com/opencontainers/image-spec). + +## v1.X rough Changelog + +All 1.X versions are compatible with older ones. + +### [v1.2](v1.2.md) + +* Implemented in Docker v1.12 (July, 2016) +* The official spec document was written in August 2016 ([#25750](https://github.com/moby/moby/pull/25750)) + +Changes: + +* `Healthcheck` struct was added to Image JSON + +### [v1.1](v1.1.md) + +* Implemented in Docker v1.10 (February, 2016) +* The official spec document was written in April 2016 ([#22264](https://github.com/moby/moby/pull/22264)) + +Changes: + +* IDs were made into SHA256 digest values rather than random values +* Layer directory names were made into deterministic values rather than random ID values +* `manifest.json` was added + +### [v1](v1.md) + +* The initial revision +* The official spec document was written in late 2014 ([#9560](https://github.com/moby/moby/pull/9560)), but actual implementations had existed even earlier + + +## Related specifications + +* [Open Containers Initiative (OCI) Image Format Specification v1.0.0](https://github.com/opencontainers/image-spec/tree/v1.0.0) +* [Docker Image Manifest Version 2, Schema 2](https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md) +* [Docker Image Manifest Version 2, Schema 1](https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md) (*DEPRECATED*) +* [Docker Registry HTTP API V2](https://docs.docker.com/registry/spec/api/) diff --git a/components/engine/integration/container/inspect_test.go b/components/engine/integration/container/inspect_test.go index 03b9e45319..0433522e68 100644 --- a/components/engine/integration/container/inspect_test.go +++ b/components/engine/integration/container/inspect_test.go @@ -22,7 +22,7 @@ func TestInspectCpusetInConfigPre120(t *testing.T) { client := request.NewAPIClient(t, client.WithVersion("1.19")) ctx := context.Background() - name := "cpusetinconfig-pre120" + name := "cpusetinconfig-pre120-" + t.Name() // Create container with up to-date-API container.Run(t, ctx, request.NewAPIClient(t), container.WithName(name), container.WithCmd("true"), diff --git a/components/engine/integration/container/ps_test.go b/components/engine/integration/container/ps_test.go index 45bcaca239..4dacef1652 100644 --- a/components/engine/integration/container/ps_test.go +++ b/components/engine/integration/container/ps_test.go @@ -17,9 +17,10 @@ func TestPsFilter(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - prev := container.Create(t, ctx, client, container.WithName("prev")) - container.Create(t, ctx, client, container.WithName("top")) - next := container.Create(t, ctx, client, container.WithName("next")) + prev := container.Create(t, ctx, client, container.WithName("prev-"+t.Name())) + topContainerName := "top-" + t.Name() + container.Create(t, ctx, client, container.WithName(topContainerName)) + next := container.Create(t, ctx, client, container.WithName("next-"+t.Name())) containerIDs := func(containers []types.Container) []string { entries := []string{} @@ -30,7 +31,7 @@ func TestPsFilter(t *testing.T) { } f1 := filters.NewArgs() - f1.Add("since", "top") + f1.Add("since", topContainerName) q1, err := client.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: f1, @@ -39,7 +40,7 @@ func TestPsFilter(t *testing.T) { assert.Check(t, is.Contains(containerIDs(q1), next)) f2 := filters.NewArgs() - f2.Add("before", "top") + f2.Add("before", topContainerName) q2, err := client.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: f2, diff --git a/components/engine/integration/container/stop_test.go b/components/engine/integration/container/stop_test.go index 2cc9b82512..04aec21594 100644 --- a/components/engine/integration/container/stop_test.go +++ b/components/engine/integration/container/stop_test.go @@ -21,7 +21,7 @@ func TestStopContainerWithRestartPolicyAlways(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - names := []string{"verifyRestart1", "verifyRestart2"} + names := []string{"verifyRestart1-" + t.Name(), "verifyRestart2-" + t.Name()} for _, name := range names { container.Run(t, ctx, client, container.WithName(name), container.WithCmd("false"), func(c *container.TestContainerConfig) { c.HostConfig.RestartPolicy.Name = "always" @@ -49,7 +49,7 @@ func TestDeleteDevicemapper(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - id := container.Run(t, ctx, client, container.WithName("foo"), container.WithCmd("echo")) + id := container.Run(t, ctx, client, container.WithName("foo-"+t.Name()), container.WithCmd("echo")) poll.WaitOn(t, container.IsStopped(ctx, client, id), poll.WithDelay(100*time.Millisecond))