diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 73820bdd2f..18c092617c 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -155,7 +155,7 @@ RUN set -x \ # both. This allows integration-cli tests to cover push/pull with both schema1 # and schema2 manifests. ENV REGISTRY_COMMIT_SCHEMA1 ec87e9b6971d831f0eff752ddb54fb64693e51cd -ENV REGISTRY_COMMIT a7ae88da459b98b481a245e5b1750134724ac67d +ENV REGISTRY_COMMIT cb08de17d74bef86ce6c5abe8b240e282f5750be RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \ diff --git a/components/engine/distribution/metadata/blobsum_service.go b/components/engine/distribution/metadata/blobsum_service.go deleted file mode 100644 index 88ed7bb197..0000000000 --- a/components/engine/distribution/metadata/blobsum_service.go +++ /dev/null @@ -1,100 +0,0 @@ -package metadata - -import ( - "encoding/json" - - "github.com/docker/distribution/digest" - "github.com/docker/docker/layer" -) - -// BlobSumService maps layer IDs to a set of known blobsums for -// the layer. -type BlobSumService struct { - store Store -} - -// maxBlobSums is the number of blobsums to keep per layer DiffID. -const maxBlobSums = 5 - -// NewBlobSumService creates a new blobsum mapping service. -func NewBlobSumService(store Store) *BlobSumService { - return &BlobSumService{ - store: store, - } -} - -func (blobserv *BlobSumService) diffIDNamespace() string { - return "blobsum-storage" -} - -func (blobserv *BlobSumService) blobSumNamespace() string { - return "blobsum-lookup" -} - -func (blobserv *BlobSumService) diffIDKey(diffID layer.DiffID) string { - return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex() -} - -func (blobserv *BlobSumService) blobSumKey(blobsum digest.Digest) string { - return string(blobsum.Algorithm()) + "/" + blobsum.Hex() -} - -// GetBlobSums finds the blobsums associated with a layer DiffID. -func (blobserv *BlobSumService) GetBlobSums(diffID layer.DiffID) ([]digest.Digest, error) { - jsonBytes, err := blobserv.store.Get(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID)) - if err != nil { - return nil, err - } - - var blobsums []digest.Digest - if err := json.Unmarshal(jsonBytes, &blobsums); err != nil { - return nil, err - } - - return blobsums, nil -} - -// GetDiffID finds a layer DiffID from a blobsum hash. -func (blobserv *BlobSumService) GetDiffID(blobsum digest.Digest) (layer.DiffID, error) { - diffIDBytes, err := blobserv.store.Get(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum)) - if err != nil { - return layer.DiffID(""), err - } - - return layer.DiffID(diffIDBytes), nil -} - -// Add associates a blobsum with a layer DiffID. If too many blobsums are -// present, the oldest one is dropped. -func (blobserv *BlobSumService) Add(diffID layer.DiffID, blobsum digest.Digest) error { - oldBlobSums, err := blobserv.GetBlobSums(diffID) - if err != nil { - oldBlobSums = nil - } - newBlobSums := make([]digest.Digest, 0, len(oldBlobSums)+1) - - // Copy all other blobsums to new slice - for _, oldSum := range oldBlobSums { - if oldSum != blobsum { - newBlobSums = append(newBlobSums, oldSum) - } - } - - newBlobSums = append(newBlobSums, blobsum) - - if len(newBlobSums) > maxBlobSums { - newBlobSums = newBlobSums[len(newBlobSums)-maxBlobSums:] - } - - jsonBytes, err := json.Marshal(newBlobSums) - if err != nil { - return err - } - - err = blobserv.store.Set(blobserv.diffIDNamespace(), blobserv.diffIDKey(diffID), jsonBytes) - if err != nil { - return err - } - - return blobserv.store.Set(blobserv.blobSumNamespace(), blobserv.blobSumKey(blobsum), []byte(diffID)) -} diff --git a/components/engine/distribution/metadata/blobsum_service_test.go b/components/engine/distribution/metadata/blobsum_service_test.go deleted file mode 100644 index dee64df1ee..0000000000 --- a/components/engine/distribution/metadata/blobsum_service_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package metadata - -import ( - "io/ioutil" - "os" - "reflect" - "testing" - - "github.com/docker/distribution/digest" - "github.com/docker/docker/layer" -) - -func TestBlobSumService(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test") - if err != nil { - t.Fatalf("could not create temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - - metadataStore, err := NewFSMetadataStore(tmpDir) - if err != nil { - t.Fatalf("could not create metadata store: %v", err) - } - blobSumService := NewBlobSumService(metadataStore) - - testVectors := []struct { - diffID layer.DiffID - blobsums []digest.Digest - }{ - { - diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), - blobsums: []digest.Digest{ - digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"), - }, - }, - { - diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"), - blobsums: []digest.Digest{ - digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"), - digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"), - }, - }, - { - diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"), - blobsums: []digest.Digest{ - digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937"), - digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e"), - digest.Digest("sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf"), - digest.Digest("sha256:8902a7ca89aabbb868835260912159026637634090dd8899eee969523252236e"), - digest.Digest("sha256:c84364306344ccc48532c52ff5209236273525231dddaaab53262322352883aa"), - digest.Digest("sha256:aa7583bbc87532a8352bbb72520a821b3623523523a8352523a52352aaa888fe"), - }, - }, - } - - // Set some associations - for _, vec := range testVectors { - for _, blobsum := range vec.blobsums { - err := blobSumService.Add(vec.diffID, blobsum) - if err != nil { - t.Fatalf("error calling Set: %v", err) - } - } - } - - // Check the correct values are read back - for _, vec := range testVectors { - blobsums, err := blobSumService.GetBlobSums(vec.diffID) - if err != nil { - t.Fatalf("error calling Get: %v", err) - } - expectedBlobsums := len(vec.blobsums) - if expectedBlobsums > 5 { - expectedBlobsums = 5 - } - if !reflect.DeepEqual(blobsums, vec.blobsums[len(vec.blobsums)-expectedBlobsums:len(vec.blobsums)]) { - t.Fatal("Get returned incorrect layer ID") - } - } - - // Test GetBlobSums on a nonexistent entry - _, err = blobSumService.GetBlobSums(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")) - if err == nil { - t.Fatal("expected error looking up nonexistent entry") - } - - // Test GetDiffID on a nonexistent entry - _, err = blobSumService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")) - if err == nil { - t.Fatal("expected error looking up nonexistent entry") - } - - // Overwrite one of the entries and read it back - err = blobSumService.Add(testVectors[1].diffID, testVectors[0].blobsums[0]) - if err != nil { - t.Fatalf("error calling Add: %v", err) - } - diffID, err := blobSumService.GetDiffID(testVectors[0].blobsums[0]) - if err != nil { - t.Fatalf("error calling GetDiffID: %v", err) - } - if diffID != testVectors[1].diffID { - t.Fatal("GetDiffID returned incorrect diffID") - } -} diff --git a/components/engine/distribution/metadata/metadata.go b/components/engine/distribution/metadata/metadata.go index ab9cc5b626..9f744d46fc 100644 --- a/components/engine/distribution/metadata/metadata.go +++ b/components/engine/distribution/metadata/metadata.go @@ -15,6 +15,8 @@ type Store interface { Get(namespace string, key string) ([]byte, error) // Set writes data indexed by namespace and key. Set(namespace, key string, value []byte) error + // Delete removes data indexed by namespace and key. + Delete(namespace, key string) error } // FSMetadataStore uses the filesystem to associate metadata with layer and @@ -63,3 +65,13 @@ func (store *FSMetadataStore) Set(namespace, key string, value []byte) error { } return os.Rename(tempFilePath, path) } + +// Delete removes data indexed by namespace and key. The data file named after +// the key, stored in the namespace's directory is deleted. +func (store *FSMetadataStore) Delete(namespace, key string) error { + store.Lock() + defer store.Unlock() + + path := store.path(namespace, key) + return os.Remove(path) +} diff --git a/components/engine/distribution/metadata/v2_metadata_service.go b/components/engine/distribution/metadata/v2_metadata_service.go new file mode 100644 index 0000000000..239cd1f45e --- /dev/null +++ b/components/engine/distribution/metadata/v2_metadata_service.go @@ -0,0 +1,137 @@ +package metadata + +import ( + "encoding/json" + + "github.com/docker/distribution/digest" + "github.com/docker/docker/layer" +) + +// V2MetadataService maps layer IDs to a set of known metadata for +// the layer. +type V2MetadataService struct { + store Store +} + +// V2Metadata contains the digest and source repository information for a layer. +type V2Metadata struct { + Digest digest.Digest + SourceRepository string +} + +// maxMetadata is the number of metadata entries to keep per layer DiffID. +const maxMetadata = 50 + +// NewV2MetadataService creates a new diff ID to v2 metadata mapping service. +func NewV2MetadataService(store Store) *V2MetadataService { + return &V2MetadataService{ + store: store, + } +} + +func (serv *V2MetadataService) diffIDNamespace() string { + return "v2metadata-by-diffid" +} + +func (serv *V2MetadataService) digestNamespace() string { + return "diffid-by-digest" +} + +func (serv *V2MetadataService) diffIDKey(diffID layer.DiffID) string { + return string(digest.Digest(diffID).Algorithm()) + "/" + digest.Digest(diffID).Hex() +} + +func (serv *V2MetadataService) digestKey(dgst digest.Digest) string { + return string(dgst.Algorithm()) + "/" + dgst.Hex() +} + +// GetMetadata finds the metadata associated with a layer DiffID. +func (serv *V2MetadataService) GetMetadata(diffID layer.DiffID) ([]V2Metadata, error) { + jsonBytes, err := serv.store.Get(serv.diffIDNamespace(), serv.diffIDKey(diffID)) + if err != nil { + return nil, err + } + + var metadata []V2Metadata + if err := json.Unmarshal(jsonBytes, &metadata); err != nil { + return nil, err + } + + return metadata, nil +} + +// GetDiffID finds a layer DiffID from a digest. +func (serv *V2MetadataService) GetDiffID(dgst digest.Digest) (layer.DiffID, error) { + diffIDBytes, err := serv.store.Get(serv.digestNamespace(), serv.digestKey(dgst)) + if err != nil { + return layer.DiffID(""), err + } + + return layer.DiffID(diffIDBytes), nil +} + +// Add associates metadata with a layer DiffID. If too many metadata entries are +// present, the oldest one is dropped. +func (serv *V2MetadataService) Add(diffID layer.DiffID, metadata V2Metadata) error { + oldMetadata, err := serv.GetMetadata(diffID) + if err != nil { + oldMetadata = nil + } + newMetadata := make([]V2Metadata, 0, len(oldMetadata)+1) + + // Copy all other metadata to new slice + for _, oldMeta := range oldMetadata { + if oldMeta != metadata { + newMetadata = append(newMetadata, oldMeta) + } + } + + newMetadata = append(newMetadata, metadata) + + if len(newMetadata) > maxMetadata { + newMetadata = newMetadata[len(newMetadata)-maxMetadata:] + } + + jsonBytes, err := json.Marshal(newMetadata) + if err != nil { + return err + } + + err = serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes) + if err != nil { + return err + } + + return serv.store.Set(serv.digestNamespace(), serv.digestKey(metadata.Digest), []byte(diffID)) +} + +// Remove unassociates a metadata entry from a layer DiffID. +func (serv *V2MetadataService) Remove(metadata V2Metadata) error { + diffID, err := serv.GetDiffID(metadata.Digest) + if err != nil { + return err + } + oldMetadata, err := serv.GetMetadata(diffID) + if err != nil { + oldMetadata = nil + } + newMetadata := make([]V2Metadata, 0, len(oldMetadata)) + + // Copy all other metadata to new slice + for _, oldMeta := range oldMetadata { + if oldMeta != metadata { + newMetadata = append(newMetadata, oldMeta) + } + } + + if len(newMetadata) == 0 { + return serv.store.Delete(serv.diffIDNamespace(), serv.diffIDKey(diffID)) + } + + jsonBytes, err := json.Marshal(newMetadata) + if err != nil { + return err + } + + return serv.store.Set(serv.diffIDNamespace(), serv.diffIDKey(diffID), jsonBytes) +} diff --git a/components/engine/distribution/metadata/v2_metadata_service_test.go b/components/engine/distribution/metadata/v2_metadata_service_test.go new file mode 100644 index 0000000000..7b0ecb1572 --- /dev/null +++ b/components/engine/distribution/metadata/v2_metadata_service_test.go @@ -0,0 +1,115 @@ +package metadata + +import ( + "encoding/hex" + "io/ioutil" + "math/rand" + "os" + "reflect" + "testing" + + "github.com/docker/distribution/digest" + "github.com/docker/docker/layer" +) + +func TestV2MetadataService(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "blobsum-storage-service-test") + if err != nil { + t.Fatalf("could not create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + metadataStore, err := NewFSMetadataStore(tmpDir) + if err != nil { + t.Fatalf("could not create metadata store: %v", err) + } + V2MetadataService := NewV2MetadataService(metadataStore) + + tooManyBlobSums := make([]V2Metadata, 100) + for i := range tooManyBlobSums { + randDigest := randomDigest() + tooManyBlobSums[i] = V2Metadata{Digest: randDigest} + } + + testVectors := []struct { + diffID layer.DiffID + metadata []V2Metadata + }{ + { + diffID: layer.DiffID("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), + metadata: []V2Metadata{ + {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")}, + }, + }, + { + diffID: layer.DiffID("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"), + metadata: []V2Metadata{ + {Digest: digest.Digest("sha256:f0cd5ca10b07f35512fc2f1cbf9a6cefbdb5cba70ac6b0c9e5988f4497f71937")}, + {Digest: digest.Digest("sha256:9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e")}, + }, + }, + { + diffID: layer.DiffID("sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"), + metadata: tooManyBlobSums, + }, + } + + // Set some associations + for _, vec := range testVectors { + for _, blobsum := range vec.metadata { + err := V2MetadataService.Add(vec.diffID, blobsum) + if err != nil { + t.Fatalf("error calling Set: %v", err) + } + } + } + + // Check the correct values are read back + for _, vec := range testVectors { + metadata, err := V2MetadataService.GetMetadata(vec.diffID) + if err != nil { + t.Fatalf("error calling Get: %v", err) + } + expectedMetadataEntries := len(vec.metadata) + if expectedMetadataEntries > 50 { + expectedMetadataEntries = 50 + } + if !reflect.DeepEqual(metadata, vec.metadata[len(vec.metadata)-expectedMetadataEntries:len(vec.metadata)]) { + t.Fatal("Get returned incorrect layer ID") + } + } + + // Test GetMetadata on a nonexistent entry + _, err = V2MetadataService.GetMetadata(layer.DiffID("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")) + if err == nil { + t.Fatal("expected error looking up nonexistent entry") + } + + // Test GetDiffID on a nonexistent entry + _, err = V2MetadataService.GetDiffID(digest.Digest("sha256:82379823067823853223359023576437723560923756b03560378f4497753917")) + if err == nil { + t.Fatal("expected error looking up nonexistent entry") + } + + // Overwrite one of the entries and read it back + err = V2MetadataService.Add(testVectors[1].diffID, testVectors[0].metadata[0]) + if err != nil { + t.Fatalf("error calling Add: %v", err) + } + diffID, err := V2MetadataService.GetDiffID(testVectors[0].metadata[0].Digest) + if err != nil { + t.Fatalf("error calling GetDiffID: %v", err) + } + if diffID != testVectors[1].diffID { + t.Fatal("GetDiffID returned incorrect diffID") + } +} + +func randomDigest() digest.Digest { + b := [32]byte{} + for i := 0; i < len(b); i++ { + b[i] = byte(rand.Intn(256)) + } + d := hex.EncodeToString(b[:]) + return digest.Digest("sha256:" + d) +} diff --git a/components/engine/distribution/pull.go b/components/engine/distribution/pull.go index db6e29d681..5f38a67673 100644 --- a/components/engine/distribution/pull.go +++ b/components/engine/distribution/pull.go @@ -61,10 +61,10 @@ func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, switch endpoint.Version { case registry.APIVersion2: return &v2Puller{ - blobSumService: metadata.NewBlobSumService(imagePullConfig.MetadataStore), - endpoint: endpoint, - config: imagePullConfig, - repoInfo: repoInfo, + V2MetadataService: metadata.NewV2MetadataService(imagePullConfig.MetadataStore), + endpoint: endpoint, + config: imagePullConfig, + repoInfo: repoInfo, }, nil case registry.APIVersion1: return &v1Puller{ diff --git a/components/engine/distribution/pull_v2.go b/components/engine/distribution/pull_v2.go index 7277d07fb1..7bb171000d 100644 --- a/components/engine/distribution/pull_v2.go +++ b/components/engine/distribution/pull_v2.go @@ -33,11 +33,11 @@ import ( var errRootFSMismatch = errors.New("layers from manifest don't match image configuration") type v2Puller struct { - blobSumService *metadata.BlobSumService - endpoint registry.APIEndpoint - config *ImagePullConfig - repoInfo *registry.RepositoryInfo - repo distribution.Repository + V2MetadataService *metadata.V2MetadataService + endpoint registry.APIEndpoint + config *ImagePullConfig + repoInfo *registry.RepositoryInfo + repo distribution.Repository // confirmedV2 is set to true if we confirm we're talking to a v2 // registry. This is used to limit fallbacks to the v1 protocol. confirmedV2 bool @@ -110,9 +110,10 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e } type v2LayerDescriptor struct { - digest digest.Digest - repo distribution.Repository - blobSumService *metadata.BlobSumService + digest digest.Digest + repoInfo *registry.RepositoryInfo + repo distribution.Repository + V2MetadataService *metadata.V2MetadataService } func (ld *v2LayerDescriptor) Key() string { @@ -124,7 +125,7 @@ func (ld *v2LayerDescriptor) ID() string { } func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) { - return ld.blobSumService.GetDiffID(ld.digest) + return ld.V2MetadataService.GetDiffID(ld.digest) } func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) { @@ -196,7 +197,7 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) { // Cache mapping from this layer's DiffID to the blobsum - ld.blobSumService.Add(diffID, ld.digest) + ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()}) } func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) { @@ -333,9 +334,10 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif } layerDescriptor := &v2LayerDescriptor{ - digest: blobSum, - repo: p.repo, - blobSumService: p.blobSumService, + digest: blobSum, + repoInfo: p.repoInfo, + repo: p.repo, + V2MetadataService: p.V2MetadataService, } descriptors = append(descriptors, layerDescriptor) @@ -398,9 +400,10 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s // to top-most, so that the downloads slice gets ordered correctly. for _, d := range mfst.References() { layerDescriptor := &v2LayerDescriptor{ - digest: d.Digest, - repo: p.repo, - blobSumService: p.blobSumService, + digest: d.Digest, + repo: p.repo, + repoInfo: p.repoInfo, + V2MetadataService: p.V2MetadataService, } descriptors = append(descriptors, layerDescriptor) diff --git a/components/engine/distribution/push.go b/components/engine/distribution/push.go index 092b07f468..445f6bb6bd 100644 --- a/components/engine/distribution/push.go +++ b/components/engine/distribution/push.go @@ -71,11 +71,11 @@ func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *reg switch endpoint.Version { case registry.APIVersion2: return &v2Pusher{ - blobSumService: metadata.NewBlobSumService(imagePushConfig.MetadataStore), - ref: ref, - endpoint: endpoint, - repoInfo: repoInfo, - config: imagePushConfig, + v2MetadataService: metadata.NewV2MetadataService(imagePushConfig.MetadataStore), + ref: ref, + endpoint: endpoint, + repoInfo: repoInfo, + config: imagePushConfig, }, nil case registry.APIVersion1: return &v1Pusher{ diff --git a/components/engine/distribution/push_v2.go b/components/engine/distribution/push_v2.go index 98fb13e5c2..68c8f69be7 100644 --- a/components/engine/distribution/push_v2.go +++ b/components/engine/distribution/push_v2.go @@ -11,6 +11,7 @@ import ( "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + distreference "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/distribution/xfer" @@ -34,12 +35,12 @@ type PushResult struct { } type v2Pusher struct { - blobSumService *metadata.BlobSumService - ref reference.Named - endpoint registry.APIEndpoint - repoInfo *registry.RepositoryInfo - config *ImagePushConfig - repo distribution.Repository + v2MetadataService *metadata.V2MetadataService + ref reference.Named + endpoint registry.APIEndpoint + repoInfo *registry.RepositoryInfo + config *ImagePushConfig + repo distribution.Repository // pushState is state built by the Download functions. pushState pushState @@ -130,9 +131,10 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, ima var descriptors []xfer.UploadDescriptor descriptorTemplate := v2PushDescriptor{ - blobSumService: p.blobSumService, - repo: p.repo, - pushState: &p.pushState, + v2MetadataService: p.v2MetadataService, + repoInfo: p.repoInfo, + repo: p.repo, + pushState: &p.pushState, } // Loop bounds condition is to avoid pushing the base layer on Windows. @@ -209,10 +211,11 @@ func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuild } type v2PushDescriptor struct { - layer layer.Layer - blobSumService *metadata.BlobSumService - repo distribution.Repository - pushState *pushState + layer layer.Layer + v2MetadataService *metadata.V2MetadataService + repoInfo reference.Named + repo distribution.Repository + pushState *pushState } func (pd *v2PushDescriptor) Key() string { @@ -240,10 +243,10 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. } pd.pushState.Unlock() - // Do we have any blobsums associated with this layer's DiffID? - possibleBlobsums, err := pd.blobSumService.GetBlobSums(diffID) + // Do we have any metadata associated with this layer's DiffID? + v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) if err == nil { - descriptor, exists, err := blobSumAlreadyExists(ctx, possibleBlobsums, pd.repo, pd.pushState) + descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState) if err != nil { progress.Update(progressOutput, pd.ID(), "Image push failed") return retryOnError(err) @@ -263,8 +266,69 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. // then push the blob. bs := pd.repo.Blobs(ctx) + var mountFrom metadata.V2Metadata + + // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload + for _, metadata := range v2Metadata { + sourceRepo, err := reference.ParseNamed(metadata.SourceRepository) + if err != nil { + continue + } + if pd.repoInfo.Hostname() == sourceRepo.Hostname() { + logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, metadata.Digest, sourceRepo.FullName()) + mountFrom = metadata + break + } + } + + var createOpts []distribution.BlobCreateOption + + if mountFrom.SourceRepository != "" { + namedRef, err := reference.WithName(mountFrom.SourceRepository) + if err != nil { + return err + } + + // TODO (brianbland): We need to construct a reference where the Name is + // only the full remote name, so clean this up when distribution has a + // richer reference package + remoteRef, err := distreference.WithName(namedRef.RemoteName()) + if err != nil { + return err + } + + canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest) + if err != nil { + return err + } + + createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) + } + // Send the layer - layerUpload, err := bs.Create(ctx) + layerUpload, err := bs.Create(ctx, createOpts...) + switch err := err.(type) { + case distribution.ErrBlobMounted: + progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) + + pd.pushState.Lock() + pd.pushState.confirmedV2 = true + pd.pushState.remoteLayers[diffID] = err.Descriptor + pd.pushState.Unlock() + + // Cache mapping from this layer's DiffID to the blobsum + if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil { + return xfer.DoNotRetry{Err: err} + } + + return nil + } + if mountFrom.SourceRepository != "" { + // unable to mount layer from this repository, so this source mapping is no longer valid + logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository) + pd.v2MetadataService.Remove(mountFrom) + } + if err != nil { return retryOnError(err) } @@ -300,7 +364,7 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. progress.Update(progressOutput, pd.ID(), "Pushed") // Cache mapping from this layer's DiffID to the blobsum - if err := pd.blobSumService.Add(diffID, pushDigest); err != nil { + if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil { return xfer.DoNotRetry{Err: err} } @@ -329,12 +393,16 @@ func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor { return pd.pushState.remoteLayers[pd.DiffID()] } -// blobSumAlreadyExists checks if the registry already know about any of the -// blobsums passed in the "blobsums" slice. If it finds one that the registry +// layerAlreadyExists checks if the registry already know about any of the +// metadata passed in the "metadata" slice. If it finds one that the registry // knows about, it returns the known digest and "true". -func blobSumAlreadyExists(ctx context.Context, blobsums []digest.Digest, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) { - for _, dgst := range blobsums { - descriptor, err := repo.Blobs(ctx).Stat(ctx, dgst) +func layerAlreadyExists(ctx context.Context, metadata []metadata.V2Metadata, repoInfo reference.Named, repo distribution.Repository, pushState *pushState) (distribution.Descriptor, bool, error) { + for _, meta := range metadata { + // Only check blobsums that are known to this repository or have an unknown source + if meta.SourceRepository != "" && meta.SourceRepository != repoInfo.FullName() { + continue + } + descriptor, err := repo.Blobs(ctx).Stat(ctx, meta.Digest) switch err { case nil: descriptor.MediaType = schema2.MediaTypeLayer diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index b05c9b0ac7..dcc5cbadde 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -45,7 +45,7 @@ clone git github.com/boltdb/bolt v1.1.0 clone git github.com/miekg/dns d27455715200c7d3e321a1e5cadb27c9ee0b0f02 # get graph and distribution packages -clone git github.com/docker/distribution a7ae88da459b98b481a245e5b1750134724ac67d +clone git github.com/docker/distribution cb08de17d74bef86ce6c5abe8b240e282f5750be clone git github.com/vbatts/tar-split v0.9.11 # get desired notary commit, might also need to be updated in Dockerfile diff --git a/components/engine/integration-cli/docker_cli_push_test.go b/components/engine/integration-cli/docker_cli_push_test.go index be5f9aad8e..c100772b00 100644 --- a/components/engine/integration-cli/docker_cli_push_test.go +++ b/components/engine/integration-cli/docker_cli_push_test.go @@ -147,6 +147,58 @@ func (s *DockerSchema1RegistrySuite) TestPushEmptyLayer(c *check.C) { testPushEmptyLayer(c) } +func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) { + sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + // tag the image to upload it to the private registry + dockerCmd(c, "tag", "busybox", sourceRepoName) + // push the image to the registry + out1, _, err := dockerCmdWithError("push", sourceRepoName) + c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out1)) + // ensure that none of the layers were mounted from another repository during push + c.Assert(strings.Contains(out1, "Mounted from"), check.Equals, false) + + destRepoName := fmt.Sprintf("%v/dockercli/crossrepopush", privateRegistryURL) + // retag the image to upload the same layers to another repo in the same registry + dockerCmd(c, "tag", "busybox", destRepoName) + // push the image to the registry + out2, _, err := dockerCmdWithError("push", destRepoName) + c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out2)) + // ensure that layers were mounted from the first repo during push + c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, true) + + // ensure that we can pull and run the cross-repo-pushed repository + dockerCmd(c, "rmi", destRepoName) + dockerCmd(c, "pull", destRepoName) + out3, _ := dockerCmd(c, "run", destRepoName, "echo", "-n", "hello world") + c.Assert(out3, check.Equals, "hello world") +} + +func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c *check.C) { + sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + // tag the image to upload it to the private registry + dockerCmd(c, "tag", "busybox", sourceRepoName) + // push the image to the registry + out1, _, err := dockerCmdWithError("push", sourceRepoName) + c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out1)) + // ensure that none of the layers were mounted from another repository during push + c.Assert(strings.Contains(out1, "Mounted from"), check.Equals, false) + + destRepoName := fmt.Sprintf("%v/dockercli/crossrepopush", privateRegistryURL) + // retag the image to upload the same layers to another repo in the same registry + dockerCmd(c, "tag", "busybox", destRepoName) + // push the image to the registry + out2, _, err := dockerCmdWithError("push", destRepoName) + c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out2)) + // schema1 registry should not support cross-repo layer mounts, so ensure that this does not happen + c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, false) + + // ensure that we can pull and run the second pushed repository + dockerCmd(c, "rmi", destRepoName) + dockerCmd(c, "pull", destRepoName) + out3, _ := dockerCmd(c, "run", destRepoName, "echo", "-n", "hello world") + c.Assert(out3, check.Equals, "hello world") +} + func (s *DockerTrustSuite) TestTrustedPush(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/trusted:latest", privateRegistryURL) // tag the image and upload it to the private registry diff --git a/components/engine/migrate/v1/migratev1.go b/components/engine/migrate/v1/migratev1.go index 77507c3dd4..9243c5a42a 100644 --- a/components/engine/migrate/v1/migratev1.go +++ b/components/engine/migrate/v1/migratev1.go @@ -476,8 +476,8 @@ func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metad if err == nil { // best effort dgst, err := digest.ParseDigest(string(checksum)) if err == nil { - blobSumService := metadata.NewBlobSumService(ms) - blobSumService.Add(layer.DiffID(), dgst) + V2MetadataService := metadata.NewV2MetadataService(ms) + V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst}) } } _, err = ls.Release(layer) diff --git a/components/engine/migrate/v1/migratev1_test.go b/components/engine/migrate/v1/migratev1_test.go index 5fe26637e0..6e8af7fdc8 100644 --- a/components/engine/migrate/v1/migratev1_test.go +++ b/components/engine/migrate/v1/migratev1_test.go @@ -210,19 +210,19 @@ func TestMigrateImages(t *testing.T) { t.Fatalf("invalid register count: expected %q, got %q", expected, actual) } - blobSumService := metadata.NewBlobSumService(ms) - blobsums, err := blobSumService.GetBlobSums(layer.EmptyLayer.DiffID()) + v2MetadataService := metadata.NewV2MetadataService(ms) + receivedMetadata, err := v2MetadataService.GetMetadata(layer.EmptyLayer.DiffID()) if err != nil { t.Fatal(err) } - expectedBlobsums := []digest.Digest{ - "sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57", - "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4", + expectedMetadata := []metadata.V2Metadata{ + {Digest: digest.Digest("sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57")}, + {Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, } - if !reflect.DeepEqual(expectedBlobsums, blobsums) { - t.Fatalf("invalid blobsums: expected %q, got %q", expectedBlobsums, blobsums) + if !reflect.DeepEqual(expectedMetadata, receivedMetadata) { + t.Fatalf("invalid metadata: expected %q, got %q", expectedMetadata, receivedMetadata) } } diff --git a/components/engine/vendor/src/github.com/docker/distribution/blobs.go b/components/engine/vendor/src/github.com/docker/distribution/blobs.go index 40cd829578..ce43ea2ef6 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/blobs.go +++ b/components/engine/vendor/src/github.com/docker/distribution/blobs.go @@ -9,6 +9,7 @@ import ( "github.com/docker/distribution/context" "github.com/docker/distribution/digest" + "github.com/docker/distribution/reference" ) var ( @@ -40,6 +41,18 @@ func (err ErrBlobInvalidDigest) Error() string { err.Digest, err.Reason) } +// ErrBlobMounted returned when a blob is mounted from another repository +// instead of initiating an upload session. +type ErrBlobMounted struct { + From reference.Canonical + Descriptor Descriptor +} + +func (err ErrBlobMounted) Error() string { + return fmt.Sprintf("blob mounted from: %v to: %v", + err.From, err.Descriptor) +} + // Descriptor describes targeted content. Used in conjunction with a blob // store, a descriptor can be used to fetch, store and target any kind of // blob. The struct also describes the wire protocol format. Fields should @@ -151,12 +164,21 @@ type BlobIngester interface { // returned handle can be written to and later resumed using an opaque // identifier. With this approach, one can Close and Resume a BlobWriter // multiple times until the BlobWriter is committed or cancelled. - Create(ctx context.Context) (BlobWriter, error) + Create(ctx context.Context, options ...BlobCreateOption) (BlobWriter, error) // Resume attempts to resume a write to a blob, identified by an id. Resume(ctx context.Context, id string) (BlobWriter, error) } +// BlobCreateOption is a general extensible function argument for blob creation +// methods. A BlobIngester may choose to honor any or none of the given +// BlobCreateOptions, which can be specific to the implementation of the +// BlobIngester receiving them. +// TODO (brianbland): unify this with ManifestServiceOption in the future +type BlobCreateOption interface { + Apply(interface{}) error +} + // BlobWriter provides a handle for inserting data into a blob store. // Instances should be obtained from BlobWriteService.Writer and // BlobWriteService.Resume. If supported by the store, a writer can be diff --git a/components/engine/vendor/src/github.com/docker/distribution/circle.yml b/components/engine/vendor/src/github.com/docker/distribution/circle.yml index e275b2e18d..e1995d4b9f 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/circle.yml +++ b/components/engine/vendor/src/github.com/docker/distribution/circle.yml @@ -11,7 +11,7 @@ machine: post: # go - - gvm install go1.5 --prefer-binary --name=stable + - gvm install go1.5.3 --prefer-binary --name=stable environment: # Convenient shortcuts to "common" locations diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go b/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go index 52c725dc2f..ad3da3efb9 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go @@ -1041,6 +1041,70 @@ var routeDescriptors = []RouteDescriptor{ deniedResponseDescriptor, }, }, + { + Name: "Mount Blob", + Description: "Mount a blob identified by the `mount` parameter from another repository.", + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + contentLengthZeroHeader, + }, + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + }, + QueryParameters: []ParameterDescriptor{ + { + Name: "mount", + Type: "query", + Format: "", + Regexp: digest.DigestRegexp, + Description: `Digest of blob to mount from the source repository.`, + }, + { + Name: "from", + Type: "query", + Format: "", + Regexp: reference.NameRegexp, + Description: `Name of the source repository.`, + }, + }, + Successes: []ResponseDescriptor{ + { + Description: "The blob has been mounted in the repository and is available at the provided location.", + StatusCode: http.StatusCreated, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Name: "Invalid Name or Digest", + StatusCode: http.StatusBadRequest, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + }, + }, + { + Name: "Not allowed", + Description: "Blob mount is not allowed because the registry is configured as a pull-through cache or for some other reason", + StatusCode: http.StatusMethodNotAllowed, + ErrorCodes: []errcode.ErrorCode{ + errcode.ErrorCodeUnsupported, + }, + }, + unauthorizedResponseDescriptor, + repositoryNotFoundResponseDescriptor, + deniedResponseDescriptor, + }, + }, }, }, }, diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go b/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go index 9819b3cb84..6b483c62ef 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go @@ -108,6 +108,8 @@ type tokenHandler struct { tokenLock sync.Mutex tokenCache string tokenExpiration time.Time + + additionalScopes map[string]struct{} } // tokenScope represents the scope at which a token will be requested. @@ -145,6 +147,7 @@ func newTokenHandler(transport http.RoundTripper, creds CredentialStore, c clock Scope: scope, Actions: actions, }, + additionalScopes: map[string]struct{}{}, } } @@ -160,7 +163,15 @@ func (th *tokenHandler) Scheme() string { } func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - if err := th.refreshToken(params); err != nil { + var additionalScopes []string + if fromParam := req.URL.Query().Get("from"); fromParam != "" { + additionalScopes = append(additionalScopes, tokenScope{ + Resource: "repository", + Scope: fromParam, + Actions: []string{"pull"}, + }.String()) + } + if err := th.refreshToken(params, additionalScopes...); err != nil { return err } @@ -169,11 +180,18 @@ func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]st return nil } -func (th *tokenHandler) refreshToken(params map[string]string) error { +func (th *tokenHandler) refreshToken(params map[string]string, additionalScopes ...string) error { th.tokenLock.Lock() defer th.tokenLock.Unlock() + var addedScopes bool + for _, scope := range additionalScopes { + if _, ok := th.additionalScopes[scope]; !ok { + th.additionalScopes[scope] = struct{}{} + addedScopes = true + } + } now := th.clock.Now() - if now.After(th.tokenExpiration) { + if now.After(th.tokenExpiration) || addedScopes { tr, err := th.fetchToken(params) if err != nil { return err @@ -223,6 +241,10 @@ func (th *tokenHandler) fetchToken(params map[string]string) (token *tokenRespon reqParams.Add("scope", scopeField) } + for scope := range th.additionalScopes { + reqParams.Add("scope", scope) + } + if th.creds != nil { username, password := th.creds.Basic(realmURL) if username != "" && password != "" { diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go b/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go index 758c6e5e31..d652121109 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go @@ -572,8 +572,57 @@ func (bs *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribut return writer.Commit(ctx, desc) } -func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { - u, err := bs.ub.BuildBlobUploadURL(bs.name) +// createOptions is a collection of blob creation modifiers relevant to general +// blob storage intended to be configured by the BlobCreateOption.Apply method. +type createOptions struct { + Mount struct { + ShouldMount bool + From reference.Canonical + } +} + +type optionFunc func(interface{}) error + +func (f optionFunc) Apply(v interface{}) error { + return f(v) +} + +// WithMountFrom returns a BlobCreateOption which designates that the blob should be +// mounted from the given canonical reference. +func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption { + return optionFunc(func(v interface{}) error { + opts, ok := v.(*createOptions) + if !ok { + return fmt.Errorf("unexpected options type: %T", v) + } + + opts.Mount.ShouldMount = true + opts.Mount.From = ref + + return nil + }) +} + +func (bs *blobs) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + var opts createOptions + + for _, option := range options { + err := option.Apply(&opts) + if err != nil { + return nil, err + } + } + + var values []url.Values + + if opts.Mount.ShouldMount { + values = append(values, url.Values{"from": {opts.Mount.From.Name()}, "mount": {opts.Mount.From.Digest().String()}}) + } + + u, err := bs.ub.BuildBlobUploadURL(bs.name, values...) + if err != nil { + return nil, err + } resp, err := bs.client.Post(u, "", nil) if err != nil { @@ -581,7 +630,14 @@ func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { } defer resp.Body.Close() - if SuccessStatus(resp.StatusCode) { + switch resp.StatusCode { + case http.StatusCreated: + desc, err := bs.statter.Stat(ctx, opts.Mount.From.Digest()) + if err != nil { + return nil, err + } + return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc} + case http.StatusAccepted: // TODO(dmcgowan): Check for invalid UUID uuid := resp.Header.Get("Docker-Upload-UUID") location, err := sanitizeLocation(resp.Header.Get("Location"), u) @@ -596,8 +652,9 @@ func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { startedAt: time.Now(), location: location, }, nil + default: + return nil, HandleErrorResponse(resp) } - return nil, HandleErrorResponse(resp) } func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {