From 176bd2dcef9ab20c22725a0481da097cd4df1ee9 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 3 Oct 2014 16:16:03 -0700 Subject: [PATCH 1/8] Make V2 code more defensive against malformed content Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: e8b8cf61a9c4f905befc3462e7e80ca7729ffe4d Component: engine --- components/engine/graph/pull.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/engine/graph/pull.go b/components/engine/graph/pull.go index 4e7d78f8f1..02d6267879 100644 --- a/components/engine/graph/pull.go +++ b/components/engine/graph/pull.go @@ -39,6 +39,9 @@ func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*re if err := json.Unmarshal(payload, &manifest); err != nil { return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) } + if manifest.SchemaVersion != 1 { + return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) + } var verified bool for _, key := range keys { @@ -454,6 +457,10 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } out.Write(sf.FormatStatus(tag, "Pulling from %s", localName)) + if len(manifest.BlobSums) == 0 { + return fmt.Errorf("no blobSums in manifest") + } + downloads := make([]downloadInfo, len(manifest.BlobSums)) for i := len(manifest.BlobSums) - 1; i >= 0; i-- { @@ -493,6 +500,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) } } else { + defer s.poolRemove("pull", "img:"+img.ID) tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob") if err != nil { return err @@ -513,7 +521,6 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri di.downloaded = true } di.imgJSON = imgJSON - defer s.poolRemove("pull", "img:"+img.ID) return nil } From 045011f555c9a492490d9f40aac9d873198a4b0c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 8 Oct 2014 14:03:39 -0700 Subject: [PATCH 2/8] Use direct registry url Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: a152f37674df3f2a31e60cbfb764fa348333e805 Component: engine --- components/engine/registry/auth.go | 13 ++++++++----- components/engine/registry/session_v2.go | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/components/engine/registry/auth.go b/components/engine/registry/auth.go index 906a37dde7..7c0709a47d 100644 --- a/components/engine/registry/auth.go +++ b/components/engine/registry/auth.go @@ -14,13 +14,16 @@ import ( "github.com/docker/docker/utils" ) -// Where we store the config file -const CONFIGFILE = ".dockercfg" +const ( + // Where we store the config file + CONFIGFILE = ".dockercfg" -// Only used for user auth + account creation -const INDEXSERVER = "https://index.docker.io/v1/" + // Only used for user auth + account creation + INDEXSERVER = "https://index.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v1/" -//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" + // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" +) var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/components/engine/registry/session_v2.go b/components/engine/registry/session_v2.go index 2e0de49bc2..c63cf7100a 100644 --- a/components/engine/registry/session_v2.go +++ b/components/engine/registry/session_v2.go @@ -57,10 +57,14 @@ func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, if err != nil { return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) } + u, err := url.Parse(REGISTRYSERVER) + if err != nil { + return nil, fmt.Errorf("invalid registry url: %s", err) + } return &url.URL{ - Scheme: e.URL.Scheme, - Host: e.URL.Host, + Scheme: u.Scheme, + Host: u.Host, Path: routePath.Path, }, nil } From 3d8d2b7af5f95d335189570261240f4bff13895e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 8 Oct 2014 16:11:04 -0700 Subject: [PATCH 3/8] Update pull message and log Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: 7d21ea6627ae964425da9bc52bbb9e21b7da3cf9 Component: engine --- components/engine/graph/pull.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/engine/graph/pull.go b/components/engine/graph/pull.go index 02d6267879..eec9866fd2 100644 --- a/components/engine/graph/pull.go +++ b/components/engine/graph/pull.go @@ -453,9 +453,10 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } if verified { - out.Write(sf.FormatStatus("", "The image you are pulling has been digitally signed by Docker, Inc.")) + out.Write(sf.FormatStatus(localName+":"+tag, "The image you are pulling has been digitally signed by Docker, Inc.")) + } else { + out.Write(sf.FormatStatus(tag, "Pulling from %s", localName)) } - out.Write(sf.FormatStatus(tag, "Pulling from %s", localName)) if len(manifest.BlobSums) == 0 { return fmt.Errorf("no blobSums in manifest") @@ -489,7 +490,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil)) downloadFunc := func(di *downloadInfo) error { - log.Infof("pulling blob %q to V1 img %s", sumStr, img.ID) + log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID) if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { if c != nil { From c77de695cd6d25b77e97b69cf4d621c63f6af47c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 8 Oct 2014 16:14:54 -0700 Subject: [PATCH 4/8] Add status message for V2 pull Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: 456f493659d92ef271bc3573c537d10ae5d0847b Component: engine --- components/engine/graph/pull.go | 41 +++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/components/engine/graph/pull.go b/components/engine/graph/pull.go index eec9866fd2..97f4d0f578 100644 --- a/components/engine/graph/pull.go +++ b/components/engine/graph/pull.go @@ -416,6 +416,7 @@ type downloadInfo struct { } func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error { + var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", remoteName) tags, err := r.GetV2RemoteTags(remoteName, nil) @@ -423,33 +424,42 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out return err } for _, t := range tags { - if err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil { return err + } else if downloaded { + layersDownloaded = true } } } else { - if err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil { return err + } else if downloaded { + layersDownloaded = true } } + requestedTag := localName + if len(tag) > 0 { + requestedTag = localName + ":" + tag + } + WriteStatus(requestedTag, out, sf, layersDownloaded) return nil } -func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error { +func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) manifestBytes, err := r.GetV2ImageManifest(remoteName, tag, nil) if err != nil { - return err + return false, err } manifest, verified, err := s.verifyManifest(eng, manifestBytes) if err != nil { - return fmt.Errorf("error verifying manifest: %s", err) + return false, fmt.Errorf("error verifying manifest: %s", err) } if len(manifest.BlobSums) != len(manifest.History) { - return fmt.Errorf("length of history not equal to number of layers") + return false, fmt.Errorf("length of history not equal to number of layers") } if verified { @@ -459,7 +469,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } if len(manifest.BlobSums) == 0 { - return fmt.Errorf("no blobSums in manifest") + return false, fmt.Errorf("no blobSums in manifest") } downloads := make([]downloadInfo, len(manifest.BlobSums)) @@ -472,7 +482,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri img, err := image.NewImgJSON(imgJSON) if err != nil { - return fmt.Errorf("failed to parse json: %s", err) + return false, fmt.Errorf("failed to parse json: %s", err) } downloads[i].img = img @@ -484,7 +494,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri chunks := strings.SplitN(sumStr, ":", 2) if len(chunks) < 2 { - return fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks) + return false, fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks) } sumType, checksum := chunks[0], chunks[1] out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil)) @@ -534,17 +544,18 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } else { err := downloadFunc(&downloads[i]) if err != nil { - return err + return false, err } } } + var layersDownloaded bool for i := len(downloads) - 1; i >= 0; i-- { d := &downloads[i] if d.err != nil { err := <-d.err if err != nil { - return err + return false, err } } if d.downloaded { @@ -556,13 +567,13 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri err = s.graph.Register(d.img, d.imgJSON, utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting")) if err != nil { - return err + return false, err } // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) } out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil)) - + layersDownloaded = true } else { out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil)) } @@ -570,8 +581,8 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } if err = s.Set(localName, tag, downloads[0].img.ID, true); err != nil { - return err + return false, err } - return nil + return layersDownloaded, nil } From 5ab2b8482f9e1ea14dc8765a1e566951369be160 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:34:34 -0700 Subject: [PATCH 5/8] Update manifest format to rename blobsums and use arrays of dictionaries Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: 15d5c7f10f56faeaa04f6374341e1cfd3e03cab1 Component: engine --- components/engine/graph/pull.go | 12 ++++++------ components/engine/registry/types.go | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/components/engine/graph/pull.go b/components/engine/graph/pull.go index 97f4d0f578..39e8926c8a 100644 --- a/components/engine/graph/pull.go +++ b/components/engine/graph/pull.go @@ -458,7 +458,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return false, fmt.Errorf("error verifying manifest: %s", err) } - if len(manifest.BlobSums) != len(manifest.History) { + if len(manifest.FSLayers) != len(manifest.History) { return false, fmt.Errorf("length of history not equal to number of layers") } @@ -468,16 +468,16 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri out.Write(sf.FormatStatus(tag, "Pulling from %s", localName)) } - if len(manifest.BlobSums) == 0 { + if len(manifest.FSLayers) == 0 { return false, fmt.Errorf("no blobSums in manifest") } - downloads := make([]downloadInfo, len(manifest.BlobSums)) + downloads := make([]downloadInfo, len(manifest.FSLayers)) - for i := len(manifest.BlobSums) - 1; i >= 0; i-- { + for i := len(manifest.FSLayers) - 1; i >= 0; i-- { var ( - sumStr = manifest.BlobSums[i] - imgJSON = []byte(manifest.History[i]) + sumStr = manifest.FSLayers[i].BlobSum + imgJSON = []byte(manifest.History[i].V1Compatibility) ) img, err := image.NewImgJSON(imgJSON) diff --git a/components/engine/registry/types.go b/components/engine/registry/types.go index 2ba5af0da4..3b429f19af 100644 --- a/components/engine/registry/types.go +++ b/components/engine/registry/types.go @@ -32,13 +32,21 @@ type RegistryInfo struct { Standalone bool `json:"standalone"` } +type FSLayer struct { + BlobSum string `json:"blobSum"` +} + +type ManifestHistory struct { + V1Compatibility string `json:"v1Compatibility"` +} + type ManifestData struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - BlobSums []string `json:"blobSums"` - History []string `json:"history"` - SchemaVersion int `json:"schemaVersion"` + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + FSLayers []*FSLayer `json:"fsLayers"` + History []*ManifestHistory `json:"history"` + SchemaVersion int `json:"schemaVersion"` } type APIVersion int From 0ae83cbb52be7017ea74dc1702be34a54358e18e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:31:54 -0700 Subject: [PATCH 6/8] Support tarsum dev version to fix issue with mtime Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: 73a9089306ac1ed25c594e54c9b2a11a0ac8e610 Component: engine --- components/engine/registry/session_v2.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/registry/session_v2.go b/components/engine/registry/session_v2.go index c63cf7100a..c0bc19b337 100644 --- a/components/engine/registry/session_v2.go +++ b/components/engine/registry/session_v2.go @@ -28,13 +28,13 @@ func newV2RegistryRouter() *mux.Router { v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") return router } From e2eeab4dd30d57ac17465be57a6743f34d914584 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 10 Oct 2014 15:20:28 -0700 Subject: [PATCH 7/8] Update verification message Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: 85fd8213afe9497da5df5583d776659ed187c9e9 Component: engine --- components/engine/graph/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/graph/pull.go b/components/engine/graph/pull.go index 39e8926c8a..327716a2f2 100644 --- a/components/engine/graph/pull.go +++ b/components/engine/graph/pull.go @@ -463,7 +463,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } if verified { - out.Write(sf.FormatStatus(localName+":"+tag, "The image you are pulling has been digitally signed by Docker, Inc.")) + out.Write(sf.FormatStatus(localName+":"+tag, "The image you are pulling has been verified")) } else { out.Write(sf.FormatStatus(tag, "Pulling from %s", localName)) } From e2a55f16e843ad14116839feaf099a8de58abcc6 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 7 Oct 2014 11:00:17 -0700 Subject: [PATCH 8/8] Enable V2 pull flow Signed-off-by: Derek McGowan (github: dmcgowan) Upstream-commit: 3be4551a9164c5fac577e9e3683dea65c2fc193a Component: engine --- components/engine/graph/pull.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/components/engine/graph/pull.go b/components/engine/graph/pull.go index 327716a2f2..5d7e84ed72 100644 --- a/components/engine/graph/pull.go +++ b/components/engine/graph/pull.go @@ -137,6 +137,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { mirrors = s.mirrors } + if isOfficial || endpoint.Version == registry.APIVersion2 { + j := job.Eng.Job("trust_update_base") + if err = j.Run(); err != nil { + return job.Errorf("error updating trust base graph: %s", err) + } + + if err := s.pullV2Repository(job.Eng, r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err == nil { + return engine.StatusOK + } else if err != registry.ErrDoesNotExist { + log.Errorf("Error from V2 registry: %s", err) + } + } + if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil { return job.Error(err) }