toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #8393 from dmcgowan/provenance_pull_enhance

Browse Source 
Make V2 code more defensive against malformed content
Upstream-commit: 9f482a66ab37ec396ac61ed0c00d59122ac07453
Component: engine
This commit is contained in:
Jessie Frazelle
2014-10-10 17:19:59 -07:00
parent d2f7355324 e2a55f16e8
commit 29fdd38079
4 changed files with 86 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/engine/graph/pull.go
+55 -23
View File
@@ -39,6 +39,9 @@ func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*re
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
}
if manifest.SchemaVersion != 1 {
return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion)
}
var verified bool
for _, key := range keys {
@@ -134,6 +137,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
mirrors = s.mirrors
}
if isOfficial || endpoint.Version == registry.APIVersion2 {
j := job.Eng.Job("trust_update_base")
if err = j.Run(); err != nil {
return job.Errorf("error updating trust base graph: %s", err)
}
if err := s.pullV2Repository(job.Eng, r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err == nil {
return engine.StatusOK
} else if err != registry.ErrDoesNotExist {
log.Errorf("Error from V2 registry: %s", err)
}
}
if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
return job.Error(err)
}
@@ -413,6 +429,7 @@ type downloadInfo struct {
}
func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
var layersDownloaded bool
if tag == "" {
log.Debugf("Pulling tag list from V2 registry for %s", remoteName)
tags, err := r.GetV2RemoteTags(remoteName, nil)
@@ -420,51 +437,65 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out
return err
}
for _, t := range tags {
if err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil {
if downloaded, err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil {
return err
} else if downloaded {
layersDownloaded = true
}
}
} else {
if err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil {
if downloaded, err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil {
return err
} else if downloaded {
layersDownloaded = true
}
}
requestedTag := localName
if len(tag) > 0 {
requestedTag = localName + ":" + tag
}
WriteStatus(requestedTag, out, sf, layersDownloaded)
return nil
}
func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) (bool, error) {
log.Debugf("Pulling tag from V2 registry: %q", tag)
manifestBytes, err := r.GetV2ImageManifest(remoteName, tag, nil)
if err != nil {
return err
return false, err
}
manifest, verified, err := s.verifyManifest(eng, manifestBytes)
if err != nil {
return fmt.Errorf("error verifying manifest: %s", err)
return false, fmt.Errorf("error verifying manifest: %s", err)
}
if len(manifest.BlobSums) != len(manifest.History) {
return fmt.Errorf("length of history not equal to number of layers")
if len(manifest.FSLayers) != len(manifest.History) {
return false, fmt.Errorf("length of history not equal to number of layers")
}
if verified {
out.Write(sf.FormatStatus("", "The image you are pulling has been digitally signed by Docker, Inc."))
out.Write(sf.FormatStatus(localName+":"+tag, "The image you are pulling has been verified"))
} else {
out.Write(sf.FormatStatus(tag, "Pulling from %s", localName))
}
out.Write(sf.FormatStatus(tag, "Pulling from %s", localName))
downloads := make([]downloadInfo, len(manifest.BlobSums))
if len(manifest.FSLayers) == 0 {
return false, fmt.Errorf("no blobSums in manifest")
}
for i := len(manifest.BlobSums) - 1; i >= 0; i-- {
downloads := make([]downloadInfo, len(manifest.FSLayers))
for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
var (
sumStr = manifest.BlobSums[i]
imgJSON = []byte(manifest.History[i])
sumStr = manifest.FSLayers[i].BlobSum
imgJSON = []byte(manifest.History[i].V1Compatibility)
)
img, err := image.NewImgJSON(imgJSON)
if err != nil {
return fmt.Errorf("failed to parse json: %s", err)
return false, fmt.Errorf("failed to parse json: %s", err)
}
downloads[i].img = img
@@ -476,13 +507,13 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
chunks := strings.SplitN(sumStr, ":", 2)
if len(chunks) < 2 {
return fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)
return false, fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)
}
sumType, checksum := chunks[0], chunks[1]
out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil))
downloadFunc := func(di *downloadInfo) error {
log.Infof("pulling blob %q to V1 img %s", sumStr, img.ID)
log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID)
if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
if c != nil {
@@ -493,6 +524,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
}
} else {
defer s.poolRemove("pull", "img:"+img.ID)
tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
if err != nil {
return err
@@ -513,7 +545,6 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
di.downloaded = true
}
di.imgJSON = imgJSON
defer s.poolRemove("pull", "img:"+img.ID)
return nil
}
@@ -526,17 +557,18 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
} else {
err := downloadFunc(&downloads[i])
if err != nil {
return err
return false, err
}
}
}
var layersDownloaded bool
for i := len(downloads) - 1; i >= 0; i-- {
d := &downloads[i]
if d.err != nil {
err := <-d.err
if err != nil {
return err
return false, err
}
}
if d.downloaded {
@@ -548,13 +580,13 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
err = s.graph.Register(d.img, d.imgJSON,
utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting"))
if err != nil {
return err
return false, err
}
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
}
out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil))
layersDownloaded = true
} else {
out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil))
}
@@ -562,8 +594,8 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
}
if err = s.Set(localName, tag, downloads[0].img.ID, true); err != nil {
return err
return false, err
}
return nil
return layersDownloaded, nil
}
components/engine/registry/auth.go
+8 -5
View File
@@ -14,13 +14,16 @@ import (
"github.com/docker/docker/utils"
)
// Where we store the config file
const CONFIGFILE = ".dockercfg"
const (
// Where we store the config file
CONFIGFILE = ".dockercfg"
// Only used for user auth + account creation
const INDEXSERVER = "https://index.docker.io/v1/"
// Only used for user auth + account creation
INDEXSERVER = "https://index.docker.io/v1/"
REGISTRYSERVER = "https://registry-1.docker.io/v1/"
//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/"
// INDEXSERVER = "https://registry-stage.hub.docker.com/v1/"
)
var (
ErrConfigFileMissing = errors.New("The Auth config file is missing")
components/engine/registry/session_v2.go
+9 -5
View File
@@ -28,13 +28,13 @@ func newV2RegistryRouter() *mux.Router {
v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags")
// Download a blob
v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")
v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")
// Upload a blob
v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob")
v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob")
// Mounting a blob in an image
v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")
v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")
return router
}
@@ -57,10 +57,14 @@ func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL,
if err != nil {
return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err)
}
u, err := url.Parse(REGISTRYSERVER)
if err != nil {
return nil, fmt.Errorf("invalid registry url: %s", err)
}
return &url.URL{
Scheme: e.URL.Scheme,
Host: e.URL.Host,
Scheme: u.Scheme,
Host: u.Host,
Path: routePath.Path,
}, nil
}
components/engine/registry/types.go
+14 -6
View File
@@ -32,13 +32,21 @@ type RegistryInfo struct {
Standalone bool `json:"standalone"`
}
type FSLayer struct {
BlobSum string `json:"blobSum"`
}
type ManifestHistory struct {
V1Compatibility string `json:"v1Compatibility"`
}
type ManifestData struct {
Name string `json:"name"`
Tag string `json:"tag"`
Architecture string `json:"architecture"`
BlobSums []string `json:"blobSums"`
History []string `json:"history"`
SchemaVersion int `json:"schemaVersion"`
Name string `json:"name"`
Tag string `json:"tag"`
Architecture string `json:"architecture"`
FSLayers []*FSLayer `json:"fsLayers"`
History []*ManifestHistory `json:"history"`
SchemaVersion int `json:"schemaVersion"`
}
type APIVersion int