From e0c6d66dac75953bc523b94faa9572a180e3dc0d Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Tue, 25 Jun 2013 15:31:22 -0700 Subject: [PATCH 01/14] Bugfixes on docs * fixed canonical link from index * added http redirect from builder/basics * fixed url in redirect_home Upstream-commit: 73afc6311daf6bce10d081352aa1efbfcb406d10 Component: engine --- components/engine/docs/sources/conf.py | 1 + components/engine/docs/theme/docker/layout.html | 5 ++++- .../engine/docs/theme/docker/redirect_build.html | 12 ++++++++++++ .../engine/docs/theme/docker/redirect_home.html | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 components/engine/docs/theme/docker/redirect_build.html diff --git a/components/engine/docs/sources/conf.py b/components/engine/docs/sources/conf.py index 41dba70201..ea8e1f43f0 100644 --- a/components/engine/docs/sources/conf.py +++ b/components/engine/docs/sources/conf.py @@ -30,6 +30,7 @@ import sys, os html_additional_pages = { 'concepts/containers': 'redirect_home.html', 'concepts/introduction': 'redirect_home.html', + 'builder/basics': 'redirect_build.html', } diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index 707888a927..f4f17f2dba 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -40,8 +40,11 @@ {%- set script_files = script_files + ['_static/js/docs.js'] %} + {%- if pagename == 'index' %} + + {% else %} - + {% endif %} {%- for cssfile in css_files %} {%- endfor %} diff --git a/components/engine/docs/theme/docker/redirect_build.html b/components/engine/docs/theme/docker/redirect_build.html new file mode 100644 index 0000000000..1f26fc3aaa --- /dev/null +++ b/components/engine/docs/theme/docker/redirect_build.html @@ -0,0 +1,12 @@ + + + + Page Moved + + + + +This page has moved. Perhaps you should visit the Builder page + + + diff --git a/components/engine/docs/theme/docker/redirect_home.html b/components/engine/docs/theme/docker/redirect_home.html index 41f8da244d..109239f819 100644 --- a/components/engine/docs/theme/docker/redirect_home.html +++ b/components/engine/docs/theme/docker/redirect_home.html @@ -2,7 +2,7 @@ Page Moved - + From e593de79dcabde82652027e4d0c9427039f152b0 Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Thu, 27 Jun 2013 01:33:55 -0300 Subject: [PATCH 02/14] following the 'go vet' suggestions. Upstream-commit: 3d5633a0a0eb98bbd833e760279720a15ced8b58 Component: engine --- components/engine/api_test.go | 14 +++++++------- components/engine/commands.go | 8 ++++---- components/engine/graph.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 12ea5ee962..b077ba12a6 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -19,25 +19,25 @@ import ( func TestGetBoolParam(t *testing.T) { if ret, err := getBoolParam("true"); err != nil || !ret { - t.Fatalf("true -> true, nil | got %b %s", ret, err) + t.Fatalf("true -> true, nil | got %t %s", ret, err) } if ret, err := getBoolParam("True"); err != nil || !ret { - t.Fatalf("True -> true, nil | got %b %s", ret, err) + t.Fatalf("True -> true, nil | got %t %s", ret, err) } if ret, err := getBoolParam("1"); err != nil || !ret { - t.Fatalf("1 -> true, nil | got %b %s", ret, err) + t.Fatalf("1 -> true, nil | got %t %s", ret, err) } if ret, err := getBoolParam(""); err != nil || ret { - t.Fatalf("\"\" -> false, nil | got %b %s", ret, err) + t.Fatalf("\"\" -> false, nil | got %t %s", ret, err) } if ret, err := getBoolParam("false"); err != nil || ret { - t.Fatalf("false -> false, nil | got %b %s", ret, err) + t.Fatalf("false -> false, nil | got %t %s", ret, err) } if ret, err := getBoolParam("0"); err != nil || ret { - t.Fatalf("0 -> false, nil | got %b %s", ret, err) + t.Fatalf("0 -> false, nil | got %t %s", ret, err) } if ret, err := getBoolParam("faux"); err == nil || ret { - t.Fatalf("faux -> false, err | got %b %s", ret, err) + t.Fatalf("faux -> false, err | got %t %s", ret, err) } } diff --git a/components/engine/commands.go b/components/engine/commands.go index 7175c07b0b..b21c8bbd7b 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -345,7 +345,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } auth.SaveConfig(cli.authConfig) if out2.Status != "" { - fmt.Fprintln(cli.out, "%s\n", out2.Status) + fmt.Fprintf(cli.out, "%s\n", out2.Status) } return nil } @@ -370,7 +370,7 @@ func (cli *DockerCli) CmdWait(args ...string) error { if err != nil { return err } - fmt.Fprintf(cli.out, "%s\n", out.StatusCode) + fmt.Fprintf(cli.out, "%d\n", out.StatusCode) } } return nil @@ -405,7 +405,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { fmt.Fprintf(cli.out, "Git commit: %s\n", out.GitCommit) } if out.GoVersion != "" { - fmt.Fprintln(cli.out, "Go version: %s\n", out.GoVersion) + fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } return nil } @@ -513,7 +513,7 @@ func (cli *DockerCli) CmdStart(args ...string) error { if err != nil { fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Fprintln(cli.out, "%s\n", name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil diff --git a/components/engine/graph.go b/components/engine/graph.go index f2b2ccec8e..0bf7eccdbe 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -189,7 +189,7 @@ func (graph *Graph) Mktemp(id string) (string, error) { return "", fmt.Errorf("Couldn't create temp: %s", err) } if tmp.Exists(id) { - return "", fmt.Errorf("Image %d already exists", id) + return "", fmt.Errorf("Image %s already exists", id) } return tmp.imageRoot(id), nil } From d68f44e1014d87b0d4bdb432d3b1883c9a4768aa Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Thu, 27 Jun 2013 01:40:13 -0300 Subject: [PATCH 03/14] following 'go vet' in utils pkg. Upstream-commit: 6cf493bea74e9caa05e1da9e970c47d1f24268d8 Component: engine --- components/engine/utils/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index d5e44ee150..eafe065a92 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -87,7 +87,7 @@ func (r *progressReader) Read(p []byte) (n int, err error) { } if r.readProgress-r.lastUpdate > updateEvery || err != nil { if r.readTotal > 0 { - fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%2.0f%%",float64(r.readProgress)/float64(r.readTotal)*100)) + fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%2.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) } else { fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a") } @@ -133,7 +133,7 @@ func HumanDuration(d time.Duration) string { } else if hours < 24*365*2 { return fmt.Sprintf("%d months", hours/24/30) } - return fmt.Sprintf("%d years", d.Hours()/24/365) + return fmt.Sprintf("%f years", d.Hours()/24/365) } // HumanSize returns a human-readable approximation of a size From 1f3a1ee5cad463799527d5ab126257e313215712 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 24 May 2013 10:37:34 -0700 Subject: [PATCH 04/14] Tentative support for independent registries Upstream-commit: de0a48bd6ff2acfb2ab546a1e718185872eea2c3 Component: engine --- components/engine/registry/registry.go | 5 +- components/engine/server.go | 79 +++++++++++++++++++++----- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index c565c29989..5d642b392f 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -56,20 +56,19 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s } // Check if an image exists in the Registry -func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { +func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } - req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) if err != nil { return false } res.Body.Close() - return res.StatusCode == 307 + return res.StatusCode == 200 } func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { diff --git a/components/engine/server.go b/components/engine/server.go index cdc01c6b09..615881cf78 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -351,17 +351,32 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag string, sf *utils.StreamFormatter) error { +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error { out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress())) repoData, err := r.GetRepositoryData(remote) if err != nil { return err } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err + var repoData *registry.RepositoryData + var err error + if registryEp == "" { + repoData, err = srv.registry.GetRepositoryData(remote) + if err != nil { + return err + } + + utils.Debugf("Updating checksums") + // Reload the json file to make sure not to overwrite faster sums + if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { + return err + } + } else { + repoData = ®istry.RepositoryData{ + Tokens: []string{}, + ImgList: make(map[string]*registry.ImgData), + Endpoints: []string{registryEp}, + } } utils.Debugf("Retrieving the tag list") @@ -369,8 +384,19 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re if err != nil { return err } + + if registryEp != "" { + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + Id: id, + Tag: tag, + Checksum: "", + } + } + } + utils.Debugf("Registering tags") - // If not specific tag have been asked, take all + // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag @@ -547,7 +573,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return imgList, nil } -func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) out.Write(sf.FormatStatus("Processing checksums")) imgList, err := srv.getImageList(localRepo) @@ -555,16 +581,35 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri return err } out.Write(sf.FormatStatus("Sending image list")) - srvName := name parts := strings.Split(name, "/") if len(parts) > 2 { srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } - repoData, err := r.PushImageJSONIndex(srvName, imgList, false, nil) - if err != nil { - return err + var repoData *registry.RepositoryData + if registryEp == "" { + repoData, err = srv.registry.PushImageJsonIndex(name, imgList, false) + if err != nil { + return err + } + } else { + repoData = ®istry.RepositoryData{ + ImgList: make(map[string]*registry.ImgData), + Tokens: []string{}, + Endpoints: []string{registryEp}, + } + tagsList, err := srv.registry.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) + if err != nil { + return err + } + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + Id: id, + Tag: tag, + Checksum: "", + } + } } for _, ep := range repoData.Endpoints { @@ -574,6 +619,9 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri if _, exists := repoData.ImgList[elem.ID]; exists { out.Write(sf.FormatStatus("Image %s already on registry, skipping", name)) continue + } else if registryEp != "" && srv.registry.LookupRemoteImage(elem.Id, registryEp, repoData.Tokens) { + fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) + continue } if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? @@ -586,9 +634,12 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri } } - if _, err := r.PushImageJSONIndex(srvName, imgList, true, repoData.Endpoints); err != nil { - return err + if registryEp == "" { + if _, err := srv.registry.PushImageJsonIndex(name, imgList, true); err != nil { + return err + } } + return nil } @@ -669,7 +720,7 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(r, out, name, localRepo, sf); err != nil { + if err := srv.pushRepository(r, out, name, registry, localRepo, sf); err != nil { return err } return nil From 6af1e08d14304523c9c8f5702e9e5eb648e56a5f Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 29 May 2013 11:24:50 -0700 Subject: [PATCH 05/14] Skip certificate check (don't error out on self-signed certs) Upstream-commit: 1c817913eee7d87badc2a2748447729619d8aaf0 Component: engine --- components/engine/registry/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 5d642b392f..864f3e8bfe 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -2,6 +2,7 @@ package registry import ( "bytes" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -164,6 +165,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() From a944a8deb62b3171b20cb0d367e61640992cc25e Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 29 May 2013 11:25:49 -0700 Subject: [PATCH 06/14] Ignore 'registry not found' when pushing on independent registries Upstream-commit: 2a1f8f6fda55a5fa2132dbad374043e9fbf6d9f6 Component: engine --- components/engine/server.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index 615881cf78..9b62b768c1 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -600,14 +600,15 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg Endpoints: []string{registryEp}, } tagsList, err := srv.registry.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) - if err != nil { + if err != nil && err.Error() != "Repository not found" { return err - } - for tag, id := range tagsList { - repoData.ImgList[id] = ®istry.ImgData{ - Id: id, - Tag: tag, - Checksum: "", + } else if err == nil { + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + Id: id, + Tag: tag, + Checksum: "", + } } } } From 3635092ee9df90a4b9480f5fbc3afd02f745cf1c Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 29 May 2013 11:39:31 -0700 Subject: [PATCH 07/14] Rolled back of previous commit (skip cert verification) Upstream-commit: 50075106b658ab4dc82d5361e4786f7d28cc570a Component: engine --- components/engine/registry/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 864f3e8bfe..30b91cef92 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -2,7 +2,6 @@ package registry import ( "bytes" - "crypto/tls" "encoding/json" "errors" "fmt" From 01a345c7fb7d8ac7872ecde2b16082db2cb027f9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 5 Jun 2013 15:12:50 -0700 Subject: [PATCH 08/14] Remove https prefix from registry Upstream-commit: 57d751c3770194ff867ebace36eb1be7d72d2fe9 Component: engine --- components/engine/registry/registry.go | 8 ++++---- components/engine/server.go | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 30b91cef92..29a1f29e1f 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -154,7 +154,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -249,7 +249,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = registry + "/v1" // FIXME: try json with UTF8 req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { @@ -285,7 +285,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = registry + "/v1" req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err @@ -323,7 +323,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = "https://" + registry + "/v1" + registry = registry + "/v1" req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { diff --git a/components/engine/server.go b/components/engine/server.go index 9b62b768c1..7f866d626b 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -418,7 +418,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote)) success := false for _, ep := range repoData.Endpoints { - if err := srv.pullImage(r, out, img.ID, "https://"+ep+"/v1", repoData.Tokens, sf); err != nil { + if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil { out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue } @@ -717,6 +717,7 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str if err2 != nil { return err2 } + if err != nil { out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) // If it fails, try to get the repository From d5e7a7e8e9799a17ae73895ca38f9a21d54ffbea Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 5 Jun 2013 15:13:01 -0700 Subject: [PATCH 09/14] Reenable writeflusher for pull Upstream-commit: 67ecd2cb822894e4d35adea031e5b79d7ca8e91a Component: engine --- components/engine/server.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index 7f866d626b..457f1c02dc 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -444,6 +444,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re return nil } + func (srv *Server) poolAdd(kind, key string) error { srv.lock.Lock() defer srv.lock.Unlock() @@ -478,7 +479,6 @@ func (srv *Server) poolRemove(kind, key string) error { } return nil } - func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig) if err != nil { @@ -489,21 +489,21 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util } defer srv.poolRemove("pull", name+":"+tag) - out = utils.NewWriteFlusher(out) - if endpoint != "" { - if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { - return err - } - return nil - } remote := name parts := strings.Split(name, "/") if len(parts) > 2 { remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } - if err := srv.pullRepository(r, out, name, remote, tag, sf); err != nil { - return err + out = utils.NewWriteFlusher(out) + + err := srv.pullRepository(r, out, name, tag, endpoint, sf) + if err != nil && endpoint != "" { + if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { + return err + } + return nil } + return nil } From 4eec06d21726cf0e78a1212e8c0d02caa1a17474 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 25 Jun 2013 21:03:42 +0200 Subject: [PATCH 10/14] Several fixes and updates to make this work with latest changes in master Upstream-commit: c3dd6e1926f9359f840a9f7b0be9cfa7fe2cfea2 Component: engine --- components/engine/server.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index 457f1c02dc..9f9dcb7731 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -353,15 +353,11 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error { out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress())) - repoData, err := r.GetRepositoryData(remote) - if err != nil { - return err - } var repoData *registry.RepositoryData var err error if registryEp == "" { - repoData, err = srv.registry.GetRepositoryData(remote) + repoData, err = r.GetRepositoryData(remote) if err != nil { return err } @@ -382,13 +378,14 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) if err != nil { + utils.Debugf("%v", err) return err } if registryEp != "" { for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ - Id: id, + ID: id, Tag: tag, Checksum: "", } @@ -418,6 +415,9 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote)) success := false for _, ep := range repoData.Endpoints { + if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { + ep = "https://" + ep + } if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil { out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue @@ -495,8 +495,7 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } out = utils.NewWriteFlusher(out) - - err := srv.pullRepository(r, out, name, tag, endpoint, sf) + err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf) if err != nil && endpoint != "" { if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { return err @@ -589,7 +588,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg var repoData *registry.RepositoryData if registryEp == "" { - repoData, err = srv.registry.PushImageJsonIndex(name, imgList, false) + repoData, err = r.PushImageJSONIndex(name, imgList, false, nil) if err != nil { return err } @@ -599,13 +598,13 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg Tokens: []string{}, Endpoints: []string{registryEp}, } - tagsList, err := srv.registry.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) + tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) if err != nil && err.Error() != "Repository not found" { return err } else if err == nil { for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ - Id: id, + ID: id, Tag: tag, Checksum: "", } @@ -614,13 +613,16 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg } for _, ep := range repoData.Endpoints { + if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { + ep = "https://" + ep + } out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { out.Write(sf.FormatStatus("Image %s already on registry, skipping", name)) continue - } else if registryEp != "" && srv.registry.LookupRemoteImage(elem.Id, registryEp, repoData.Tokens) { + } else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) { fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) continue } @@ -636,7 +638,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg } if registryEp == "" { - if _, err := srv.registry.PushImageJsonIndex(name, imgList, true); err != nil { + if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil { return err } } @@ -722,7 +724,7 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(r, out, name, registry, localRepo, sf); err != nil { + if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil { return err } return nil From 0081ddc5450dc7a209ac0cf348f5e528fcb6c575 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 27 Jun 2013 19:17:20 +0200 Subject: [PATCH 11/14] Allow push by ID when using a custom registry Upstream-commit: e095a1572fb8a2ba17ea231abcb3704cf83a96e4 Component: engine --- components/engine/commands.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 6ada02d552..4ef63f4a22 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -736,23 +736,30 @@ func (cli *DockerCli) CmdPush(args ...string) error { return err } - if len(strings.SplitN(name, "/", 2)) == 1 { - return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) + if *registry == "" { + // If we're not using a custom registry, we know the restrictions + // applied to repository names and can warn the user in advance. + // Custom repositories can have different rules, and we must also + // allow pushing by image ID. + if len(strings.SplitN(name, "/", 2)) == 1 { + return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) + } + + nameParts := strings.SplitN(name, "/", 2) + validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + if !validNamespace.MatchString(nameParts[0]) { + return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) + } + validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + if !validRepo.MatchString(nameParts[1]) { + return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) + } } buf, err := json.Marshal(cli.authConfig) if err != nil { return err } - nameParts := strings.SplitN(name, "/", 2) - validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) - if !validNamespace.MatchString(nameParts[0]) { - return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) - } - validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) - if !validRepo.MatchString(nameParts[1]) { - return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) - } v := url.Values{} v.Set("registry", *registry) From c2894b0393dc0ef790cf292077188994d450ce17 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 28 Jun 2013 18:16:19 +0200 Subject: [PATCH 12/14] go fmt pass Upstream-commit: ad2f826a825be2889057a773eff1a2bc605e8e80 Component: engine --- components/engine/server.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index 9f9dcb7731..0e5aa97411 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -369,8 +369,8 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re } } else { repoData = ®istry.RepositoryData{ - Tokens: []string{}, - ImgList: make(map[string]*registry.ImgData), + Tokens: []string{}, + ImgList: make(map[string]*registry.ImgData), Endpoints: []string{registryEp}, } } @@ -385,8 +385,8 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re if registryEp != "" { for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ - ID: id, - Tag: tag, + ID: id, + Tag: tag, Checksum: "", } } @@ -444,7 +444,6 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re return nil } - func (srv *Server) poolAdd(kind, key string) error { srv.lock.Lock() defer srv.lock.Unlock() @@ -594,8 +593,8 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg } } else { repoData = ®istry.RepositoryData{ - ImgList: make(map[string]*registry.ImgData), - Tokens: []string{}, + ImgList: make(map[string]*registry.ImgData), + Tokens: []string{}, Endpoints: []string{registryEp}, } tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) @@ -604,8 +603,8 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, reg } else if err == nil { for tag, id := range tagsList { repoData.ImgList[id] = ®istry.ImgData{ - ID: id, - Tag: tag, + ID: id, + Tag: tag, Checksum: "", } } From 58057e83720c392e9ae62587ef690d0303fdac07 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Fri, 28 Jun 2013 09:21:34 -0700 Subject: [PATCH 13/14] testing|stabilization, issue 973: Use docker-golang PPA and lts-raring kernel Upstream-commit: af2a92f22b053d7c04a4416da87523d4296e103c Component: engine --- components/engine/testing/Vagrantfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/engine/testing/Vagrantfile b/components/engine/testing/Vagrantfile index f2f6ca8248..e3b25a6f9d 100644 --- a/components/engine/testing/Vagrantfile +++ b/components/engine/testing/Vagrantfile @@ -19,9 +19,10 @@ Vagrant::Config.run do |config| config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." config.vm.network :hostonly, BUILDBOT_IP + # Deploy buildbot and its dependencies if it was not done if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? - pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " + pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; " # Deploy buildbot CI pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ "pip install -r #{CFG_PATH}/requirements.txt; " \ @@ -29,7 +30,7 @@ Vagrant::Config.run do |config| "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ - "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ + "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; " # Activate new kernel pkg_cmd << "shutdown -r +1; " @@ -40,6 +41,7 @@ end # Providers were added on Vagrant >= 1.1.0 Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| config.vm.provider :aws do |aws, override| + aws.tags = { 'Name' => 'docker-ci' } aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"] aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"] aws.keypair_name = ENV["AWS_KEYPAIR_NAME"] From 8c9ba170e13cf0aae97d0b196171706451bacda8 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 28 Jun 2013 18:42:37 +0200 Subject: [PATCH 14/14] Fixed issue in registry.GetRemoteTags Upstream-commit: 72befeef24b3fb0c77f56e3ac2bbfdb4744cbd7a Component: engine --- components/engine/registry/registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 29a1f29e1f..456e432198 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -155,6 +155,9 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } for _, host := range registries { endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) + if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { + endpoint = "https://" + endpoint + } req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err